projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'perf-tools-fixes-for-v6.17-2025-09-16' of git://git.kernel.org/pub/scm...
[linux-2.6-block.git] / fs / f2fs / node.c
CommitLineData
7c1a000d 1// SPDX-License-Identifier: GPL-2.0
0a8165d7 2/*
e05df3b1
JK		 3 * fs/f2fs/node.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
e05df3b1
JK		 7 */
8#include <linux/fs.h>
9#include <linux/f2fs_fs.h>
10#include <linux/mpage.h>
4034247a 11#include <linux/sched/mm.h>
e05df3b1
JK		 12#include <linux/blkdev.h>
13#include <linux/pagevec.h>
14#include <linux/swap.h>
15
16#include "f2fs.h"
17#include "node.h"
18#include "segment.h"
87905682 19#include "xattr.h"
52118743 20#include "iostat.h"
51dd6249 21#include <trace/events/f2fs.h>
e05df3b1 22
d1e1ff97 23#define on_f2fs_build_free_nids(nm_i) mutex_is_locked(&(nm_i)->build_lock)
f978f5a0 24
e05df3b1
JK		 25static struct kmem_cache *nat_entry_slab;
26static struct kmem_cache *free_nid_slab;
aec71382 27static struct kmem_cache *nat_entry_set_slab;
50fa53ec 28static struct kmem_cache *fsync_node_entry_slab;
e05df3b1 29
a4f843bd
JK		 30/*
31 * Check whether the given nid is within node id range.
32 */
4d57b86d 33int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid)
a4f843bd
JK		 34{
35 if (unlikely(nid < F2FS_ROOT_INO(sbi) || nid >= NM_I(sbi)->max_nid)) {
36 set_sbi_flag(sbi, SBI_NEED_FSCK);
dcbb4c10
JP		 37 f2fs_warn(sbi, "%s: out-of-range nid=%x, run fsck to fix.",
38 __func__, nid);
95fa90c9 39 f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE);
10f966bb 40 return -EFSCORRUPTED;
a4f843bd
JK		 41 }
42 return 0;
43}
44
4d57b86d 45bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type)
cdfc41c1 46{
6fb03f3a 47 struct f2fs_nm_info *nm_i = NM_I(sbi);
d6d2b491 48 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
cdfc41c1 49 struct sysinfo val;
e5e7ea3c 50 unsigned long avail_ram;
cdfc41c1 51 unsigned long mem_size = 0;
6fb03f3a 52 bool res = false;
cdfc41c1 53
d6d2b491
ST		 54 if (!nm_i)
55 return true;
56
cdfc41c1 57 si_meminfo(&val);
e5e7ea3c
JK		 58
59 /* only uses low memory */
60 avail_ram = val.totalram - val.totalhigh;
61
429511cd 62 /*
71644dff 63 * give 25%, 25%, 50%, 50%, 25%, 25% memory for each components respectively
429511cd 64 */
6fb03f3a 65 if (type == FREE_NIDS) {
9a4ffdf5 66 mem_size = (nm_i->nid_cnt[FREE_NID] *
b8559dc2 67 sizeof(struct free_nid)) >> PAGE_SHIFT;
e5e7ea3c 68 res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);
6fb03f3a 69 } else if (type == NAT_ENTRIES) {
a95ba66a
JK		 70 mem_size = (nm_i->nat_cnt[TOTAL_NAT] *
71 sizeof(struct nat_entry)) >> PAGE_SHIFT;
e5e7ea3c 72 res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);
e589c2c4
JK		 73 if (excess_cached_nats(sbi))
74 res = false;
a1257023
JK		 75 } else if (type == DIRTY_DENTS) {
76 if (sbi->sb->s_bdi->wb.dirty_exceeded)
77 return false;
78 mem_size = get_pages(sbi, F2FS_DIRTY_DENTS);
79 res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);
e5e7ea3c
JK		 80 } else if (type == INO_ENTRIES) {
81 int i;
82
39d787be 83 for (i = 0; i < MAX_INO_ENTRY; i++)
8f73cbb7
KM		 84 mem_size += sbi->im[i].ino_num *
85 sizeof(struct ino_entry);
86 mem_size >>= PAGE_SHIFT;
e5e7ea3c 87 res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);
71644dff
JK		 88 } else if (type == READ_EXTENT_CACHE || type == AGE_EXTENT_CACHE) {
89 enum extent_type etype = type == READ_EXTENT_CACHE ?
90 EX_READ : EX_BLOCK_AGE;
91 struct extent_tree_info *eti = &sbi->extent_tree[etype];
e7547dac
JK		 92
93 mem_size = (atomic_read(&eti->total_ext_tree) *
7441ccef 94 sizeof(struct extent_tree) +
e7547dac 95 atomic_read(&eti->total_ext_node) *
09cbfeaf 96 sizeof(struct extent_node)) >> PAGE_SHIFT;
71644dff 97 res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);
d6d2b491
ST		 98 } else if (type == DISCARD_CACHE) {
99 mem_size = (atomic_read(&dcc->discard_cmd_cnt) *
100 sizeof(struct discard_cmd)) >> PAGE_SHIFT;
101 res = mem_size < (avail_ram * nm_i->ram_thresh / 100);
6ce19aff
CY		 102 } else if (type == COMPRESS_PAGE) {
103#ifdef CONFIG_F2FS_FS_COMPRESSION
104 unsigned long free_ram = val.freeram;
105
106 /*
107 * free memory is lower than watermark or cached page count
108 * exceed threshold, deny caching compress page.
109 */
110 res = (free_ram > avail_ram * sbi->compress_watermark / 100) &&
111 (COMPRESS_MAPPING(sbi)->nrpages <
112 free_ram * sbi->compress_percent / 100);
113#else
114 res = false;
115#endif
1e84371f 116 } else {
1663cae4
JK		 117 if (!sbi->sb->s_bdi->wb.dirty_exceeded)
118 return true;
6fb03f3a
JK		 119 }
120 return res;
cdfc41c1
JK		 121}
122
f16ebe0d 123static void clear_node_folio_dirty(struct folio *folio)
e05df3b1 124{
f16ebe0d
MWO		 125 if (folio_test_dirty(folio)) {
126 f2fs_clear_page_cache_dirty_tag(folio);
127 folio_clear_dirty_for_io(folio);
128 dec_page_count(F2FS_F_SB(folio), F2FS_DIRTY_NODES);
e05df3b1 129 }
f16ebe0d 130 folio_clear_uptodate(folio);
e05df3b1
JK		 131}
132
350b8441 133static struct folio *get_current_nat_folio(struct f2fs_sb_info *sbi, nid_t nid)
e05df3b1 134{
350b8441 135 return f2fs_get_meta_folio_retry(sbi, current_nat_addr(sbi, nid));
e05df3b1
JK		 136}
137
3a19caf1 138static struct folio *get_next_nat_folio(struct f2fs_sb_info *sbi, nid_t nid)
e05df3b1 139{
350b8441 140 struct folio *src_folio;
5df510c8 141 struct folio *dst_folio;
e05df3b1
JK		 142 pgoff_t dst_off;
143 void *src_addr;
144 void *dst_addr;
145 struct f2fs_nm_info *nm_i = NM_I(sbi);
146
80551d17 147 dst_off = next_nat_addr(sbi, current_nat_addr(sbi, nid));
e05df3b1
JK		 148
149 /* get current nat block page with lock */
350b8441
MWO		 150 src_folio = get_current_nat_folio(sbi, nid);
151 if (IS_ERR(src_folio))
3a19caf1 152 return src_folio;
5df510c8 153 dst_folio = f2fs_grab_meta_folio(sbi, dst_off);
350b8441 154 f2fs_bug_on(sbi, folio_test_dirty(src_folio));
e05df3b1 155
350b8441 156 src_addr = folio_address(src_folio);
5df510c8 157 dst_addr = folio_address(dst_folio);
09cbfeaf 158 memcpy(dst_addr, src_addr, PAGE_SIZE);
5df510c8 159 folio_mark_dirty(dst_folio);
350b8441 160 f2fs_folio_put(src_folio, true);
e05df3b1
JK		 161
162 set_to_next_nat(nm_i, nid);
163
3a19caf1 164 return dst_folio;
e05df3b1
JK		 165}
166
32410577
CY		 167static struct nat_entry *__alloc_nat_entry(struct f2fs_sb_info *sbi,
168 nid_t nid, bool no_fail)
12f9ef37
YH		 169{
170 struct nat_entry *new;
171
32410577
CY		 172 new = f2fs_kmem_cache_alloc(nat_entry_slab,
173 GFP_F2FS_ZERO, no_fail, sbi);
12f9ef37
YH		 174 if (new) {
175 nat_set_nid(new, nid);
176 nat_reset_flag(new);
177 }
178 return new;
179}
180
181static void __free_nat_entry(struct nat_entry *e)
182{
183 kmem_cache_free(nat_entry_slab, e);
184}
185
186/* must be locked by nat_tree_lock */
187static struct nat_entry *__init_nat_entry(struct f2fs_nm_info *nm_i,
40aa9e12 188 struct nat_entry *ne, struct f2fs_nat_entry *raw_ne, bool no_fail, bool init_dirty)
12f9ef37
YH		 189{
190 if (no_fail)
191 f2fs_radix_tree_insert(&nm_i->nat_root, nat_get_nid(ne), ne);
192 else if (radix_tree_insert(&nm_i->nat_root, nat_get_nid(ne), ne))
193 return NULL;
194
195 if (raw_ne)
196 node_info_from_raw_nat(&ne->ni, raw_ne);
22969158 197
40aa9e12 198 if (init_dirty) {
199 INIT_LIST_HEAD(&ne->list);
200 nm_i->nat_cnt[TOTAL_NAT]++;
201 return ne;
202 }
203
22969158 204 spin_lock(&nm_i->nat_list_lock);
12f9ef37 205 list_add_tail(&ne->list, &nm_i->nat_entries);
22969158
CY		 206 spin_unlock(&nm_i->nat_list_lock);
207
a95ba66a
JK		 208 nm_i->nat_cnt[TOTAL_NAT]++;
209 nm_i->nat_cnt[RECLAIMABLE_NAT]++;
12f9ef37
YH		 210 return ne;
211}
212
0349b7f9 213static struct nat_entry *__lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t n, bool for_dirty)
e05df3b1 214{
22969158
CY		 215 struct nat_entry *ne;
216
217 ne = radix_tree_lookup(&nm_i->nat_root, n);
218
0349b7f9 219 /*
220 * for recent accessed nat entry which will not be dirtied soon
221 * later, move it to tail of lru list.
222 */
223 if (ne && !get_nat_flag(ne, IS_DIRTY) && !for_dirty) {
22969158
CY		 224 spin_lock(&nm_i->nat_list_lock);
225 if (!list_empty(&ne->list))
226 list_move_tail(&ne->list, &nm_i->nat_entries);
227 spin_unlock(&nm_i->nat_list_lock);
228 }
229
230 return ne;
e05df3b1
JK		 231}
232
233static unsigned int __gang_lookup_nat_cache(struct f2fs_nm_info *nm_i,
234 nid_t start, unsigned int nr, struct nat_entry **ep)
235{
236 return radix_tree_gang_lookup(&nm_i->nat_root, (void **)ep, start, nr);
237}
238
239static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e)
240{
e05df3b1 241 radix_tree_delete(&nm_i->nat_root, nat_get_nid(e));
a95ba66a
JK		 242 nm_i->nat_cnt[TOTAL_NAT]--;
243 nm_i->nat_cnt[RECLAIMABLE_NAT]--;
12f9ef37 244 __free_nat_entry(e);
e05df3b1
JK		 245}
246
780de47c
CY		 247static struct nat_entry_set *__grab_nat_entry_set(struct f2fs_nm_info *nm_i,
248 struct nat_entry *ne)
309cc2b6
JK		 249{
250 nid_t set = NAT_BLOCK_OFFSET(ne->ni.nid);
251 struct nat_entry_set *head;
252
309cc2b6
JK		 253 head = radix_tree_lookup(&nm_i->nat_set_root, set);
254 if (!head) {
32410577
CY		 255 head = f2fs_kmem_cache_alloc(nat_entry_set_slab,
256 GFP_NOFS, true, NULL);
309cc2b6
JK		 257
258 INIT_LIST_HEAD(&head->entry_list);
259 INIT_LIST_HEAD(&head->set_list);
260 head->set = set;
261 head->entry_cnt = 0;
9be32d72 262 f2fs_radix_tree_insert(&nm_i->nat_set_root, set, head);
309cc2b6 263 }
780de47c
CY		 264 return head;
265}
266
267static void __set_nat_cache_dirty(struct f2fs_nm_info *nm_i,
40aa9e12 268 struct nat_entry *ne, bool init_dirty)
780de47c
CY		 269{
270 struct nat_entry_set *head;
271 bool new_ne = nat_get_blkaddr(ne) == NEW_ADDR;
272
273 if (!new_ne)
274 head = __grab_nat_entry_set(nm_i, ne);
275
276 /*
277 * update entry_cnt in below condition:
278 * 1. update NEW_ADDR to valid block address;
279 * 2. update old block address to new one;
280 */
281 if (!new_ne && (get_nat_flag(ne, IS_PREALLOC) ||
282 !get_nat_flag(ne, IS_DIRTY)))
283 head->entry_cnt++;
284
285 set_nat_flag(ne, IS_PREALLOC, new_ne);
febeca6d
CY		 286
287 if (get_nat_flag(ne, IS_DIRTY))
288 goto refresh_list;
289
a95ba66a 290 nm_i->nat_cnt[DIRTY_NAT]++;
40aa9e12 291 if (!init_dirty)
292 nm_i->nat_cnt[RECLAIMABLE_NAT]--;
309cc2b6 293 set_nat_flag(ne, IS_DIRTY, true);
febeca6d 294refresh_list:
22969158 295 spin_lock(&nm_i->nat_list_lock);
780de47c 296 if (new_ne)
febeca6d
CY		 297 list_del_init(&ne->list);
298 else
299 list_move_tail(&ne->list, &head->entry_list);
22969158 300 spin_unlock(&nm_i->nat_list_lock);
309cc2b6
JK		 301}
302
303static void __clear_nat_cache_dirty(struct f2fs_nm_info *nm_i,
0b28b71e 304 struct nat_entry_set *set, struct nat_entry *ne)
309cc2b6 305{
22969158 306 spin_lock(&nm_i->nat_list_lock);
0b28b71e 307 list_move_tail(&ne->list, &nm_i->nat_entries);
22969158
CY		 308 spin_unlock(&nm_i->nat_list_lock);
309
0b28b71e
KM		 310 set_nat_flag(ne, IS_DIRTY, false);
311 set->entry_cnt--;
a95ba66a
JK		 312 nm_i->nat_cnt[DIRTY_NAT]--;
313 nm_i->nat_cnt[RECLAIMABLE_NAT]++;
309cc2b6
JK		 314}
315
316static unsigned int __gang_lookup_nat_set(struct f2fs_nm_info *nm_i,
317 nid_t start, unsigned int nr, struct nat_entry_set **ep)
318{
319 return radix_tree_gang_lookup(&nm_i->nat_set_root, (void **)ep,
320 start, nr);
321}
322
019a8912 323bool f2fs_in_warm_node_list(struct f2fs_sb_info *sbi, struct folio *folio)
50fa53ec 324{
fb92a5c9 325 return is_node_folio(folio) && IS_DNODE(folio) && is_cold_node(folio);
50fa53ec
CY		 326}
327
328void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi)
329{
330 spin_lock_init(&sbi->fsync_node_lock);
331 INIT_LIST_HEAD(&sbi->fsync_node_list);
332 sbi->fsync_seg_id = 0;
333 sbi->fsync_node_num = 0;
334}
335
336static unsigned int f2fs_add_fsync_node_entry(struct f2fs_sb_info *sbi,
963da02b 337 struct folio *folio)
50fa53ec
CY		 338{
339 struct fsync_node_entry *fn;
340 unsigned long flags;
341 unsigned int seq_id;
342
32410577
CY		 343 fn = f2fs_kmem_cache_alloc(fsync_node_entry_slab,
344 GFP_NOFS, true, NULL);
50fa53ec 345
963da02b
MWO		 346 folio_get(folio);
347 fn->folio = folio;
50fa53ec
CY		 348 INIT_LIST_HEAD(&fn->list);
349
350 spin_lock_irqsave(&sbi->fsync_node_lock, flags);
351 list_add_tail(&fn->list, &sbi->fsync_node_list);
352 fn->seq_id = sbi->fsync_seg_id++;
353 seq_id = fn->seq_id;
354 sbi->fsync_node_num++;
355 spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);
356
357 return seq_id;
358}
359
963da02b 360void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct folio *folio)
50fa53ec
CY		 361{
362 struct fsync_node_entry *fn;
363 unsigned long flags;
364
365 spin_lock_irqsave(&sbi->fsync_node_lock, flags);
366 list_for_each_entry(fn, &sbi->fsync_node_list, list) {
963da02b 367 if (fn->folio == folio) {
50fa53ec
CY		 368 list_del(&fn->list);
369 sbi->fsync_node_num--;
370 spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);
371 kmem_cache_free(fsync_node_entry_slab, fn);
963da02b 372 folio_put(folio);
50fa53ec
CY		 373 return;
374 }
375 }
376 spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);
377 f2fs_bug_on(sbi, 1);
378}
379
380void f2fs_reset_fsync_node_info(struct f2fs_sb_info *sbi)
381{
382 unsigned long flags;
383
384 spin_lock_irqsave(&sbi->fsync_node_lock, flags);
385 sbi->fsync_seg_id = 0;
386 spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);
387}
388
4d57b86d 389int f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid)
e05df3b1
JK		 390{
391 struct f2fs_nm_info *nm_i = NM_I(sbi);
392 struct nat_entry *e;
2dcf51ab 393 bool need = false;
e05df3b1 394
e4544b63 395 f2fs_down_read(&nm_i->nat_tree_lock);
0349b7f9 396 e = __lookup_nat_cache(nm_i, nid, false);
2dcf51ab
JK		 397 if (e) {
398 if (!get_nat_flag(e, IS_CHECKPOINTED) &&
399 !get_nat_flag(e, HAS_FSYNCED_INODE))
400 need = true;
401 }
e4544b63 402 f2fs_up_read(&nm_i->nat_tree_lock);
2dcf51ab 403 return need;
e05df3b1
JK		 404}
405
4d57b86d 406bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid)
479f40c4
JK		 407{
408 struct f2fs_nm_info *nm_i = NM_I(sbi);
409 struct nat_entry *e;
2dcf51ab 410 bool is_cp = true;
479f40c4 411
e4544b63 412 f2fs_down_read(&nm_i->nat_tree_lock);
0349b7f9 413 e = __lookup_nat_cache(nm_i, nid, false);
2dcf51ab
JK		 414 if (e && !get_nat_flag(e, IS_CHECKPOINTED))
415 is_cp = false;
e4544b63 416 f2fs_up_read(&nm_i->nat_tree_lock);
2dcf51ab 417 return is_cp;
479f40c4
JK		 418}
419
4d57b86d 420bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino)
b6fe5873
JK		 421{
422 struct f2fs_nm_info *nm_i = NM_I(sbi);
423 struct nat_entry *e;
88bd02c9 424 bool need_update = true;
b6fe5873 425
e4544b63 426 f2fs_down_read(&nm_i->nat_tree_lock);
0349b7f9 427 e = __lookup_nat_cache(nm_i, ino, false);
88bd02c9
JK		 428 if (e && get_nat_flag(e, HAS_LAST_FSYNC) &&
429 (get_nat_flag(e, IS_CHECKPOINTED) ||
430 get_nat_flag(e, HAS_FSYNCED_INODE)))
431 need_update = false;
e4544b63 432 f2fs_up_read(&nm_i->nat_tree_lock);
88bd02c9 433 return need_update;
b6fe5873
JK		 434}
435
12f9ef37 436/* must be locked by nat_tree_lock */
1515aef0 437static void cache_nat_entry(struct f2fs_sb_info *sbi, nid_t nid,
e05df3b1
JK		 438 struct f2fs_nat_entry *ne)
439{
1515aef0 440 struct f2fs_nm_info *nm_i = NM_I(sbi);
12f9ef37 441 struct nat_entry *new, *e;
9be32d72 442
0df035c7 443 /* Let's mitigate lock contention of nat_tree_lock during checkpoint */
e4544b63 444 if (f2fs_rwsem_is_locked(&sbi->cp_global_sem))
0df035c7
JK		 445 return;
446
32410577 447 new = __alloc_nat_entry(sbi, nid, false);
12f9ef37
YH		 448 if (!new)
449 return;
450
e4544b63 451 f2fs_down_write(&nm_i->nat_tree_lock);
0349b7f9 452 e = __lookup_nat_cache(nm_i, nid, false);
12f9ef37 453 if (!e)
40aa9e12 454 e = __init_nat_entry(nm_i, new, ne, false, false);
12f9ef37 455 else
0c0b471e
EB		 456 f2fs_bug_on(sbi, nat_get_ino(e) != le32_to_cpu(ne->ino) ||
457 nat_get_blkaddr(e) !=
458 le32_to_cpu(ne->block_addr) ||
1515aef0 459 nat_get_version(e) != ne->version);
e4544b63 460 f2fs_up_write(&nm_i->nat_tree_lock);
12f9ef37
YH		 461 if (e != new)
462 __free_nat_entry(new);
e05df3b1
JK		 463}
464
465static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni,
479f40c4 466 block_t new_blkaddr, bool fsync_done)
e05df3b1
JK		 467{
468 struct f2fs_nm_info *nm_i = NM_I(sbi);
469 struct nat_entry *e;
32410577 470 struct nat_entry *new = __alloc_nat_entry(sbi, ni->nid, true);
40aa9e12 471 bool init_dirty = false;
9be32d72 472
e4544b63 473 f2fs_down_write(&nm_i->nat_tree_lock);
0349b7f9 474 e = __lookup_nat_cache(nm_i, ni->nid, true);
e05df3b1 475 if (!e) {
40aa9e12 476 init_dirty = true;
477 e = __init_nat_entry(nm_i, new, NULL, true, true);
5c27f4ee 478 copy_node_info(&e->ni, ni);
9850cf4a 479 f2fs_bug_on(sbi, ni->blk_addr == NEW_ADDR);
e05df3b1
JK		 480 } else if (new_blkaddr == NEW_ADDR) {
481 /*
482 * when nid is reallocated,
483 * previous nat entry can be remained in nat cache.
484 * So, reinitialize it with new information.
485 */
5c27f4ee 486 copy_node_info(&e->ni, ni);
9850cf4a 487 f2fs_bug_on(sbi, ni->blk_addr != NULL_ADDR);
e05df3b1 488 }
12f9ef37
YH		 489 /* let's free early to reduce memory consumption */
490 if (e != new)
491 __free_nat_entry(new);
e05df3b1 492
e05df3b1 493 /* sanity check */
9850cf4a
JK		 494 f2fs_bug_on(sbi, nat_get_blkaddr(e) != ni->blk_addr);
495 f2fs_bug_on(sbi, nat_get_blkaddr(e) == NULL_ADDR &&
e05df3b1 496 new_blkaddr == NULL_ADDR);
9850cf4a 497 f2fs_bug_on(sbi, nat_get_blkaddr(e) == NEW_ADDR &&
e05df3b1 498 new_blkaddr == NEW_ADDR);
93770ab7 499 f2fs_bug_on(sbi, __is_valid_data_blkaddr(nat_get_blkaddr(e)) &&
e05df3b1
JK		 500 new_blkaddr == NEW_ADDR);
501
e1c42045 502 /* increment version no as node is removed */
e05df3b1
JK		 503 if (nat_get_blkaddr(e) != NEW_ADDR && new_blkaddr == NULL_ADDR) {
504 unsigned char version = nat_get_version(e);
5f029c04 505
e05df3b1
JK		 506 nat_set_version(e, inc_node_version(version));
507 }
508
509 /* change address */
510 nat_set_blkaddr(e, new_blkaddr);
93770ab7 511 if (!__is_valid_data_blkaddr(new_blkaddr))
88bd02c9 512 set_nat_flag(e, IS_CHECKPOINTED, false);
40aa9e12 513 __set_nat_cache_dirty(nm_i, e, init_dirty);
479f40c4
JK		 514
515 /* update fsync_mark if its inode nat entry is still alive */
d5b692b7 516 if (ni->nid != ni->ino)
0349b7f9 517 e = __lookup_nat_cache(nm_i, ni->ino, false);
88bd02c9
JK		 518 if (e) {
519 if (fsync_done && ni->nid == ni->ino)
520 set_nat_flag(e, HAS_FSYNCED_INODE, true);
521 set_nat_flag(e, HAS_LAST_FSYNC, fsync_done);
522 }
e4544b63 523 f2fs_up_write(&nm_i->nat_tree_lock);
e05df3b1
JK		 524}
525
4d57b86d 526int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink)
e05df3b1
JK		 527{
528 struct f2fs_nm_info *nm_i = NM_I(sbi);
1b38dc8e 529 int nr = nr_shrink;
e05df3b1 530
e4544b63 531 if (!f2fs_down_write_trylock(&nm_i->nat_tree_lock))
b873b798 532 return 0;
e05df3b1 533
22969158
CY		 534 spin_lock(&nm_i->nat_list_lock);
535 while (nr_shrink) {
e05df3b1 536 struct nat_entry *ne;
22969158
CY		 537
538 if (list_empty(&nm_i->nat_entries))
539 break;
540
e05df3b1
JK		 541 ne = list_first_entry(&nm_i->nat_entries,
542 struct nat_entry, list);
22969158
CY		 543 list_del(&ne->list);
544 spin_unlock(&nm_i->nat_list_lock);
545
e05df3b1
JK		 546 __del_from_nat_cache(nm_i, ne);
547 nr_shrink--;
22969158
CY		 548
549 spin_lock(&nm_i->nat_list_lock);
e05df3b1 550 }
22969158
CY		 551 spin_unlock(&nm_i->nat_list_lock);
552
e4544b63 553 f2fs_up_write(&nm_i->nat_tree_lock);
1b38dc8e 554 return nr - nr_shrink;
e05df3b1
JK		 555}
556
7735730d 557int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid,
a9419b63 558 struct node_info *ni, bool checkpoint_context)
e05df3b1
JK		 559{
560 struct f2fs_nm_info *nm_i = NM_I(sbi);
561 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
b7ad7512 562 struct f2fs_journal *journal = curseg->journal;
e05df3b1
JK		 563 nid_t start_nid = START_NID(nid);
564 struct f2fs_nat_block *nat_blk;
aa0c14ef 565 struct folio *folio = NULL;
e05df3b1
JK		 566 struct f2fs_nat_entry ne;
567 struct nat_entry *e;
66a82d1f 568 pgoff_t index;
e05df3b1 569 int i;
1773f63d 570 bool need_cache = true;
e05df3b1 571
76f01376 572 ni->flag = 0;
e05df3b1 573 ni->nid = nid;
2eeb0dce 574retry:
e05df3b1 575 /* Check nat cache */
e4544b63 576 f2fs_down_read(&nm_i->nat_tree_lock);
0349b7f9 577 e = __lookup_nat_cache(nm_i, nid, false);
e05df3b1
JK		 578 if (e) {
579 ni->ino = nat_get_ino(e);
580 ni->blk_addr = nat_get_blkaddr(e);
581 ni->version = nat_get_version(e);
e4544b63 582 f2fs_up_read(&nm_i->nat_tree_lock);
1773f63d
CY		 583 if (IS_ENABLED(CONFIG_F2FS_CHECK_FS)) {
584 need_cache = false;
585 goto sanity_check;
586 }
7735730d 587 return 0;
1515aef0 588 }
e05df3b1 589
2eeb0dce
JK		 590 /*
591 * Check current segment summary by trying to grab journal_rwsem first.
592 * This sem is on the critical path on the checkpoint requiring the above
593 * nat_tree_lock. Therefore, we should retry, if we failed to grab here
594 * while not bothering checkpoint.
595 */
e4544b63 596 if (!f2fs_rwsem_is_locked(&sbi->cp_global_sem) || checkpoint_context) {
2eeb0dce 597 down_read(&curseg->journal_rwsem);
e4544b63 598 } else if (f2fs_rwsem_is_contended(&nm_i->nat_tree_lock) ||
a9419b63 599 !down_read_trylock(&curseg->journal_rwsem)) {
e4544b63 600 f2fs_up_read(&nm_i->nat_tree_lock);
2eeb0dce
JK		 601 goto retry;
602 }
3547ea96 603
4d57b86d 604 i = f2fs_lookup_journal_in_cursum(journal, NAT_JOURNAL, nid, 0);
e05df3b1 605 if (i >= 0) {
dfc08a12 606 ne = nat_in_journal(journal, i);
e05df3b1
JK		 607 node_info_from_raw_nat(ni, &ne);
608 }
544b53da 609 up_read(&curseg->journal_rwsem);
66a82d1f 610 if (i >= 0) {
e4544b63 611 f2fs_up_read(&nm_i->nat_tree_lock);
1773f63d 612 goto sanity_check;
66a82d1f 613 }
e05df3b1
JK		 614
615 /* Fill node_info from nat page */
66a82d1f 616 index = current_nat_addr(sbi, nid);
e4544b63 617 f2fs_up_read(&nm_i->nat_tree_lock);
66a82d1f 618
aa0c14ef
MWO		 619 folio = f2fs_get_meta_folio(sbi, index);
620 if (IS_ERR(folio))
621 return PTR_ERR(folio);
7735730d 622
aa0c14ef 623 nat_blk = folio_address(folio);
e05df3b1
JK		 624 ne = nat_blk->entries[nid - start_nid];
625 node_info_from_raw_nat(ni, &ne);
aa0c14ef 626 f2fs_folio_put(folio, true);
1773f63d
CY		 627sanity_check:
628 if (__is_valid_data_blkaddr(ni->blk_addr) &&
629 !f2fs_is_valid_blkaddr(sbi, ni->blk_addr,
630 DATA_GENERIC_ENHANCE)) {
631 set_sbi_flag(sbi, SBI_NEED_FSCK);
632 f2fs_err_ratelimited(sbi,
633 "f2fs_get_node_info of %pS: inconsistent nat entry, "
634 "ino:%u, nid:%u, blkaddr:%u, ver:%u, flag:%u",
635 __builtin_return_address(0),
636 ni->ino, ni->nid, ni->blk_addr, ni->version, ni->flag);
637 f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT);
638 return -EFSCORRUPTED;
639 }
93770ab7 640
e05df3b1 641 /* cache nat entry */
1773f63d
CY		 642 if (need_cache)
643 cache_nat_entry(sbi, nid, &ne);
7735730d 644 return 0;
e05df3b1
JK		 645}
646
79344efb
JK		 647/*
648 * readahead MAX_RA_NODE number of node pages.
649 */
4e9e8f81 650static void f2fs_ra_node_pages(struct folio *parent, int start, int n)
79344efb 651{
4e9e8f81 652 struct f2fs_sb_info *sbi = F2FS_F_SB(parent);
79344efb
JK		 653 struct blk_plug plug;
654 int i, end;
655 nid_t nid;
656
657 blk_start_plug(&plug);
658
659 /* Then, try readahead for siblings of the desired node */
660 end = start + n;
d7e9a903 661 end = min(end, (int)NIDS_PER_BLOCK);
79344efb 662 for (i = start; i < end; i++) {
fddd722e 663 nid = get_nid(parent, i, false);
4d57b86d 664 f2fs_ra_node_page(sbi, nid);
79344efb
JK		 665 }
666
667 blk_finish_plug(&plug);
668}
669
4d57b86d 670pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs)
3cf45747
CY		 671{
672 const long direct_index = ADDRS_PER_INODE(dn->inode);
d02a6e61
CY		 673 const long direct_blks = ADDRS_PER_BLOCK(dn->inode);
674 const long indirect_blks = ADDRS_PER_BLOCK(dn->inode) * NIDS_PER_BLOCK;
675 unsigned int skipped_unit = ADDRS_PER_BLOCK(dn->inode);
3cf45747
CY		 676 int cur_level = dn->cur_level;
677 int max_level = dn->max_level;
678 pgoff_t base = 0;
679
680 if (!dn->max_level)
681 return pgofs + 1;
682
683 while (max_level-- > cur_level)
684 skipped_unit *= NIDS_PER_BLOCK;
685
686 switch (dn->max_level) {
687 case 3:
688 base += 2 * indirect_blks;
df561f66 689 fallthrough;
3cf45747
CY		 690 case 2:
691 base += 2 * direct_blks;
df561f66 692 fallthrough;
3cf45747
CY		 693 case 1:
694 base += direct_index;
695 break;
696 default:
697 f2fs_bug_on(F2FS_I_SB(dn->inode), 1);
698 }
699
700 return ((pgofs - base) / skipped_unit + 1) * skipped_unit + base;
701}
702
0a8165d7 703/*
e05df3b1
JK		 704 * The maximum depth is four.
705 * Offset[0] will have raw inode offset.
706 */
81ca7350 707static int get_node_path(struct inode *inode, long block,
de93653f 708 int offset[4], unsigned int noffset[4])
e05df3b1 709{
81ca7350 710 const long direct_index = ADDRS_PER_INODE(inode);
d02a6e61 711 const long direct_blks = ADDRS_PER_BLOCK(inode);
e05df3b1 712 const long dptrs_per_blk = NIDS_PER_BLOCK;
d02a6e61 713 const long indirect_blks = ADDRS_PER_BLOCK(inode) * NIDS_PER_BLOCK;
e05df3b1
JK		 714 const long dindirect_blks = indirect_blks * NIDS_PER_BLOCK;
715 int n = 0;
716 int level = 0;
717
718 noffset[0] = 0;
719
720 if (block < direct_index) {
25c0a6e5 721 offset[n] = block;
e05df3b1
JK		 722 goto got;
723 }
724 block -= direct_index;
725 if (block < direct_blks) {
726 offset[n++] = NODE_DIR1_BLOCK;
727 noffset[n] = 1;
25c0a6e5 728 offset[n] = block;
e05df3b1
JK		 729 level = 1;
730 goto got;
731 }
732 block -= direct_blks;
733 if (block < direct_blks) {
734 offset[n++] = NODE_DIR2_BLOCK;
735 noffset[n] = 2;
25c0a6e5 736 offset[n] = block;
e05df3b1
JK		 737 level = 1;
738 goto got;
739 }
740 block -= direct_blks;
741 if (block < indirect_blks) {
742 offset[n++] = NODE_IND1_BLOCK;
743 noffset[n] = 3;
744 offset[n++] = block / direct_blks;
745 noffset[n] = 4 + offset[n - 1];
25c0a6e5 746 offset[n] = block % direct_blks;
e05df3b1
JK		 747 level = 2;
748 goto got;
749 }
750 block -= indirect_blks;
751 if (block < indirect_blks) {
752 offset[n++] = NODE_IND2_BLOCK;
753 noffset[n] = 4 + dptrs_per_blk;
754 offset[n++] = block / direct_blks;
755 noffset[n] = 5 + dptrs_per_blk + offset[n - 1];
25c0a6e5 756 offset[n] = block % direct_blks;
e05df3b1
JK		 757 level = 2;
758 goto got;
759 }
760 block -= indirect_blks;
761 if (block < dindirect_blks) {
762 offset[n++] = NODE_DIND_BLOCK;
763 noffset[n] = 5 + (dptrs_per_blk * 2);
764 offset[n++] = block / indirect_blks;
765 noffset[n] = 6 + (dptrs_per_blk * 2) +
766 offset[n - 1] * (dptrs_per_blk + 1);
767 offset[n++] = (block / direct_blks) % dptrs_per_blk;
768 noffset[n] = 7 + (dptrs_per_blk * 2) +
769 offset[n - 2] * (dptrs_per_blk + 1) +
770 offset[n - 1];
25c0a6e5 771 offset[n] = block % direct_blks;
e05df3b1
JK		 772 level = 3;
773 goto got;
774 } else {
adb6dc19 775 return -E2BIG;
e05df3b1
JK		 776 }
777got:
778 return level;
779}
780
fb733f98
MWO		 781static struct folio *f2fs_get_node_folio_ra(struct folio *parent, int start);
782
e05df3b1
JK		 783/*
784 * Caller should call f2fs_put_dnode(dn).
4f4124d0 785 * Also, it should grab and release a rwsem by calling f2fs_lock_op() and
7a88ddb5 786 * f2fs_unlock_op() only if mode is set with ALLOC_NODE.
e05df3b1 787 */
4d57b86d 788int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)
e05df3b1 789{
4081363f 790 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
fb733f98
MWO		 791 struct folio *nfolio[4];
792 struct folio *parent = NULL;
e05df3b1
JK		 793 int offset[4];
794 unsigned int noffset[4];
795 nid_t nids[4];
3cf45747 796 int level, i = 0;
e05df3b1
JK		 797 int err = 0;
798
81ca7350 799 level = get_node_path(dn->inode, index, offset, noffset);
adb6dc19
JK		 800 if (level < 0)
801 return level;
e05df3b1
JK		 802
803 nids[0] = dn->inode->i_ino;
e05df3b1 804
f1d54e07 805 if (!dn->inode_folio) {
fb733f98
MWO		 806 nfolio[0] = f2fs_get_inode_folio(sbi, nids[0]);
807 if (IS_ERR(nfolio[0]))
808 return PTR_ERR(nfolio[0]);
809 } else {
f1d54e07 810 nfolio[0] = dn->inode_folio;
1646cfac 811 }
f1a3b98e
JK		 812
813 /* if inline_data is set, should not report any block indices */
814 if (f2fs_has_inline_data(dn->inode) && index) {
76629165 815 err = -ENOENT;
fb733f98 816 f2fs_folio_put(nfolio[0], true);
f1a3b98e
JK		 817 goto release_out;
818 }
819
fb733f98 820 parent = nfolio[0];
52c2db3f 821 if (level != 0)
fddd722e 822 nids[1] = get_nid(parent, offset[0], true);
f1d54e07
MWO		 823 dn->inode_folio = nfolio[0];
824 dn->inode_folio_locked = true;
e05df3b1
JK		 825
826 /* get indirect or direct nodes */
827 for (i = 1; i <= level; i++) {
828 bool done = false;
829
77de19b6
CY		 830 if (nids[i] && nids[i] == dn->inode->i_ino) {
831 err = -EFSCORRUPTED;
832 f2fs_err_ratelimited(sbi,
833 "inode mapping table is corrupted, run fsck to fix it, "
834 "ino:%lu, nid:%u, level:%d, offset:%d",
835 dn->inode->i_ino, nids[i], level, offset[level]);
836 set_sbi_flag(sbi, SBI_NEED_FSCK);
837 goto release_pages;
838 }
839
266e97a8 840 if (!nids[i] && mode == ALLOC_NODE) {
e05df3b1 841 /* alloc new node */
4d57b86d 842 if (!f2fs_alloc_nid(sbi, &(nids[i]))) {
e05df3b1
JK		 843 err = -ENOSPC;
844 goto release_pages;
845 }
846
847 dn->nid = nids[i];
fb733f98
MWO		 848 nfolio[i] = f2fs_new_node_folio(dn, noffset[i]);
849 if (IS_ERR(nfolio[i])) {
4d57b86d 850 f2fs_alloc_nid_failed(sbi, nids[i]);
fb733f98 851 err = PTR_ERR(nfolio[i]);
e05df3b1
JK		 852 goto release_pages;
853 }
854
66bca01b 855 set_nid(parent, offset[i - 1], nids[i], i == 1);
4d57b86d 856 f2fs_alloc_nid_done(sbi, nids[i]);
e05df3b1 857 done = true;
266e97a8 858 } else if (mode == LOOKUP_NODE_RA && i == level && level > 1) {
fb733f98
MWO		 859 nfolio[i] = f2fs_get_node_folio_ra(parent, offset[i - 1]);
860 if (IS_ERR(nfolio[i])) {
861 err = PTR_ERR(nfolio[i]);
e05df3b1
JK		 862 goto release_pages;
863 }
864 done = true;
865 }
866 if (i == 1) {
f1d54e07 867 dn->inode_folio_locked = false;
fb733f98 868 folio_unlock(parent);
e05df3b1 869 } else {
fb733f98 870 f2fs_folio_put(parent, true);
e05df3b1
JK		 871 }
872
873 if (!done) {
fb733f98
MWO		 874 nfolio[i] = f2fs_get_node_folio(sbi, nids[i]);
875 if (IS_ERR(nfolio[i])) {
876 err = PTR_ERR(nfolio[i]);
877 f2fs_folio_put(nfolio[0], false);
e05df3b1
JK		 878 goto release_out;
879 }
880 }
881 if (i < level) {
fb733f98 882 parent = nfolio[i];
fddd722e 883 nids[i + 1] = get_nid(parent, offset[i], false);
e05df3b1
JK		 884 }
885 }
886 dn->nid = nids[level];
887 dn->ofs_in_node = offset[level];
6f7ec661 888 dn->node_folio = nfolio[level];
a2ced1ce 889 dn->data_blkaddr = f2fs_data_blkaddr(dn);
94afd6d6
CY		 890
891 if (is_inode_flag_set(dn->inode, FI_COMPRESSED_FILE) &&
892 f2fs_sb_has_readonly(sbi)) {
4b99ecd3
CY		 893 unsigned int cluster_size = F2FS_I(dn->inode)->i_cluster_size;
894 unsigned int ofs_in_node = dn->ofs_in_node;
895 pgoff_t fofs = index;
896 unsigned int c_len;
94afd6d6
CY		 897 block_t blkaddr;
898
4b99ecd3
CY		 899 /* should align fofs and ofs_in_node to cluster_size */
900 if (fofs % cluster_size) {
901 fofs = round_down(fofs, cluster_size);
902 ofs_in_node = round_down(ofs_in_node, cluster_size);
903 }
904
905 c_len = f2fs_cluster_blocks_are_contiguous(dn, ofs_in_node);
94afd6d6
CY		 906 if (!c_len)
907 goto out;
908
6f7ec661 909 blkaddr = data_blkaddr(dn->inode, dn->node_folio, ofs_in_node);
94afd6d6 910 if (blkaddr == COMPRESS_ADDR)
6f7ec661 911 blkaddr = data_blkaddr(dn->inode, dn->node_folio,
4b99ecd3 912 ofs_in_node + 1);
94afd6d6 913
e7547dac 914 f2fs_update_read_extent_tree_range_compressed(dn->inode,
4b99ecd3 915 fofs, blkaddr, cluster_size, c_len);
94afd6d6
CY		 916 }
917out:
e05df3b1
JK		 918 return 0;
919
920release_pages:
fb733f98 921 f2fs_folio_put(parent, true);
e05df3b1 922 if (i > 1)
fb733f98 923 f2fs_folio_put(nfolio[0], false);
e05df3b1 924release_out:
f1d54e07 925 dn->inode_folio = NULL;
6f7ec661 926 dn->node_folio = NULL;
3cf45747
CY		 927 if (err == -ENOENT) {
928 dn->cur_level = i;
929 dn->max_level = level;
0a2aa8fb 930 dn->ofs_in_node = offset[level];
3cf45747 931 }
e05df3b1
JK		 932 return err;
933}
934
7735730d 935static int truncate_node(struct dnode_of_data *dn)
e05df3b1 936{
4081363f 937 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
e05df3b1 938 struct node_info ni;
7735730d 939 int err;
0ea295dd 940 pgoff_t index;
e05df3b1 941
a9419b63 942 err = f2fs_get_node_info(sbi, dn->nid, &ni, false);
7735730d
CY		 943 if (err)
944 return err;
e05df3b1 945
6babe00c
CY		 946 if (ni.blk_addr != NEW_ADDR &&
947 !f2fs_is_valid_blkaddr(sbi, ni.blk_addr, DATA_GENERIC_ENHANCE)) {
948 f2fs_err_ratelimited(sbi,
949 "nat entry is corrupted, run fsck to fix it, ino:%u, "
950 "nid:%u, blkaddr:%u", ni.ino, ni.nid, ni.blk_addr);
951 set_sbi_flag(sbi, SBI_NEED_FSCK);
952 f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT);
953 return -EFSCORRUPTED;
954 }
955
e05df3b1 956 /* Deallocate node address */
e53c568f 957 f2fs_invalidate_blocks(sbi, ni.blk_addr, 1);
000519f2 958 dec_valid_node_count(sbi, dn->inode, dn->nid == dn->inode->i_ino);
479f40c4 959 set_node_addr(sbi, &ni, NULL_ADDR, false);
e05df3b1
JK		 960
961 if (dn->nid == dn->inode->i_ino) {
4d57b86d 962 f2fs_remove_orphan_inode(sbi, dn->nid);
e05df3b1 963 dec_valid_inode_count(sbi);
0f18b462 964 f2fs_inode_synced(dn->inode);
e05df3b1 965 }
000519f2 966
f16ebe0d 967 clear_node_folio_dirty(dn->node_folio);
caf0047e 968 set_sbi_flag(sbi, SBI_IS_DIRTY);
e05df3b1 969
6f7ec661
MWO		 970 index = dn->node_folio->index;
971 f2fs_folio_put(dn->node_folio, true);
bf39c00a
JK		 972
973 invalidate_mapping_pages(NODE_MAPPING(sbi),
0ea295dd 974 index, index);
bf39c00a 975
6f7ec661 976 dn->node_folio = NULL;
51dd6249 977 trace_f2fs_truncate_node(dn->inode, dn->nid, ni.blk_addr);
7735730d
CY		 978
979 return 0;
e05df3b1
JK		 980}
981
982static int truncate_dnode(struct dnode_of_data *dn)
983{
a6ec8378 984 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
df410140 985 struct folio *folio;
7735730d 986 int err;
e05df3b1
JK		 987
988 if (dn->nid == 0)
989 return 1;
990
991 /* get direct node */
df410140
MWO		 992 folio = f2fs_get_node_folio(sbi, dn->nid);
993 if (PTR_ERR(folio) == -ENOENT)
e05df3b1 994 return 1;
df410140
MWO		 995 else if (IS_ERR(folio))
996 return PTR_ERR(folio);
e05df3b1 997
a5f3be6e 998 if (IS_INODE(folio) || ino_of_node(folio) != dn->inode->i_ino) {
a6ec8378 999 f2fs_err(sbi, "incorrect node reference, ino: %lu, nid: %u, ino_of_node: %u",
28fde0d7 1000 dn->inode->i_ino, dn->nid, ino_of_node(folio));
a6ec8378
CY		 1001 set_sbi_flag(sbi, SBI_NEED_FSCK);
1002 f2fs_handle_error(sbi, ERROR_INVALID_NODE_REFERENCE);
df410140 1003 f2fs_folio_put(folio, true);
a6ec8378
CY		 1004 return -EFSCORRUPTED;
1005 }
1006
e05df3b1 1007 /* Make dnode_of_data for parameter */
6f7ec661 1008 dn->node_folio = folio;
e05df3b1 1009 dn->ofs_in_node = 0;
a6ec8378 1010 f2fs_truncate_data_blocks_range(dn, ADDRS_PER_BLOCK(dn->inode));
7735730d 1011 err = truncate_node(dn);
0135c482 1012 if (err) {
df410140 1013 f2fs_folio_put(folio, true);
7735730d 1014 return err;
0135c482 1015 }
7735730d 1016
e05df3b1
JK		 1017 return 1;
1018}
1019
1020static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs,
1021 int ofs, int depth)
1022{
e05df3b1 1023 struct dnode_of_data rdn = *dn;
878a05b0 1024 struct folio *folio;
e05df3b1
JK		 1025 struct f2fs_node *rn;
1026 nid_t child_nid;
1027 unsigned int child_nofs;
1028 int freed = 0;
1029 int i, ret;
1030
1031 if (dn->nid == 0)
1032 return NIDS_PER_BLOCK + 1;
1033
51dd6249
NJ		 1034 trace_f2fs_truncate_nodes_enter(dn->inode, dn->nid, dn->data_blkaddr);
1035
878a05b0
MWO		 1036 folio = f2fs_get_node_folio(F2FS_I_SB(dn->inode), dn->nid);
1037 if (IS_ERR(folio)) {
1038 trace_f2fs_truncate_nodes_exit(dn->inode, PTR_ERR(folio));
1039 return PTR_ERR(folio);
51dd6249 1040 }
e05df3b1 1041
4e9e8f81 1042 f2fs_ra_node_pages(folio, ofs, NIDS_PER_BLOCK);
79344efb 1043
8591db2a 1044 rn = F2FS_NODE(folio);
e05df3b1
JK		 1045 if (depth < 3) {
1046 for (i = ofs; i < NIDS_PER_BLOCK; i++, freed++) {
1047 child_nid = le32_to_cpu(rn->in.nid[i]);
1048 if (child_nid == 0)
1049 continue;
1050 rdn.nid = child_nid;
1051 ret = truncate_dnode(&rdn);
1052 if (ret < 0)
1053 goto out_err;
66bca01b 1054 if (set_nid(folio, i, 0, false))
12719ae1 1055 dn->node_changed = true;
e05df3b1
JK		 1056 }
1057 } else {
1058 child_nofs = nofs + ofs * (NIDS_PER_BLOCK + 1) + 1;
1059 for (i = ofs; i < NIDS_PER_BLOCK; i++) {
1060 child_nid = le32_to_cpu(rn->in.nid[i]);
1061 if (child_nid == 0) {
1062 child_nofs += NIDS_PER_BLOCK + 1;
1063 continue;
1064 }
1065 rdn.nid = child_nid;
1066 ret = truncate_nodes(&rdn, child_nofs, 0, depth - 1);
1067 if (ret == (NIDS_PER_BLOCK + 1)) {
66bca01b 1068 if (set_nid(folio, i, 0, false))
12719ae1 1069 dn->node_changed = true;
e05df3b1
JK		 1070 child_nofs += ret;
1071 } else if (ret < 0 && ret != -ENOENT) {
1072 goto out_err;
1073 }
1074 }
1075 freed = child_nofs;
1076 }
1077
1078 if (!ofs) {
1079 /* remove current indirect node */
6f7ec661 1080 dn->node_folio = folio;
7735730d
CY		 1081 ret = truncate_node(dn);
1082 if (ret)
1083 goto out_err;
e05df3b1
JK		 1084 freed++;
1085 } else {
878a05b0 1086 f2fs_folio_put(folio, true);
e05df3b1 1087 }
51dd6249 1088 trace_f2fs_truncate_nodes_exit(dn->inode, freed);
e05df3b1
JK		 1089 return freed;
1090
1091out_err:
878a05b0 1092 f2fs_folio_put(folio, true);
51dd6249 1093 trace_f2fs_truncate_nodes_exit(dn->inode, ret);
e05df3b1
JK		 1094 return ret;
1095}
1096
1097static int truncate_partial_nodes(struct dnode_of_data *dn,
1098 struct f2fs_inode *ri, int *offset, int depth)
1099{
d68833a0 1100 struct folio *folios[2];
e05df3b1
JK		 1101 nid_t nid[3];
1102 nid_t child_nid;
1103 int err = 0;
1104 int i;
1105 int idx = depth - 2;
1106
fddd722e 1107 nid[0] = get_nid(dn->inode_folio, offset[0], true);
e05df3b1
JK		 1108 if (!nid[0])
1109 return 0;
1110
1111 /* get indirect nodes in the path */
a225dca3 1112 for (i = 0; i < idx + 1; i++) {
e1c42045 1113 /* reference count'll be increased */
d68833a0
MWO		 1114 folios[i] = f2fs_get_node_folio(F2FS_I_SB(dn->inode), nid[i]);
1115 if (IS_ERR(folios[i])) {
1116 err = PTR_ERR(folios[i]);
a225dca3 1117 idx = i - 1;
e05df3b1
JK		 1118 goto fail;
1119 }
fddd722e 1120 nid[i + 1] = get_nid(folios[i], offset[i + 1], false);
e05df3b1
JK		 1121 }
1122
4e9e8f81 1123 f2fs_ra_node_pages(folios[idx], offset[idx + 1], NIDS_PER_BLOCK);
79344efb 1124
e05df3b1 1125 /* free direct nodes linked to a partial indirect node */
a225dca3 1126 for (i = offset[idx + 1]; i < NIDS_PER_BLOCK; i++) {
fddd722e 1127 child_nid = get_nid(folios[idx], i, false);
e05df3b1
JK		 1128 if (!child_nid)
1129 continue;
1130 dn->nid = child_nid;
1131 err = truncate_dnode(dn);
1132 if (err < 0)
1133 goto fail;
66bca01b 1134 if (set_nid(folios[idx], i, 0, false))
12719ae1 1135 dn->node_changed = true;
e05df3b1
JK		 1136 }
1137
a225dca3 1138 if (offset[idx + 1] == 0) {
6f7ec661 1139 dn->node_folio = folios[idx];
e05df3b1 1140 dn->nid = nid[idx];
7735730d
CY		 1141 err = truncate_node(dn);
1142 if (err)
1143 goto fail;
e05df3b1 1144 } else {
d68833a0 1145 f2fs_folio_put(folios[idx], true);
e05df3b1
JK		 1146 }
1147 offset[idx]++;
a225dca3 1148 offset[idx + 1] = 0;
1149 idx--;
e05df3b1 1150fail:
a225dca3 1151 for (i = idx; i >= 0; i--)
d68833a0 1152 f2fs_folio_put(folios[i], true);
51dd6249
NJ		 1153
1154 trace_f2fs_truncate_partial_nodes(dn->inode, nid, depth, err);
1155
e05df3b1
JK		 1156 return err;
1157}
1158
0a8165d7 1159/*
e05df3b1
JK		 1160 * All the block addresses of data and nodes should be nullified.
1161 */
4d57b86d 1162int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from)
e05df3b1 1163{
4081363f 1164 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
e05df3b1
JK		 1165 int err = 0, cont = 1;
1166 int level, offset[4], noffset[4];
7dd690c8 1167 unsigned int nofs = 0;
58bfaf44 1168 struct f2fs_inode *ri;
e05df3b1 1169 struct dnode_of_data dn;
520b17e0 1170 struct folio *folio;
e05df3b1 1171
51dd6249
NJ		 1172 trace_f2fs_truncate_inode_blocks_enter(inode, from);
1173
81ca7350 1174 level = get_node_path(inode, from, offset, noffset);
e6494977
CY		 1175 if (level <= 0) {
1176 if (!level) {
1177 level = -EFSCORRUPTED;
1178 f2fs_err(sbi, "%s: inode ino=%lx has corrupted node block, from:%lu addrs:%u",
1179 __func__, inode->i_ino,
1180 from, ADDRS_PER_INODE(inode));
1181 set_sbi_flag(sbi, SBI_NEED_FSCK);
1182 }
9039d835 1183 trace_f2fs_truncate_inode_blocks_exit(inode, level);
adb6dc19 1184 return level;
9039d835 1185 }
ff373558 1186
1cf6b567 1187 folio = f2fs_get_inode_folio(sbi, inode->i_ino);
520b17e0
MWO		 1188 if (IS_ERR(folio)) {
1189 trace_f2fs_truncate_inode_blocks_exit(inode, PTR_ERR(folio));
1190 return PTR_ERR(folio);
51dd6249 1191 }
e05df3b1 1192
214235c2 1193 set_new_dnode(&dn, inode, folio, NULL, 0);
520b17e0 1194 folio_unlock(folio);
e05df3b1 1195
9d717807 1196 ri = F2FS_INODE(folio);
e05df3b1
JK		 1197 switch (level) {
1198 case 0:
1199 case 1:
1200 nofs = noffset[1];
1201 break;
1202 case 2:
1203 nofs = noffset[1];
1204 if (!offset[level - 1])
1205 goto skip_partial;
58bfaf44 1206 err = truncate_partial_nodes(&dn, ri, offset, level);
e05df3b1
JK		 1207 if (err < 0 && err != -ENOENT)
1208 goto fail;
1209 nofs += 1 + NIDS_PER_BLOCK;
1210 break;
1211 case 3:
1212 nofs = 5 + 2 * NIDS_PER_BLOCK;
1213 if (!offset[level - 1])
1214 goto skip_partial;
58bfaf44 1215 err = truncate_partial_nodes(&dn, ri, offset, level);
e05df3b1
JK		 1216 if (err < 0 && err != -ENOENT)
1217 goto fail;
1218 break;
1219 default:
1220 BUG();
1221 }
1222
1223skip_partial:
1224 while (cont) {
fddd722e 1225 dn.nid = get_nid(folio, offset[0], true);
e05df3b1
JK		 1226 switch (offset[0]) {
1227 case NODE_DIR1_BLOCK:
1228 case NODE_DIR2_BLOCK:
1229 err = truncate_dnode(&dn);
1230 break;
1231
1232 case NODE_IND1_BLOCK:
1233 case NODE_IND2_BLOCK:
1234 err = truncate_nodes(&dn, nofs, offset[1], 2);
1235 break;
1236
1237 case NODE_DIND_BLOCK:
1238 err = truncate_nodes(&dn, nofs, offset[1], 3);
1239 cont = 0;
1240 break;
1241
1242 default:
1243 BUG();
1244 }
92c556ed 1245 if (err == -ENOENT) {
520b17e0 1246 set_sbi_flag(F2FS_F_SB(folio), SBI_NEED_FSCK);
92c556ed
CY		 1247 f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
1248 f2fs_err_ratelimited(sbi,
1249 "truncate node fail, ino:%lu, nid:%u, "
1250 "offset[0]:%d, offset[1]:%d, nofs:%d",
1251 inode->i_ino, dn.nid, offset[0],
1252 offset[1], nofs);
1253 err = 0;
1254 }
1255 if (err < 0)
e05df3b1 1256 goto fail;
fddd722e 1257 if (offset[1] == 0 && get_nid(folio, offset[0], true)) {
520b17e0 1258 folio_lock(folio);
019a8912 1259 BUG_ON(!is_node_folio(folio));
66bca01b 1260 set_nid(folio, offset[0], 0, true);
520b17e0 1261 folio_unlock(folio);
e05df3b1
JK		 1262 }
1263 offset[1] = 0;
1264 offset[0]++;
1265 nofs += err;
1266 }
1267fail:
520b17e0 1268 f2fs_folio_put(folio, false);
51dd6249 1269 trace_f2fs_truncate_inode_blocks_exit(inode, err);
e05df3b1
JK		 1270 return err > 0 ? 0 : err;
1271}
1272
9c77f754 1273/* caller must lock inode page */
4d57b86d 1274int f2fs_truncate_xattr_node(struct inode *inode)
4f16fb0f 1275{
4081363f 1276 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
4f16fb0f
JK		 1277 nid_t nid = F2FS_I(inode)->i_xattr_nid;
1278 struct dnode_of_data dn;
a8a4ad25 1279 struct folio *nfolio;
7735730d 1280 int err;
4f16fb0f
JK		 1281
1282 if (!nid)
1283 return 0;
1284
a8a4ad25
MWO		 1285 nfolio = f2fs_get_xnode_folio(sbi, nid);
1286 if (IS_ERR(nfolio))
1287 return PTR_ERR(nfolio);
4f16fb0f 1288
214235c2 1289 set_new_dnode(&dn, inode, NULL, nfolio, nid);
7735730d
CY		 1290 err = truncate_node(&dn);
1291 if (err) {
a8a4ad25 1292 f2fs_folio_put(nfolio, true);
7735730d
CY		 1293 return err;
1294 }
1295
205b9822 1296 f2fs_i_xnid_write(inode, 0);
65985d93 1297
4f16fb0f
JK		 1298 return 0;
1299}
1300
39936837 1301/*
4f4124d0
CY		 1302 * Caller should grab and release a rwsem by calling f2fs_lock_op() and
1303 * f2fs_unlock_op().
39936837 1304 */
4d57b86d 1305int f2fs_remove_inode_page(struct inode *inode)
e05df3b1 1306{
e05df3b1 1307 struct dnode_of_data dn;
13ec7297 1308 int err;
e05df3b1 1309
c2e69583 1310 set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino);
4d57b86d 1311 err = f2fs_get_dnode_of_data(&dn, 0, LOOKUP_NODE);
13ec7297
CY		 1312 if (err)
1313 return err;
e05df3b1 1314
4d57b86d 1315 err = f2fs_truncate_xattr_node(inode);
13ec7297 1316 if (err) {
c2e69583 1317 f2fs_put_dnode(&dn);
13ec7297 1318 return err;
e05df3b1 1319 }
c2e69583
JK		 1320
1321 /* remove potential inline_data blocks */
cf5817ce
JK		 1322 if (!IS_DEVICE_ALIASING(inode) &&
1323 (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
1324 S_ISLNK(inode->i_mode)))
4d57b86d 1325 f2fs_truncate_data_blocks_range(&dn, 1);
c2e69583 1326
e1c42045 1327 /* 0 is possible, after f2fs_new_inode() has failed */
8d714f8a
JK		 1328 if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) {
1329 f2fs_put_dnode(&dn);
1330 return -EIO;
1331 }
8b6810f8
CY		 1332
1333 if (unlikely(inode->i_blocks != 0 && inode->i_blocks != 8)) {
097a7686
CY		 1334 f2fs_warn(F2FS_I_SB(inode),
1335 "f2fs_remove_inode_page: inconsistent i_blocks, ino:%lu, iblocks:%llu",
1336 inode->i_ino, (unsigned long long)inode->i_blocks);
8b6810f8
CY		 1337 set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK);
1338 }
c2e69583
JK		 1339
1340 /* will put inode & node pages */
7735730d
CY		 1341 err = truncate_node(&dn);
1342 if (err) {
1343 f2fs_put_dnode(&dn);
1344 return err;
1345 }
13ec7297 1346 return 0;
e05df3b1
JK		 1347}
1348
7c99299c 1349struct folio *f2fs_new_inode_folio(struct inode *inode)
e05df3b1 1350{
e05df3b1
JK		 1351 struct dnode_of_data dn;
1352
1353 /* allocate inode page for new inode */
1354 set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino);
44a83ff6 1355
7c99299c
MWO		 1356 /* caller should f2fs_folio_put(folio, true); */
1357 return f2fs_new_node_folio(&dn, 0);
e05df3b1
JK		 1358}
1359
cfa80914 1360struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs)
e05df3b1 1361{
4081363f 1362 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
25cc5d3b 1363 struct node_info new_ni;
cfa80914 1364 struct folio *folio;
e05df3b1
JK		 1365 int err;
1366
91942321 1367 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
e05df3b1
JK		 1368 return ERR_PTR(-EPERM);
1369
cfa80914
MWO		 1370 folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), dn->nid, false);
1371 if (IS_ERR(folio))
1372 return folio;
e05df3b1 1373
0abd675e 1374 if (unlikely((err = inc_valid_node_count(sbi, dn->inode, !ofs))))
9c02740c 1375 goto fail;
0abd675e 1376
25cc5d3b 1377#ifdef CONFIG_F2FS_CHECK_FS
a9419b63 1378 err = f2fs_get_node_info(sbi, dn->nid, &new_ni, false);
7735730d
CY		 1379 if (err) {
1380 dec_valid_node_count(sbi, dn->inode, !ofs);
1381 goto fail;
1382 }
141170b7
CY		 1383 if (unlikely(new_ni.blk_addr != NULL_ADDR)) {
1384 err = -EFSCORRUPTED;
0fa4e57c 1385 dec_valid_node_count(sbi, dn->inode, !ofs);
141170b7 1386 set_sbi_flag(sbi, SBI_NEED_FSCK);
81520c68 1387 f2fs_warn_ratelimited(sbi,
6b1ad395 1388 "f2fs_new_node_folio: inconsistent nat entry, "
81520c68
CY		 1389 "ino:%u, nid:%u, blkaddr:%u, ver:%u, flag:%u",
1390 new_ni.ino, new_ni.nid, new_ni.blk_addr,
1391 new_ni.version, new_ni.flag);
1392 f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT);
141170b7
CY		 1393 goto fail;
1394 }
25cc5d3b
JK		 1395#endif
1396 new_ni.nid = dn->nid;
e05df3b1 1397 new_ni.ino = dn->inode->i_ino;
25cc5d3b
JK		 1398 new_ni.blk_addr = NULL_ADDR;
1399 new_ni.flag = 0;
1400 new_ni.version = 0;
479f40c4 1401 set_node_addr(sbi, &new_ni, NEW_ADDR, false);
9c02740c 1402
cfa80914 1403 f2fs_folio_wait_writeback(folio, NODE, true, true);
06bf1182 1404 fill_node_footer(folio, dn->nid, dn->inode->i_ino, ofs, true);
53987453 1405 set_cold_node(folio, S_ISDIR(dn->inode->i_mode));
cfa80914
MWO		 1406 if (!folio_test_uptodate(folio))
1407 folio_mark_uptodate(folio);
1408 if (folio_mark_dirty(folio))
12719ae1 1409 dn->node_changed = true;
e05df3b1 1410
4bc8e9bc 1411 if (f2fs_has_xattr_block(ofs))
205b9822 1412 f2fs_i_xnid_write(dn->inode, dn->nid);
479bd73a 1413
e05df3b1
JK		 1414 if (ofs == 0)
1415 inc_valid_inode_count(sbi);
cfa80914 1416 return folio;
e05df3b1 1417fail:
f16ebe0d 1418 clear_node_folio_dirty(folio);
cfa80914 1419 f2fs_folio_put(folio, true);
e05df3b1
JK		 1420 return ERR_PTR(err);
1421}
1422
56ae674c
JK		 1423/*
1424 * Caller should do after getting the following values.
f0fac66f
MWO		 1425 * 0: f2fs_folio_put(folio, false)
1426 * LOCKED_PAGE or error: f2fs_folio_put(folio, true)
56ae674c 1427 */
f0fac66f 1428static int read_node_folio(struct folio *folio, blk_opf_t op_flags)
e05df3b1 1429{
f0fac66f 1430 struct f2fs_sb_info *sbi = F2FS_F_SB(folio);
e05df3b1 1431 struct node_info ni;
cf04e8eb 1432 struct f2fs_io_info fio = {
05ca3632 1433 .sbi = sbi,
cf04e8eb 1434 .type = NODE,
04d328de
MC		 1435 .op = REQ_OP_READ,
1436 .op_flags = op_flags,
d342b7ad 1437 .folio = folio,
4375a336 1438 .encrypted_page = NULL,
cf04e8eb 1439 };
7735730d 1440 int err;
e05df3b1 1441
2eaa98e5 1442 if (folio_test_uptodate(folio)) {
7c213e98 1443 if (!f2fs_inode_chksum_verify(sbi, folio)) {
2eaa98e5 1444 folio_clear_uptodate(folio);
10f966bb 1445 return -EFSBADCRC;
b42b179b 1446 }
3bdad3c7 1447 return LOCKED_PAGE;
54c55c4e 1448 }
3bdad3c7 1449
2eaa98e5 1450 err = f2fs_get_node_info(sbi, folio->index, &ni, false);
7735730d
CY		 1451 if (err)
1452 return err;
e05df3b1 1453
b7ec2061 1454 /* NEW_ADDR can be seen, after cp_error drops some dirty node pages */
e6ecb142 1455 if (unlikely(ni.blk_addr == NULL_ADDR || ni.blk_addr == NEW_ADDR)) {
2eaa98e5 1456 folio_clear_uptodate(folio);
e05df3b1 1457 return -ENOENT;
393ff91f
JK		 1458 }
1459
7a9d7548 1460 fio.new_blkaddr = fio.old_blkaddr = ni.blk_addr;
8b83ac81
CY		 1461
1462 err = f2fs_submit_page_bio(&fio);
1463
1464 if (!err)
34a23525 1465 f2fs_update_iostat(sbi, NULL, FS_NODE_READ_IO, F2FS_BLKSIZE);
8b83ac81
CY		 1466
1467 return err;
e05df3b1
JK		 1468}
1469
0a8165d7 1470/*
e05df3b1
JK		 1471 * Readahead a node page
1472 */
4d57b86d 1473void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid)
e05df3b1 1474{
9d79652e 1475 struct folio *afolio;
56ae674c 1476 int err;
e05df3b1 1477
e8458725
CY		 1478 if (!nid)
1479 return;
4d57b86d 1480 if (f2fs_check_nid_range(sbi, nid))
a4f843bd 1481 return;
e8458725 1482
9d79652e
MWO		 1483 afolio = xa_load(&NODE_MAPPING(sbi)->i_pages, nid);
1484 if (afolio)
393ff91f 1485 return;
e05df3b1 1486
9d79652e
MWO		 1487 afolio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), nid, false);
1488 if (IS_ERR(afolio))
e05df3b1
JK		 1489 return;
1490
f0fac66f 1491 err = read_node_folio(afolio, REQ_RAHEAD);
9d79652e 1492 f2fs_folio_put(afolio, err ? true : false);
e05df3b1
JK		 1493}
1494
1cf6b567 1495static int sanity_check_node_footer(struct f2fs_sb_info *sbi,
f24f7f8c 1496 struct folio *folio, pgoff_t nid,
1cf6b567
CY		 1497 enum node_type ntype)
1498{
a63f2de2 1499 if (unlikely(nid != nid_of_node(folio) ||
a5f3be6e 1500 (ntype == NODE_TYPE_INODE && !IS_INODE(folio)) ||
2aac2538 1501 (ntype == NODE_TYPE_XATTR &&
6d3a7f65 1502 !f2fs_has_xattr_block(ofs_of_node(folio))) ||
1788971e 1503 time_to_inject(sbi, FAULT_INCONSISTENT_FOOTER))) {
1cf6b567
CY		 1504 f2fs_warn(sbi, "inconsistent node block, node_type:%d, nid:%lu, "
1505 "node_footer[nid:%u,ino:%u,ofs:%u,cpver:%llu,blkaddr:%u]",
a63f2de2 1506 ntype, nid, nid_of_node(folio), ino_of_node(folio),
6d3a7f65 1507 ofs_of_node(folio), cpver_of_node(folio),
f24f7f8c 1508 next_blkaddr_of_node(folio));
1cf6b567
CY		 1509 set_sbi_flag(sbi, SBI_NEED_FSCK);
1510 f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER);
1511 return -EFSCORRUPTED;
1512 }
1513 return 0;
1514}
1515
4d417ae2 1516static struct folio *__get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid,
fb733f98 1517 struct folio *parent, int start, enum node_type ntype)
e05df3b1 1518{
48a34c59 1519 struct folio *folio;
0e022ea8 1520 int err;
e05df3b1 1521
e05df3b1
JK		 1522 if (!nid)
1523 return ERR_PTR(-ENOENT);
4d57b86d 1524 if (f2fs_check_nid_range(sbi, nid))
a4f843bd 1525 return ERR_PTR(-EINVAL);
afcb7ca0 1526repeat:
48a34c59
MWO		 1527 folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), nid, false);
1528 if (IS_ERR(folio))
4d417ae2 1529 return folio;
e05df3b1 1530
f0fac66f 1531 err = read_node_folio(folio, 0);
539d3345 1532 if (err < 0)
a7b8618a 1533 goto out_put_err;
539d3345 1534 if (err == LOCKED_PAGE)
56ae674c 1535 goto page_hit;
e05df3b1 1536
0e022ea8 1537 if (parent)
4e9e8f81 1538 f2fs_ra_node_pages(parent, start + 1, MAX_RA_NODE);
c718379b 1539
48a34c59 1540 folio_lock(folio);
0e022ea8 1541
019a8912 1542 if (unlikely(!is_node_folio(folio))) {
48a34c59 1543 f2fs_folio_put(folio, true);
afcb7ca0
JK		 1544 goto repeat;
1545 }
1563ac75 1546
48a34c59 1547 if (unlikely(!folio_test_uptodate(folio))) {
1f258ec1 1548 err = -EIO;
1563ac75 1549 goto out_err;
1f258ec1 1550 }
704956ec 1551
7c213e98 1552 if (!f2fs_inode_chksum_verify(sbi, folio)) {
10f966bb 1553 err = -EFSBADCRC;
704956ec
CY		 1554 goto out_err;
1555 }
e0f56cb4 1556page_hit:
f24f7f8c 1557 err = sanity_check_node_footer(sbi, folio, nid, ntype);
1cf6b567 1558 if (!err)
4d417ae2 1559 return folio;
0c9df7fb 1560out_err:
48a34c59 1561 folio_clear_uptodate(folio);
a7b8618a 1562out_put_err:
f0fac66f 1563 /* ENOENT comes from read_node_folio which is not an error. */
82c7863e 1564 if (err != -ENOENT)
48a34c59
MWO		 1565 f2fs_handle_page_eio(sbi, folio, NODE);
1566 f2fs_folio_put(folio, true);
a7b8618a 1567 return ERR_PTR(err);
e05df3b1
JK		 1568}
1569
aa220ced
MWO		 1570struct folio *f2fs_get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid)
1571{
1572 return __get_node_folio(sbi, nid, NULL, 0, NODE_TYPE_REGULAR);
1573}
1574
1cf6b567 1575struct folio *f2fs_get_inode_folio(struct f2fs_sb_info *sbi, pgoff_t ino)
4d417ae2 1576{
1cf6b567 1577 return __get_node_folio(sbi, ino, NULL, 0, NODE_TYPE_INODE);
4d417ae2
MWO		 1578}
1579
153e4a7f
MWO		 1580struct folio *f2fs_get_xnode_folio(struct f2fs_sb_info *sbi, pgoff_t xnid)
1581{
1582 return __get_node_folio(sbi, xnid, NULL, 0, NODE_TYPE_XATTR);
1583}
1584
fb733f98 1585static struct folio *f2fs_get_node_folio_ra(struct folio *parent, int start)
0e022ea8 1586{
fb733f98 1587 struct f2fs_sb_info *sbi = F2FS_F_SB(parent);
fddd722e 1588 nid_t nid = get_nid(parent, start, false);
0e022ea8 1589
fb733f98 1590 return __get_node_folio(sbi, nid, parent, start, NODE_TYPE_REGULAR);
0e022ea8
CY		 1591}
1592
2049d4fc
JK		 1593static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino)
1594{
1595 struct inode *inode;
a4d07702 1596 struct folio *folio;
0f3311a8 1597 int ret;
2049d4fc
JK		 1598
1599 /* should flush inline_data before evict_inode */
1600 inode = ilookup(sbi->sb, ino);
1601 if (!inode)
1602 return;
1603
a4d07702 1604 folio = f2fs_filemap_get_folio(inode->i_mapping, 0,
01eccef7 1605 FGP_LOCK|FGP_NOWAIT, 0);
a4d07702 1606 if (IS_ERR(folio))
2049d4fc
JK		 1607 goto iput_out;
1608
a4d07702
MWO		 1609 if (!folio_test_uptodate(folio))
1610 goto folio_out;
2049d4fc 1611
a4d07702
MWO		 1612 if (!folio_test_dirty(folio))
1613 goto folio_out;
2049d4fc 1614
a4d07702
MWO		 1615 if (!folio_clear_dirty_for_io(folio))
1616 goto folio_out;
2049d4fc 1617
a4d07702 1618 ret = f2fs_write_inline_data(inode, folio);
0f3311a8 1619 inode_dec_dirty_pages(inode);
4d57b86d 1620 f2fs_remove_dirty_inode(inode);
0f3311a8 1621 if (ret)
a4d07702
MWO		 1622 folio_mark_dirty(folio);
1623folio_out:
1624 f2fs_folio_put(folio, true);
2049d4fc
JK		 1625iput_out:
1626 iput(inode);
1627}
1628
e11a3113 1629static struct folio *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino)
608514de 1630{
028a63a6 1631 pgoff_t index;
4f4a4f0f 1632 struct folio_batch fbatch;
18f3814f 1633 struct folio *last_folio = NULL;
4f4a4f0f 1634 int nr_folios;
608514de 1635
4f4a4f0f 1636 folio_batch_init(&fbatch);
608514de 1637 index = 0;
028a63a6 1638
4f4a4f0f
VMO		 1639 while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index,
1640 (pgoff_t)-1, PAGECACHE_TAG_DIRTY,
1641 &fbatch))) {
028a63a6 1642 int i;
608514de 1643
4f4a4f0f 1644 for (i = 0; i < nr_folios; i++) {
18f3814f 1645 struct folio *folio = fbatch.folios[i];
608514de
JK		 1646
1647 if (unlikely(f2fs_cp_error(sbi))) {
18f3814f 1648 f2fs_folio_put(last_folio, false);
4f4a4f0f 1649 folio_batch_release(&fbatch);
608514de
JK		 1650 return ERR_PTR(-EIO);
1651 }
1652
fb92a5c9 1653 if (!IS_DNODE(folio) || !is_cold_node(folio))
608514de 1654 continue;
28fde0d7 1655 if (ino_of_node(folio) != ino)
608514de
JK		 1656 continue;
1657
18f3814f 1658 folio_lock(folio);
608514de 1659
019a8912 1660 if (unlikely(!is_node_folio(folio))) {
608514de 1661continue_unlock:
18f3814f 1662 folio_unlock(folio);
608514de
JK		 1663 continue;
1664 }
28fde0d7 1665 if (ino_of_node(folio) != ino)
608514de
JK		 1666 goto continue_unlock;
1667
18f3814f 1668 if (!folio_test_dirty(folio)) {
608514de
JK		 1669 /* someone wrote it for us */
1670 goto continue_unlock;
1671 }
1672
18f3814f
MWO		 1673 if (last_folio)
1674 f2fs_folio_put(last_folio, false);
608514de 1675
18f3814f
MWO		 1676 folio_get(folio);
1677 last_folio = folio;
1678 folio_unlock(folio);
608514de 1679 }
4f4a4f0f 1680 folio_batch_release(&fbatch);
608514de
JK		 1681 cond_resched();
1682 }
e11a3113 1683 return last_folio;
608514de
JK		 1684}
1685
80f31d2a 1686static bool __write_node_folio(struct folio *folio, bool atomic, bool *submitted,
b0af6d49 1687 struct writeback_control *wbc, bool do_balance,
50fa53ec 1688 enum iostat_type io_type, unsigned int *seq_id)
faa24895 1689{
72206620 1690 struct f2fs_sb_info *sbi = F2FS_F_SB(folio);
faa24895
JK		 1691 nid_t nid;
1692 struct node_info ni;
1693 struct f2fs_io_info fio = {
1694 .sbi = sbi,
28fde0d7 1695 .ino = ino_of_node(folio),
faa24895
JK		 1696 .type = NODE,
1697 .op = REQ_OP_WRITE,
1698 .op_flags = wbc_to_write_flags(wbc),
d342b7ad 1699 .folio = folio,
faa24895 1700 .encrypted_page = NULL,
2eae077e 1701 .submitted = 0,
b0af6d49 1702 .io_type = io_type,
578c6478 1703 .io_wbc = wbc,
faa24895 1704 };
50fa53ec 1705 unsigned int seq;
faa24895 1706
4deccfbd 1707 trace_f2fs_writepage(folio, NODE);
faa24895 1708
6d7c865c 1709 if (unlikely(f2fs_cp_error(sbi))) {
b62e71be
CY		 1710 /* keep node pages in remount-ro mode */
1711 if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_READONLY)
1712 goto redirty_out;
4deccfbd 1713 folio_clear_uptodate(folio);
28607bf3 1714 dec_page_count(sbi, F2FS_DIRTY_NODES);
4deccfbd 1715 folio_unlock(folio);
80f31d2a 1716 return true;
6d7c865c 1717 }
db198ae0 1718
faa24895
JK		 1719 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1720 goto redirty_out;
faa24895 1721
100c0655
JK		 1722 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
1723 wbc->sync_mode == WB_SYNC_NONE &&
fb92a5c9 1724 IS_DNODE(folio) && is_cold_node(folio))
fd8c8caf
CY		 1725 goto redirty_out;
1726
faa24895 1727 /* get old block addr of this node page */
a63f2de2 1728 nid = nid_of_node(folio);
4deccfbd 1729 f2fs_bug_on(sbi, folio->index != nid);
faa24895 1730
a9419b63 1731 if (f2fs_get_node_info(sbi, nid, &ni, !do_balance))
7735730d
CY		 1732 goto redirty_out;
1733
402dd9f0 1734 f2fs_down_read(&sbi->node_write);
faa24895 1735
faa24895
JK		 1736 /* This page is already truncated */
1737 if (unlikely(ni.blk_addr == NULL_ADDR)) {
4deccfbd 1738 folio_clear_uptodate(folio);
faa24895 1739 dec_page_count(sbi, F2FS_DIRTY_NODES);
e4544b63 1740 f2fs_up_read(&sbi->node_write);
4deccfbd 1741 folio_unlock(folio);
80f31d2a 1742 return true;
faa24895
JK		 1743 }
1744
c9b60788 1745 if (__is_valid_data_blkaddr(ni.blk_addr) &&
93770ab7
CY		 1746 !f2fs_is_valid_blkaddr(sbi, ni.blk_addr,
1747 DATA_GENERIC_ENHANCE)) {
e4544b63 1748 f2fs_up_read(&sbi->node_write);
c9b60788 1749 goto redirty_out;
89d13c38 1750 }
c9b60788 1751
b722ff8a 1752 if (atomic && !test_opt(sbi, NOBARRIER))
e7c75ab0
JK		 1753 fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
1754
dc5a9412 1755 /* should add to global list before clearing PAGECACHE status */
1a58a41c 1756 if (f2fs_in_warm_node_list(sbi, folio)) {
963da02b 1757 seq = f2fs_add_fsync_node_entry(sbi, folio);
50fa53ec
CY		 1758 if (seq_id)
1759 *seq_id = seq;
1760 }
1761
4deccfbd 1762 folio_start_writeback(folio);
dc5a9412 1763
faa24895 1764 fio.old_blkaddr = ni.blk_addr;
4d57b86d 1765 f2fs_do_write_node_page(nid, &fio);
ac576da7 1766 set_node_addr(sbi, &ni, fio.new_blkaddr, is_fsync_dnode(folio));
faa24895 1767 dec_page_count(sbi, F2FS_DIRTY_NODES);
e4544b63 1768 f2fs_up_read(&sbi->node_write);
faa24895 1769
4deccfbd 1770 folio_unlock(folio);
faa24895 1771
d68f735b 1772 if (unlikely(f2fs_cp_error(sbi))) {
b9109b0e 1773 f2fs_submit_merged_write(sbi, NODE);
d68f735b
JK		 1774 submitted = NULL;
1775 }
1776 if (submitted)
1777 *submitted = fio.submitted;
faa24895 1778
401db79f
YS		 1779 if (do_balance)
1780 f2fs_balance_fs(sbi, false);
80f31d2a 1781 return true;
faa24895
JK		 1782
1783redirty_out:
4deccfbd 1784 folio_redirty_for_writepage(wbc, folio);
80f31d2a
CH		 1785 folio_unlock(folio);
1786 return false;
faa24895
JK		 1787}
1788
c795d9db 1789int f2fs_move_node_folio(struct folio *node_folio, int gc_type)
f15194fc 1790{
48018b4c
CY		 1791 int err = 0;
1792
f15194fc
YH		 1793 if (gc_type == FG_GC) {
1794 struct writeback_control wbc = {
1795 .sync_mode = WB_SYNC_ALL,
1796 .nr_to_write = 1,
f15194fc
YH		 1797 };
1798
c795d9db 1799 f2fs_folio_wait_writeback(node_folio, NODE, true, true);
8d64d365 1800
c795d9db 1801 folio_mark_dirty(node_folio);
8d64d365 1802
c795d9db 1803 if (!folio_clear_dirty_for_io(node_folio)) {
48018b4c 1804 err = -EAGAIN;
f15194fc 1805 goto out_page;
48018b4c 1806 }
f15194fc 1807
80f31d2a
CH		 1808 if (!__write_node_folio(node_folio, false, NULL,
1809 &wbc, false, FS_GC_NODE_IO, NULL))
48018b4c 1810 err = -EAGAIN;
f15194fc
YH		 1811 goto release_page;
1812 } else {
1813 /* set page dirty and write it */
c795d9db
MWO		 1814 if (!folio_test_writeback(node_folio))
1815 folio_mark_dirty(node_folio);
f15194fc
YH		 1816 }
1817out_page:
c795d9db 1818 folio_unlock(node_folio);
f15194fc 1819release_page:
c795d9db 1820 f2fs_folio_put(node_folio, false);
48018b4c 1821 return err;
f15194fc
YH		 1822}
1823
4d57b86d 1824int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode,
50fa53ec
CY		 1825 struct writeback_control *wbc, bool atomic,
1826 unsigned int *seq_id)
e05df3b1 1827{
028a63a6 1828 pgoff_t index;
e6e46e1e 1829 struct folio_batch fbatch;
c267ec15 1830 int ret = 0;
e11a3113 1831 struct folio *last_folio = NULL;
608514de 1832 bool marked = false;
26de9b11 1833 nid_t ino = inode->i_ino;
e6e46e1e 1834 int nr_folios;
bab475c5 1835 int nwritten = 0;
52681375 1836
608514de 1837 if (atomic) {
e11a3113
MWO		 1838 last_folio = last_fsync_dnode(sbi, ino);
1839 if (IS_ERR_OR_NULL(last_folio))
1840 return PTR_ERR_OR_ZERO(last_folio);
608514de
JK		 1841 }
1842retry:
e6e46e1e 1843 folio_batch_init(&fbatch);
52681375 1844 index = 0;
028a63a6 1845
e6e46e1e
VMO		 1846 while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index,
1847 (pgoff_t)-1, PAGECACHE_TAG_DIRTY,
1848 &fbatch))) {
028a63a6 1849 int i;
52681375 1850
e6e46e1e 1851 for (i = 0; i < nr_folios; i++) {
e23bebc3 1852 struct folio *folio = fbatch.folios[i];
d68f735b 1853 bool submitted = false;
52681375
JK		 1854
1855 if (unlikely(f2fs_cp_error(sbi))) {
e11a3113 1856 f2fs_folio_put(last_folio, false);
e6e46e1e 1857 folio_batch_release(&fbatch);
9de69279
CY		 1858 ret = -EIO;
1859 goto out;
52681375
JK		 1860 }
1861
fb92a5c9 1862 if (!IS_DNODE(folio) || !is_cold_node(folio))
52681375 1863 continue;
28fde0d7 1864 if (ino_of_node(folio) != ino)
52681375
JK		 1865 continue;
1866
e23bebc3 1867 folio_lock(folio);
52681375 1868
019a8912 1869 if (unlikely(!is_node_folio(folio))) {
52681375 1870continue_unlock:
e23bebc3 1871 folio_unlock(folio);
52681375
JK		 1872 continue;
1873 }
28fde0d7 1874 if (ino_of_node(folio) != ino)
52681375
JK		 1875 goto continue_unlock;
1876
e11a3113 1877 if (!folio_test_dirty(folio) && folio != last_folio) {
52681375
JK		 1878 /* someone wrote it for us */
1879 goto continue_unlock;
1880 }
1881
e23bebc3 1882 f2fs_folio_wait_writeback(folio, NODE, true, true);
52681375 1883
b07bfa70 1884 set_fsync_mark(folio, 0);
4f3466d7 1885 set_dentry_mark(folio, 0);
d29fd172 1886
e11a3113 1887 if (!atomic || folio == last_folio) {
b07bfa70 1888 set_fsync_mark(folio, 1);
47c8ebcc 1889 percpu_counter_inc(&sbi->rf_node_block_count);
a5f3be6e 1890 if (IS_INODE(folio)) {
26de9b11
JK		 1891 if (is_inode_flag_set(inode,
1892 FI_DIRTY_INODE))
f9237928 1893 f2fs_update_inode(inode, folio);
4f3466d7 1894 set_dentry_mark(folio,
4d57b86d 1895 f2fs_need_dentry_mark(sbi, ino));
26de9b11 1896 }
a87aff1d 1897 /* may be written by other thread */
e23bebc3
MWO		 1898 if (!folio_test_dirty(folio))
1899 folio_mark_dirty(folio);
608514de
JK		 1900 }
1901
e23bebc3 1902 if (!folio_clear_dirty_for_io(folio))
608514de 1903 goto continue_unlock;
52681375 1904
80f31d2a 1905 if (!__write_node_folio(folio, atomic &&
e11a3113 1906 folio == last_folio,
b0af6d49 1907 &submitted, wbc, true,
0638f28b 1908 FS_NODE_IO, seq_id)) {
e11a3113 1909 f2fs_folio_put(last_folio, false);
0638f28b
CH		 1910 folio_batch_release(&fbatch);
1911 ret = -EIO;
1912 goto out;
608514de 1913 }
0638f28b
CH		 1914 if (submitted)
1915 nwritten++;
3f5f4959 1916
e11a3113 1917 if (folio == last_folio) {
e23bebc3 1918 f2fs_folio_put(folio, false);
0638f28b 1919 folio_batch_release(&fbatch);
608514de 1920 marked = true;
0638f28b 1921 goto out;
c267ec15 1922 }
52681375 1923 }
e6e46e1e 1924 folio_batch_release(&fbatch);
52681375 1925 cond_resched();
52681375 1926 }
0638f28b 1927 if (atomic && !marked) {
dcbb4c10 1928 f2fs_debug(sbi, "Retry to write fsync mark: ino=%u, idx=%lx",
e11a3113
MWO		 1929 ino, last_folio->index);
1930 folio_lock(last_folio);
1931 f2fs_folio_wait_writeback(last_folio, NODE, true, true);
1932 folio_mark_dirty(last_folio);
1933 folio_unlock(last_folio);
608514de
JK		 1934 goto retry;
1935 }
9de69279 1936out:
bab475c5
CY		 1937 if (nwritten)
1938 f2fs_submit_merged_write_cond(sbi, NULL, NULL, ino, NODE);
0638f28b 1939 return ret;
52681375
JK		 1940}
1941
052a82d8
CY		 1942static int f2fs_match_ino(struct inode *inode, unsigned long ino, void *data)
1943{
1944 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1945 bool clean;
1946
1947 if (inode->i_ino != ino)
1948 return 0;
1949
1950 if (!is_inode_flag_set(inode, FI_DIRTY_INODE))
1951 return 0;
1952
1953 spin_lock(&sbi->inode_lock[DIRTY_META]);
1954 clean = list_empty(&F2FS_I(inode)->gdirty_list);
1955 spin_unlock(&sbi->inode_lock[DIRTY_META]);
1956
1957 if (clean)
1958 return 0;
1959
1960 inode = igrab(inode);
1961 if (!inode)
1962 return 0;
1963 return 1;
1964}
1965
de90f761 1966static bool flush_dirty_inode(struct folio *folio)
052a82d8 1967{
de90f761 1968 struct f2fs_sb_info *sbi = F2FS_F_SB(folio);
052a82d8 1969 struct inode *inode;
28fde0d7 1970 nid_t ino = ino_of_node(folio);
052a82d8
CY		 1971
1972 inode = find_inode_nowait(sbi->sb, ino, f2fs_match_ino, NULL);
1973 if (!inode)
1974 return false;
1975
f9237928 1976 f2fs_update_inode(inode, folio);
de90f761 1977 folio_unlock(folio);
052a82d8
CY		 1978
1979 iput(inode);
1980 return true;
1981}
1982
68e79baf 1983void f2fs_flush_inline_data(struct f2fs_sb_info *sbi)
34c061ad
SL		 1984{
1985 pgoff_t index = 0;
a40a4ad1
VMO		 1986 struct folio_batch fbatch;
1987 int nr_folios;
34c061ad 1988
a40a4ad1 1989 folio_batch_init(&fbatch);
34c061ad 1990
a40a4ad1
VMO		 1991 while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index,
1992 (pgoff_t)-1, PAGECACHE_TAG_DIRTY,
1993 &fbatch))) {
34c061ad
SL		 1994 int i;
1995
a40a4ad1 1996 for (i = 0; i < nr_folios; i++) {
015d9c56 1997 struct folio *folio = fbatch.folios[i];
34c061ad 1998
a5f3be6e 1999 if (!IS_INODE(folio))
34c061ad
SL		 2000 continue;
2001
015d9c56 2002 folio_lock(folio);
34c061ad 2003
019a8912 2004 if (unlikely(!is_node_folio(folio)))
015d9c56
MWO		 2005 goto unlock;
2006 if (!folio_test_dirty(folio))
2007 goto unlock;
34c061ad
SL		 2008
2009 /* flush inline_data, if it's async context. */
4ecaf580
MWO		 2010 if (folio_test_f2fs_inline(folio)) {
2011 folio_clear_f2fs_inline(folio);
015d9c56 2012 folio_unlock(folio);
28fde0d7 2013 flush_inline_data(sbi, ino_of_node(folio));
34c061ad
SL		 2014 continue;
2015 }
015d9c56
MWO		 2016unlock:
2017 folio_unlock(folio);
34c061ad 2018 }
a40a4ad1 2019 folio_batch_release(&fbatch);
34c061ad
SL		 2020 cond_resched();
2021 }
34c061ad
SL		 2022}
2023
4d57b86d
CY		 2024int f2fs_sync_node_pages(struct f2fs_sb_info *sbi,
2025 struct writeback_control *wbc,
b0af6d49 2026 bool do_balance, enum iostat_type io_type)
52681375 2027{
028a63a6 2028 pgoff_t index;
7525486a 2029 struct folio_batch fbatch;
52681375 2030 int step = 0;
12bb0a8f 2031 int nwritten = 0;
3f5f4959 2032 int ret = 0;
7525486a 2033 int nr_folios, done = 0;
e05df3b1 2034
7525486a 2035 folio_batch_init(&fbatch);
e05df3b1
JK		 2036
2037next_step:
2038 index = 0;
028a63a6 2039
7525486a
VMO		 2040 while (!done && (nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi),
2041 &index, (pgoff_t)-1, PAGECACHE_TAG_DIRTY,
2042 &fbatch))) {
028a63a6 2043 int i;
e05df3b1 2044
7525486a 2045 for (i = 0; i < nr_folios; i++) {
5d0a9128 2046 struct folio *folio = fbatch.folios[i];
d68f735b 2047 bool submitted = false;
e05df3b1 2048
c29fd0c0
CY		 2049 /* give a priority to WB_SYNC threads */
2050 if (atomic_read(&sbi->wb_sync_req[NODE]) &&
2051 wbc->sync_mode == WB_SYNC_NONE) {
2052 done = 1;
2053 break;
2054 }
2055
e05df3b1
JK		 2056 /*
2057 * flushing sequence with step:
2058 * 0. indirect nodes
2059 * 1. dentry dnodes
2060 * 2. file dnodes
2061 */
fb92a5c9 2062 if (step == 0 && IS_DNODE(folio))
e05df3b1 2063 continue;
fb92a5c9 2064 if (step == 1 && (!IS_DNODE(folio) ||
1fd0dffd 2065 is_cold_node(folio)))
e05df3b1 2066 continue;
fb92a5c9 2067 if (step == 2 && (!IS_DNODE(folio) ||
1fd0dffd 2068 !is_cold_node(folio)))
e05df3b1 2069 continue;
9a4cbc9e 2070lock_node:
4b270a8c 2071 if (wbc->sync_mode == WB_SYNC_ALL)
5d0a9128
MWO		 2072 folio_lock(folio);
2073 else if (!folio_trylock(folio))
e05df3b1
JK		 2074 continue;
2075
019a8912 2076 if (unlikely(!is_node_folio(folio))) {
e05df3b1 2077continue_unlock:
5d0a9128 2078 folio_unlock(folio);
e05df3b1
JK		 2079 continue;
2080 }
e05df3b1 2081
5d0a9128 2082 if (!folio_test_dirty(folio)) {
e05df3b1
JK		 2083 /* someone wrote it for us */
2084 goto continue_unlock;
2085 }
2086
b0f3b87f
JK		 2087 /* flush inline_data/inode, if it's async context. */
2088 if (!do_balance)
2089 goto write_node;
2090
2091 /* flush inline_data */
4ecaf580
MWO		 2092 if (folio_test_f2fs_inline(folio)) {
2093 folio_clear_f2fs_inline(folio);
5d0a9128 2094 folio_unlock(folio);
28fde0d7 2095 flush_inline_data(sbi, ino_of_node(folio));
9a4cbc9e 2096 goto lock_node;
052a82d8
CY		 2097 }
2098
2099 /* flush dirty inode */
a5f3be6e 2100 if (IS_INODE(folio) && flush_dirty_inode(folio))
7859e97f 2101 goto lock_node;
b0f3b87f 2102write_node:
5d0a9128 2103 f2fs_folio_wait_writeback(folio, NODE, true, true);
fa3d2bdf 2104
5d0a9128 2105 if (!folio_clear_dirty_for_io(folio))
e05df3b1
JK		 2106 goto continue_unlock;
2107
b07bfa70 2108 set_fsync_mark(folio, 0);
4f3466d7 2109 set_dentry_mark(folio, 0);
52746519 2110
80f31d2a
CH		 2111 if (!__write_node_folio(folio, false, &submitted,
2112 wbc, do_balance, io_type, NULL)) {
43ba56a0
CY		 2113 folio_batch_release(&fbatch);
2114 ret = -EIO;
2115 goto out;
43ba56a0 2116 }
80f31d2a
CH		 2117 if (submitted)
2118 nwritten++;
e05df3b1
JK		 2119
2120 if (--wbc->nr_to_write == 0)
2121 break;
2122 }
7525486a 2123 folio_batch_release(&fbatch);
e05df3b1
JK		 2124 cond_resched();
2125
2126 if (wbc->nr_to_write == 0) {
2127 step = 2;
2128 break;
2129 }
2130 }
2131
2132 if (step < 2) {
100c0655
JK		 2133 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
2134 wbc->sync_mode == WB_SYNC_NONE && step == 1)
fd8c8caf 2135 goto out;
e05df3b1
JK		 2136 step++;
2137 goto next_step;
2138 }
fd8c8caf 2139out:
3f5f4959 2140 if (nwritten)
b9109b0e 2141 f2fs_submit_merged_write(sbi, NODE);
db198ae0
CY		 2142
2143 if (unlikely(f2fs_cp_error(sbi)))
2144 return -EIO;
3f5f4959 2145 return ret;
e05df3b1
JK		 2146}
2147
50fa53ec
CY		 2148int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi,
2149 unsigned int seq_id)
cfe58f9d 2150{
50fa53ec 2151 struct fsync_node_entry *fn;
50fa53ec
CY		 2152 struct list_head *head = &sbi->fsync_node_list;
2153 unsigned long flags;
2154 unsigned int cur_seq_id = 0;
cfe58f9d 2155
50fa53ec 2156 while (seq_id && cur_seq_id < seq_id) {
963da02b
MWO		 2157 struct folio *folio;
2158
50fa53ec
CY		 2159 spin_lock_irqsave(&sbi->fsync_node_lock, flags);
2160 if (list_empty(head)) {
2161 spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);
2162 break;
2163 }
2164 fn = list_first_entry(head, struct fsync_node_entry, list);
2165 if (fn->seq_id > seq_id) {
2166 spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);
2167 break;
2168 }
2169 cur_seq_id = fn->seq_id;
963da02b
MWO		 2170 folio = fn->folio;
2171 folio_get(folio);
50fa53ec 2172 spin_unlock_irqrestore(&sbi->fsync_node_lock, flags);
4ef51a8f 2173
963da02b 2174 f2fs_folio_wait_writeback(folio, NODE, true, false);
cfe58f9d 2175
963da02b 2176 folio_put(folio);
cfe58f9d
JK		 2177 }
2178
08c3eab5 2179 return filemap_check_errors(NODE_MAPPING(sbi));
cfe58f9d
JK		 2180}
2181
e05df3b1
JK		 2182static int f2fs_write_node_pages(struct address_space *mapping,
2183 struct writeback_control *wbc)
2184{
4081363f 2185 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
9dfa1baf 2186 struct blk_plug plug;
50c8cdb3 2187 long diff;
e05df3b1 2188
0771fcc7
CY		 2189 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
2190 goto skip_write;
2191
4660f9c0 2192 /* balancing f2fs's metadata in background */
7bcd0cfa 2193 f2fs_balance_fs_bg(sbi, true);
e05df3b1 2194
a7fdffbd 2195 /* collect a number of dirty node pages and write together */
812a9597
JK		 2196 if (wbc->sync_mode != WB_SYNC_ALL &&
2197 get_pages(sbi, F2FS_DIRTY_NODES) <
2198 nr_pages_to_skip(sbi, NODE))
d3baf95d 2199 goto skip_write;
a7fdffbd 2200
c29fd0c0
CY		 2201 if (wbc->sync_mode == WB_SYNC_ALL)
2202 atomic_inc(&sbi->wb_sync_req[NODE]);
34415099
CY		 2203 else if (atomic_read(&sbi->wb_sync_req[NODE])) {
2204 /* to avoid potential deadlock */
2205 if (current->plug)
2206 blk_finish_plug(current->plug);
c29fd0c0 2207 goto skip_write;
34415099 2208 }
c29fd0c0 2209
d31c7c3f
YH		 2210 trace_f2fs_writepages(mapping->host, wbc, NODE);
2211
50c8cdb3 2212 diff = nr_pages_to_write(sbi, NODE, wbc);
9dfa1baf 2213 blk_start_plug(&plug);
4d57b86d 2214 f2fs_sync_node_pages(sbi, wbc, true, FS_NODE_IO);
9dfa1baf 2215 blk_finish_plug(&plug);
50c8cdb3 2216 wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff);
c29fd0c0
CY		 2217
2218 if (wbc->sync_mode == WB_SYNC_ALL)
2219 atomic_dec(&sbi->wb_sync_req[NODE]);
e05df3b1 2220 return 0;
d3baf95d
JK		 2221
2222skip_write:
2223 wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_NODES);
d31c7c3f 2224 trace_f2fs_writepages(mapping->host, wbc, NODE);
d3baf95d 2225 return 0;
e05df3b1
JK		 2226}
2227
cbc975b1
MWO		 2228static bool f2fs_dirty_node_folio(struct address_space *mapping,
2229 struct folio *folio)
e05df3b1 2230{
92f750d8 2231 trace_f2fs_set_page_dirty(folio, NODE);
26c6b887 2232
cbc975b1
MWO		 2233 if (!folio_test_uptodate(folio))
2234 folio_mark_uptodate(folio);
54c55c4e 2235#ifdef CONFIG_F2FS_CHECK_FS
a5f3be6e 2236 if (IS_INODE(folio))
e3f1b76d 2237 f2fs_inode_chksum_set(F2FS_M_SB(mapping), folio);
54c55c4e 2238#endif
9b7eadd9 2239 if (filemap_dirty_folio(mapping, folio)) {
29c87793 2240 inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES);
4ecaf580 2241 folio_set_f2fs_reference(folio);
cbc975b1 2242 return true;
e05df3b1 2243 }
cbc975b1 2244 return false;
e05df3b1
JK		 2245}
2246
0a8165d7 2247/*
e05df3b1
JK		 2248 * Structure of the f2fs node operations
2249 */
2250const struct address_space_operations f2fs_node_aops = {
e05df3b1 2251 .writepages = f2fs_write_node_pages,
cbc975b1 2252 .dirty_folio = f2fs_dirty_node_folio,
91503996 2253 .invalidate_folio = f2fs_invalidate_folio,
c26cd045 2254 .release_folio = f2fs_release_folio,
1d5b9bd6 2255 .migrate_folio = filemap_migrate_folio,
e05df3b1
JK		 2256};
2257
8a7ed66a
JK		 2258static struct free_nid *__lookup_free_nid_list(struct f2fs_nm_info *nm_i,
2259 nid_t n)
e05df3b1 2260{
8a7ed66a 2261 return radix_tree_lookup(&nm_i->free_nid_root, n);
e05df3b1
JK		 2262}
2263
9a4ffdf5 2264static int __insert_free_nid(struct f2fs_sb_info *sbi,
b815bdc7 2265 struct free_nid *i)
e05df3b1 2266{
b8559dc2 2267 struct f2fs_nm_info *nm_i = NM_I(sbi);
a0761f63 2268 int err = radix_tree_insert(&nm_i->free_nid_root, i->nid, i);
5f029c04 2269
a0761f63
FL		 2270 if (err)
2271 return err;
eb0aa4b8 2272
b815bdc7
LS		 2273 nm_i->nid_cnt[FREE_NID]++;
2274 list_add_tail(&i->list, &nm_i->free_nid_list);
eb0aa4b8 2275 return 0;
b8559dc2
CY		 2276}
2277
9a4ffdf5 2278static void __remove_free_nid(struct f2fs_sb_info *sbi,
a0761f63 2279 struct free_nid *i, enum nid_state state)
b8559dc2
CY		 2280{
2281 struct f2fs_nm_info *nm_i = NM_I(sbi);
2282
9a4ffdf5
CY		 2283 f2fs_bug_on(sbi, state != i->state);
2284 nm_i->nid_cnt[state]--;
2285 if (state == FREE_NID)
2286 list_del(&i->list);
a0761f63
FL		 2287 radix_tree_delete(&nm_i->free_nid_root, i->nid);
2288}
2289
2290static void __move_free_nid(struct f2fs_sb_info *sbi, struct free_nid *i,
2291 enum nid_state org_state, enum nid_state dst_state)
b8559dc2
CY		 2292{
2293 struct f2fs_nm_info *nm_i = NM_I(sbi);
2294
a0761f63
FL		 2295 f2fs_bug_on(sbi, org_state != i->state);
2296 i->state = dst_state;
2297 nm_i->nid_cnt[org_state]--;
2298 nm_i->nid_cnt[dst_state]++;
2299
2300 switch (dst_state) {
2301 case PREALLOC_NID:
2302 list_del(&i->list);
2303 break;
2304 case FREE_NID:
2305 list_add_tail(&i->list, &nm_i->free_nid_list);
2306 break;
2307 default:
2308 BUG_ON(1);
2309 }
e05df3b1
JK		 2310}
2311
5921aaa1
L		 2312static void update_free_nid_bitmap(struct f2fs_sb_info *sbi, nid_t nid,
2313 bool set, bool build)
2314{
2315 struct f2fs_nm_info *nm_i = NM_I(sbi);
2316 unsigned int nat_ofs = NAT_BLOCK_OFFSET(nid);
2317 unsigned int nid_ofs = nid - START_NID(nid);
2318
2319 if (!test_bit_le(nat_ofs, nm_i->nat_block_bitmap))
2320 return;
2321
2322 if (set) {
2323 if (test_bit_le(nid_ofs, nm_i->free_nid_bitmap[nat_ofs]))
2324 return;
2325 __set_bit_le(nid_ofs, nm_i->free_nid_bitmap[nat_ofs]);
2326 nm_i->free_nid_count[nat_ofs]++;
2327 } else {
2328 if (!test_bit_le(nid_ofs, nm_i->free_nid_bitmap[nat_ofs]))
2329 return;
2330 __clear_bit_le(nid_ofs, nm_i->free_nid_bitmap[nat_ofs]);
2331 if (!build)
2332 nm_i->free_nid_count[nat_ofs]--;
2333 }
2334}
2335
4ac91242 2336/* return if the nid is recognized as free */
5921aaa1
L		 2337static bool add_free_nid(struct f2fs_sb_info *sbi,
2338 nid_t nid, bool build, bool update)
e05df3b1 2339{
6fb03f3a 2340 struct f2fs_nm_info *nm_i = NM_I(sbi);
30a61ddf 2341 struct free_nid *i, *e;
59bbd474 2342 struct nat_entry *ne;
9b6fc988 2343 int err;
30a61ddf 2344 bool ret = false;
9198aceb
JK		 2345
2346 /* 0 nid should not be used */
cfb271d4 2347 if (unlikely(nid == 0))
4ac91242 2348 return false;
59bbd474 2349
626bcf2b
CY		 2350 if (unlikely(f2fs_check_nid_range(sbi, nid)))
2351 return false;
2352
32410577 2353 i = f2fs_kmem_cache_alloc(free_nid_slab, GFP_NOFS, true, NULL);
e05df3b1 2354 i->nid = nid;
9a4ffdf5 2355 i->state = FREE_NID;
e05df3b1 2356
9b6fc988
CY		 2357 err = radix_tree_preload(GFP_NOFS | __GFP_NOFAIL);
2358 f2fs_bug_on(sbi, err);
2359
2360 err = -EINVAL;
769ec6e5 2361
b8559dc2 2362 spin_lock(&nm_i->nid_list_lock);
30a61ddf
CY		 2363
2364 if (build) {
2365 /*
2366 * Thread A Thread B
2367 * - f2fs_create
2368 * - f2fs_new_inode
4d57b86d 2369 * - f2fs_alloc_nid
9a4ffdf5 2370 * - __insert_nid_to_list(PREALLOC_NID)
30a61ddf 2371 * - f2fs_balance_fs_bg
4d57b86d
CY		 2372 * - f2fs_build_free_nids
2373 * - __f2fs_build_free_nids
30a61ddf
CY		 2374 * - scan_nat_page
2375 * - add_free_nid
2376 * - __lookup_nat_cache
2377 * - f2fs_add_link
4d57b86d 2378 * - f2fs_init_inode_metadata
7c99299c 2379 * - f2fs_new_inode_folio
6b1ad395 2380 * - f2fs_new_node_folio
30a61ddf 2381 * - set_node_addr
4d57b86d 2382 * - f2fs_alloc_nid_done
9a4ffdf5
CY		 2383 * - __remove_nid_from_list(PREALLOC_NID)
2384 * - __insert_nid_to_list(FREE_NID)
30a61ddf 2385 */
0349b7f9 2386 ne = __lookup_nat_cache(nm_i, nid, false);
30a61ddf
CY		 2387 if (ne && (!get_nat_flag(ne, IS_CHECKPOINTED) ||
2388 nat_get_blkaddr(ne) != NULL_ADDR))
2389 goto err_out;
2390
2391 e = __lookup_free_nid_list(nm_i, nid);
2392 if (e) {
9a4ffdf5 2393 if (e->state == FREE_NID)
30a61ddf
CY		 2394 ret = true;
2395 goto err_out;
2396 }
2397 }
2398 ret = true;
b815bdc7 2399 err = __insert_free_nid(sbi, i);
30a61ddf 2400err_out:
5921aaa1
L		 2401 if (update) {
2402 update_free_nid_bitmap(sbi, nid, ret, build);
2403 if (!build)
2404 nm_i->available_nids++;
2405 }
eb0aa4b8
JK		 2406 spin_unlock(&nm_i->nid_list_lock);
2407 radix_tree_preload_end();
5921aaa1 2408
30a61ddf 2409 if (err)
e05df3b1 2410 kmem_cache_free(free_nid_slab, i);
30a61ddf 2411 return ret;
e05df3b1
JK		 2412}
2413
b8559dc2 2414static void remove_free_nid(struct f2fs_sb_info *sbi, nid_t nid)
e05df3b1 2415{
b8559dc2 2416 struct f2fs_nm_info *nm_i = NM_I(sbi);
e05df3b1 2417 struct free_nid *i;
cf0ee0f0
CY		 2418 bool need_free = false;
2419
b8559dc2 2420 spin_lock(&nm_i->nid_list_lock);
8a7ed66a 2421 i = __lookup_free_nid_list(nm_i, nid);
9a4ffdf5 2422 if (i && i->state == FREE_NID) {
a0761f63 2423 __remove_free_nid(sbi, i, FREE_NID);
cf0ee0f0 2424 need_free = true;
e05df3b1 2425 }
b8559dc2 2426 spin_unlock(&nm_i->nid_list_lock);
cf0ee0f0
CY		 2427
2428 if (need_free)
2429 kmem_cache_free(free_nid_slab, i);
e05df3b1
JK		 2430}
2431
e2374015 2432static int scan_nat_page(struct f2fs_sb_info *sbi,
6b5c4a03 2433 struct f2fs_nat_block *nat_blk, nid_t start_nid)
e05df3b1 2434{
6fb03f3a 2435 struct f2fs_nm_info *nm_i = NM_I(sbi);
e05df3b1 2436 block_t blk_addr;
4ac91242 2437 unsigned int nat_ofs = NAT_BLOCK_OFFSET(start_nid);
e05df3b1
JK		 2438 int i;
2439
23380b85 2440 __set_bit_le(nat_ofs, nm_i->nat_block_bitmap);
4ac91242 2441
e05df3b1
JK		 2442 i = start_nid % NAT_ENTRY_PER_BLOCK;
2443
2444 for (; i < NAT_ENTRY_PER_BLOCK; i++, start_nid++) {
cfb271d4 2445 if (unlikely(start_nid >= nm_i->max_nid))
04431c44 2446 break;
23d38844
HL		 2447
2448 blk_addr = le32_to_cpu(nat_blk->entries[i].block_addr);
e2374015
CY		 2449
2450 if (blk_addr == NEW_ADDR)
a5e80e18 2451 return -EFSCORRUPTED;
e2374015 2452
5921aaa1
L		 2453 if (blk_addr == NULL_ADDR) {
2454 add_free_nid(sbi, start_nid, true, true);
2455 } else {
2456 spin_lock(&NM_I(sbi)->nid_list_lock);
2457 update_free_nid_bitmap(sbi, start_nid, false, true);
2458 spin_unlock(&NM_I(sbi)->nid_list_lock);
2459 }
4ac91242 2460 }
e2374015
CY		 2461
2462 return 0;
4ac91242
CY		 2463}
2464
2fbaa25f 2465static void scan_curseg_cache(struct f2fs_sb_info *sbi)
4ac91242 2466{
4ac91242
CY		 2467 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
2468 struct f2fs_journal *journal = curseg->journal;
2fbaa25f
CY		 2469 int i;
2470
2471 down_read(&curseg->journal_rwsem);
2472 for (i = 0; i < nats_in_cursum(journal); i++) {
2473 block_t addr;
2474 nid_t nid;
2475
2476 addr = le32_to_cpu(nat_in_journal(journal, i).block_addr);
2477 nid = le32_to_cpu(nid_in_journal(journal, i));
2478 if (addr == NULL_ADDR)
5921aaa1 2479 add_free_nid(sbi, nid, true, false);
2fbaa25f
CY		 2480 else
2481 remove_free_nid(sbi, nid);
2482 }
2483 up_read(&curseg->journal_rwsem);
2484}
2485
2486static void scan_free_nid_bits(struct f2fs_sb_info *sbi)
2487{
2488 struct f2fs_nm_info *nm_i = NM_I(sbi);
4ac91242 2489 unsigned int i, idx;
97456574 2490 nid_t nid;
4ac91242 2491
e4544b63 2492 f2fs_down_read(&nm_i->nat_tree_lock);
4ac91242
CY		 2493
2494 for (i = 0; i < nm_i->nat_blocks; i++) {
2495 if (!test_bit_le(i, nm_i->nat_block_bitmap))
2496 continue;
586d1492
CY		 2497 if (!nm_i->free_nid_count[i])
2498 continue;
4ac91242 2499 for (idx = 0; idx < NAT_ENTRY_PER_BLOCK; idx++) {
97456574
FL		 2500 idx = find_next_bit_le(nm_i->free_nid_bitmap[i],
2501 NAT_ENTRY_PER_BLOCK, idx);
2502 if (idx >= NAT_ENTRY_PER_BLOCK)
2503 break;
4ac91242
CY		 2504
2505 nid = i * NAT_ENTRY_PER_BLOCK + idx;
5921aaa1 2506 add_free_nid(sbi, nid, true, false);
4ac91242 2507
9a4ffdf5 2508 if (nm_i->nid_cnt[FREE_NID] >= MAX_FREE_NIDS)
4ac91242
CY		 2509 goto out;
2510 }
2511 }
2512out:
2fbaa25f 2513 scan_curseg_cache(sbi);
4ac91242 2514
e4544b63 2515 f2fs_up_read(&nm_i->nat_tree_lock);
e05df3b1
JK		 2516}
2517
e2374015 2518static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi,
4d57b86d 2519 bool sync, bool mount)
e05df3b1 2520{
e05df3b1 2521 struct f2fs_nm_info *nm_i = NM_I(sbi);
e2374015 2522 int i = 0, ret;
55008d84 2523 nid_t nid = nm_i->next_scan_nid;
e05df3b1 2524
e9cdd307
YH		 2525 if (unlikely(nid >= nm_i->max_nid))
2526 nid = 0;
2527
e2cab031
ST		 2528 if (unlikely(nid % NAT_ENTRY_PER_BLOCK))
2529 nid = NAT_BLOCK_OFFSET(nid) * NAT_ENTRY_PER_BLOCK;
2530
55008d84 2531 /* Enough entries */
9a4ffdf5 2532 if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK)
e2374015 2533 return 0;
e05df3b1 2534
4d57b86d 2535 if (!sync && !f2fs_available_free_memory(sbi, FREE_NIDS))
e2374015 2536 return 0;
e05df3b1 2537
4ac91242
CY		 2538 if (!mount) {
2539 /* try to find free nids in free_nid_bitmap */
2540 scan_free_nid_bits(sbi);
2541
74986213 2542 if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK)
e2374015 2543 return 0;
22ad0b6a
JK		 2544 }
2545
55008d84 2546 /* readahead nat pages to be scanned */
4d57b86d 2547 f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), FREE_NID_PAGES,
26879fb1 2548 META_NAT, true);
e05df3b1 2549
e4544b63 2550 f2fs_down_read(&nm_i->nat_tree_lock);
a5131193 2551
e05df3b1 2552 while (1) {
66e83361
YH		 2553 if (!test_bit_le(NAT_BLOCK_OFFSET(nid),
2554 nm_i->nat_block_bitmap)) {
350b8441 2555 struct folio *folio = get_current_nat_folio(sbi, nid);
e05df3b1 2556
350b8441
MWO		 2557 if (IS_ERR(folio)) {
2558 ret = PTR_ERR(folio);
edc55aaf 2559 } else {
6b5c4a03
MWO		 2560 ret = scan_nat_page(sbi, folio_address(folio),
2561 nid);
350b8441 2562 f2fs_folio_put(folio, true);
edc55aaf 2563 }
e2374015
CY		 2564
2565 if (ret) {
e4544b63 2566 f2fs_up_read(&nm_i->nat_tree_lock);
a5e80e18
ZN		 2567
2568 if (ret == -EFSCORRUPTED) {
2569 f2fs_err(sbi, "NAT is corrupt, run fsck to fix it");
2570 set_sbi_flag(sbi, SBI_NEED_FSCK);
2571 f2fs_handle_error(sbi,
2572 ERROR_INCONSISTENT_NAT);
2573 }
2574
edc55aaf 2575 return ret;
e2374015 2576 }
66e83361 2577 }
e05df3b1
JK		 2578
2579 nid += (NAT_ENTRY_PER_BLOCK - (nid % NAT_ENTRY_PER_BLOCK));
cfb271d4 2580 if (unlikely(nid >= nm_i->max_nid))
e05df3b1 2581 nid = 0;
55008d84 2582
a6d494b6 2583 if (++i >= FREE_NID_PAGES)
e05df3b1
JK		 2584 break;
2585 }
2586
55008d84
JK		 2587 /* go to the next free nat pages to find free nids abundantly */
2588 nm_i->next_scan_nid = nid;
e05df3b1
JK		 2589
2590 /* find free nids from current sum_pages */
2fbaa25f 2591 scan_curseg_cache(sbi);
dfc08a12 2592
e4544b63 2593 f2fs_up_read(&nm_i->nat_tree_lock);
2db2388f 2594
4d57b86d 2595 f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nm_i->next_scan_nid),
ea1a29a0 2596 nm_i->ra_nid_pages, META_NAT, false);
e2374015
CY		 2597
2598 return 0;
e05df3b1
JK		 2599}
2600
e2374015 2601int f2fs_build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount)
2411cf5b 2602{
e2374015
CY		 2603 int ret;
2604
2411cf5b 2605 mutex_lock(&NM_I(sbi)->build_lock);
e2374015 2606 ret = __f2fs_build_free_nids(sbi, sync, mount);
2411cf5b 2607 mutex_unlock(&NM_I(sbi)->build_lock);
e2374015
CY		 2608
2609 return ret;
2411cf5b
CY		 2610}
2611
e05df3b1
JK		 2612/*
2613 * If this function returns success, caller can obtain a new nid
2614 * from second parameter of this function.
2615 * The returned nid could be used ino as well as nid when inode is created.
2616 */
4d57b86d 2617bool f2fs_alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid)
e05df3b1
JK		 2618{
2619 struct f2fs_nm_info *nm_i = NM_I(sbi);
2620 struct free_nid *i = NULL;
e05df3b1 2621retry:
c40e15a9 2622 if (time_to_inject(sbi, FAULT_ALLOC_NID))
cb78942b 2623 return false;
7fa750a1 2624
b8559dc2 2625 spin_lock(&nm_i->nid_list_lock);
e05df3b1 2626
04d47e67
CY		 2627 if (unlikely(nm_i->available_nids == 0)) {
2628 spin_unlock(&nm_i->nid_list_lock);
2629 return false;
2630 }
e05df3b1 2631
4d57b86d
CY		 2632 /* We should not use stale free nids created by f2fs_build_free_nids */
2633 if (nm_i->nid_cnt[FREE_NID] && !on_f2fs_build_free_nids(nm_i)) {
9a4ffdf5
CY		 2634 f2fs_bug_on(sbi, list_empty(&nm_i->free_nid_list));
2635 i = list_first_entry(&nm_i->free_nid_list,
b8559dc2 2636 struct free_nid, list);
55008d84 2637 *nid = i->nid;
b8559dc2 2638
a0761f63 2639 __move_free_nid(sbi, i, FREE_NID, PREALLOC_NID);
04d47e67 2640 nm_i->available_nids--;
4ac91242 2641
346fe752 2642 update_free_nid_bitmap(sbi, *nid, false, false);
4ac91242 2643
b8559dc2 2644 spin_unlock(&nm_i->nid_list_lock);
55008d84
JK		 2645 return true;
2646 }
b8559dc2 2647 spin_unlock(&nm_i->nid_list_lock);
55008d84
JK		 2648
2649 /* Let's scan nat pages and its caches to get free nids */
f84262b0
JK		 2650 if (!f2fs_build_free_nids(sbi, true, false))
2651 goto retry;
2652 return false;
e05df3b1
JK		 2653}
2654
0a8165d7 2655/*
4d57b86d 2656 * f2fs_alloc_nid() should be called prior to this function.
e05df3b1 2657 */
4d57b86d 2658void f2fs_alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid)
e05df3b1
JK		 2659{
2660 struct f2fs_nm_info *nm_i = NM_I(sbi);
2661 struct free_nid *i;
2662
b8559dc2 2663 spin_lock(&nm_i->nid_list_lock);
8a7ed66a 2664 i = __lookup_free_nid_list(nm_i, nid);
b8559dc2 2665 f2fs_bug_on(sbi, !i);
a0761f63 2666 __remove_free_nid(sbi, i, PREALLOC_NID);
b8559dc2 2667 spin_unlock(&nm_i->nid_list_lock);
cf0ee0f0
CY		 2668
2669 kmem_cache_free(free_nid_slab, i);
e05df3b1
JK		 2670}
2671
0a8165d7 2672/*
4d57b86d 2673 * f2fs_alloc_nid() should be called prior to this function.
e05df3b1 2674 */
4d57b86d 2675void f2fs_alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid)
e05df3b1 2676{
49952fa1
JK		 2677 struct f2fs_nm_info *nm_i = NM_I(sbi);
2678 struct free_nid *i;
cf0ee0f0 2679 bool need_free = false;
49952fa1 2680
65985d93
JK		 2681 if (!nid)
2682 return;
2683
b8559dc2 2684 spin_lock(&nm_i->nid_list_lock);
8a7ed66a 2685 i = __lookup_free_nid_list(nm_i, nid);
b8559dc2
CY		 2686 f2fs_bug_on(sbi, !i);
2687
4d57b86d 2688 if (!f2fs_available_free_memory(sbi, FREE_NIDS)) {
a0761f63 2689 __remove_free_nid(sbi, i, PREALLOC_NID);
cf0ee0f0 2690 need_free = true;
95630cba 2691 } else {
a0761f63 2692 __move_free_nid(sbi, i, PREALLOC_NID, FREE_NID);
95630cba 2693 }
04d47e67
CY		 2694
2695 nm_i->available_nids++;
2696
346fe752 2697 update_free_nid_bitmap(sbi, nid, true, false);
4ac91242 2698
b8559dc2 2699 spin_unlock(&nm_i->nid_list_lock);
cf0ee0f0
CY		 2700
2701 if (need_free)
2702 kmem_cache_free(free_nid_slab, i);
e05df3b1
JK		 2703}
2704
4d57b86d 2705int f2fs_try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink)
31696580
CY		 2706{
2707 struct f2fs_nm_info *nm_i = NM_I(sbi);
31696580
CY		 2708 int nr = nr_shrink;
2709
9a4ffdf5 2710 if (nm_i->nid_cnt[FREE_NID] <= MAX_FREE_NIDS)
ad4edb83
JK		 2711 return 0;
2712
31696580
CY		 2713 if (!mutex_trylock(&nm_i->build_lock))
2714 return 0;
2715
042be373
CY		 2716 while (nr_shrink && nm_i->nid_cnt[FREE_NID] > MAX_FREE_NIDS) {
2717 struct free_nid *i, *next;
2718 unsigned int batch = SHRINK_NID_BATCH_SIZE;
b8559dc2 2719
042be373
CY		 2720 spin_lock(&nm_i->nid_list_lock);
2721 list_for_each_entry_safe(i, next, &nm_i->free_nid_list, list) {
2722 if (!nr_shrink || !batch ||
2723 nm_i->nid_cnt[FREE_NID] <= MAX_FREE_NIDS)
2724 break;
2725 __remove_free_nid(sbi, i, FREE_NID);
2726 kmem_cache_free(free_nid_slab, i);
2727 nr_shrink--;
2728 batch--;
2729 }
2730 spin_unlock(&nm_i->nid_list_lock);
31696580 2731 }
042be373 2732
31696580
CY		 2733 mutex_unlock(&nm_i->build_lock);
2734
2735 return nr - nr_shrink;
2736}
2737
39b53c0b 2738int f2fs_recover_inline_xattr(struct inode *inode, struct folio *folio)
28cdce04 2739{
28cdce04
CY		 2740 void *src_addr, *dst_addr;
2741 size_t inline_size;
2557c3ea 2742 struct folio *ifolio;
28cdce04
CY		 2743 struct f2fs_inode *ri;
2744
2557c3ea
MWO		 2745 ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino);
2746 if (IS_ERR(ifolio))
2747 return PTR_ERR(ifolio);
28cdce04 2748
9d717807 2749 ri = F2FS_INODE(folio);
1eca05aa 2750 if (ri->i_inline & F2FS_INLINE_XATTR) {
96dd0251
CY		 2751 if (!f2fs_has_inline_xattr(inode)) {
2752 set_inode_flag(inode, FI_INLINE_XATTR);
2753 stat_inc_inline_xattr(inode);
2754 }
1eca05aa 2755 } else {
96dd0251
CY		 2756 if (f2fs_has_inline_xattr(inode)) {
2757 stat_dec_inline_xattr(inode);
2758 clear_inode_flag(inode, FI_INLINE_XATTR);
2759 }
e3b4d43f
JK		 2760 goto update_inode;
2761 }
2762
e0691a05
MWO		 2763 dst_addr = inline_xattr_addr(inode, ifolio);
2764 src_addr = inline_xattr_addr(inode, folio);
28cdce04
CY		 2765 inline_size = inline_xattr_size(inode);
2766
2557c3ea 2767 f2fs_folio_wait_writeback(ifolio, NODE, true, true);
28cdce04 2768 memcpy(dst_addr, src_addr, inline_size);
e3b4d43f 2769update_inode:
f9237928 2770 f2fs_update_inode(inode, ifolio);
2557c3ea 2771 f2fs_folio_put(ifolio, true);
9627a7b3 2772 return 0;
28cdce04
CY		 2773}
2774
447e4fb5 2775int f2fs_recover_xattr_data(struct inode *inode, struct folio *folio)
abb2366c 2776{
4081363f 2777 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
abb2366c 2778 nid_t prev_xnid = F2FS_I(inode)->i_xattr_nid;
87905682
YH		 2779 nid_t new_xnid;
2780 struct dnode_of_data dn;
abb2366c 2781 struct node_info ni;
401da8dd 2782 struct folio *xfolio;
7735730d 2783 int err;
abb2366c 2784
abb2366c
JK		 2785 if (!prev_xnid)
2786 goto recover_xnid;
2787
d260081c 2788 /* 1: invalidate the previous xattr nid */
a9419b63 2789 err = f2fs_get_node_info(sbi, prev_xnid, &ni, false);
7735730d
CY		 2790 if (err)
2791 return err;
2792
e53c568f 2793 f2fs_invalidate_blocks(sbi, ni.blk_addr, 1);
000519f2 2794 dec_valid_node_count(sbi, inode, false);
479f40c4 2795 set_node_addr(sbi, &ni, NULL_ADDR, false);
abb2366c
JK		 2796
2797recover_xnid:
d260081c 2798 /* 2: update xattr nid in inode */
4d57b86d 2799 if (!f2fs_alloc_nid(sbi, &new_xnid))
87905682
YH		 2800 return -ENOSPC;
2801
2802 set_new_dnode(&dn, inode, NULL, NULL, new_xnid);
401da8dd
MWO		 2803 xfolio = f2fs_new_node_folio(&dn, XATTR_NODE_OFFSET);
2804 if (IS_ERR(xfolio)) {
4d57b86d 2805 f2fs_alloc_nid_failed(sbi, new_xnid);
401da8dd 2806 return PTR_ERR(xfolio);
87905682
YH		 2807 }
2808
4d57b86d
CY		 2809 f2fs_alloc_nid_done(sbi, new_xnid);
2810 f2fs_update_inode_page(inode);
d260081c
CY		 2811
2812 /* 3: update and set xattr node page dirty */
447e4fb5 2813 if (folio) {
8591db2a 2814 memcpy(F2FS_NODE(xfolio), F2FS_NODE(folio),
50a472bb 2815 VALID_XATTR_BLOCK_SIZE);
401da8dd 2816 folio_mark_dirty(xfolio);
86d7d57a 2817 }
401da8dd 2818 f2fs_folio_put(xfolio, true);
abb2366c 2819
d260081c 2820 return 0;
abb2366c
JK		 2821}
2822
b77dc031 2823int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct folio *folio)
e05df3b1 2824{
58bfaf44 2825 struct f2fs_inode *src, *dst;
28fde0d7 2826 nid_t ino = ino_of_node(folio);
e05df3b1 2827 struct node_info old_ni, new_ni;
8b78cea8 2828 struct folio *ifolio;
7735730d 2829 int err;
e05df3b1 2830
a9419b63 2831 err = f2fs_get_node_info(sbi, ino, &old_ni, false);
7735730d
CY		 2832 if (err)
2833 return err;
e8271fa3
JK		 2834
2835 if (unlikely(old_ni.blk_addr != NULL_ADDR))
2836 return -EINVAL;
e8ea9b3d 2837retry:
8b78cea8
MWO		 2838 ifolio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), ino, false);
2839 if (IS_ERR(ifolio)) {
4034247a 2840 memalloc_retry_wait(GFP_NOFS);
e8ea9b3d
JK		 2841 goto retry;
2842 }
e05df3b1 2843
e1c42045 2844 /* Should not use this inode from free nid list */
b8559dc2 2845 remove_free_nid(sbi, ino);
e05df3b1 2846
8b78cea8
MWO		 2847 if (!folio_test_uptodate(ifolio))
2848 folio_mark_uptodate(ifolio);
06bf1182 2849 fill_node_footer(ifolio, ino, ino, 0, true);
53987453 2850 set_cold_node(ifolio, false);
e05df3b1 2851
9d717807
MWO		 2852 src = F2FS_INODE(folio);
2853 dst = F2FS_INODE(ifolio);
e05df3b1 2854
36218b81 2855 memcpy(dst, src, offsetof(struct f2fs_inode, i_ext));
58bfaf44
JK		 2856 dst->i_size = 0;
2857 dst->i_blocks = cpu_to_le64(1);
2858 dst->i_links = cpu_to_le32(1);
2859 dst->i_xattr_nid = 0;
7a2af766 2860 dst->i_inline = src->i_inline & (F2FS_INLINE_XATTR | F2FS_EXTRA_ATTR);
5c57132e 2861 if (dst->i_inline & F2FS_EXTRA_ATTR) {
7a2af766 2862 dst->i_extra_isize = src->i_extra_isize;
6afc662e 2863
7beb01f7 2864 if (f2fs_sb_has_flexible_inline_xattr(sbi) &&
6afc662e
CY		 2865 F2FS_FITS_IN_INODE(src, le16_to_cpu(src->i_extra_isize),
2866 i_inline_xattr_size))
2867 dst->i_inline_xattr_size = src->i_inline_xattr_size;
2868
7beb01f7 2869 if (f2fs_sb_has_project_quota(sbi) &&
5c57132e
CY		 2870 F2FS_FITS_IN_INODE(src, le16_to_cpu(src->i_extra_isize),
2871 i_projid))
2872 dst->i_projid = src->i_projid;
5cd1f387 2873
7beb01f7 2874 if (f2fs_sb_has_inode_crtime(sbi) &&
5cd1f387
CY		 2875 F2FS_FITS_IN_INODE(src, le16_to_cpu(src->i_extra_isize),
2876 i_crtime_nsec)) {
2877 dst->i_crtime = src->i_crtime;
2878 dst->i_crtime_nsec = src->i_crtime_nsec;
2879 }
5c57132e 2880 }
e05df3b1
JK		 2881
2882 new_ni = old_ni;
2883 new_ni.ino = ino;
2884
0abd675e 2885 if (unlikely(inc_valid_node_count(sbi, NULL, true)))
65e5cd0a 2886 WARN_ON(1);
479f40c4 2887 set_node_addr(sbi, &new_ni, NEW_ADDR, false);
e05df3b1 2888 inc_valid_inode_count(sbi);
8b78cea8
MWO		 2889 folio_mark_dirty(ifolio);
2890 f2fs_folio_put(ifolio, true);
e05df3b1
JK		 2891 return 0;
2892}
2893
7735730d 2894int f2fs_restore_node_summary(struct f2fs_sb_info *sbi,
e05df3b1
JK		 2895 unsigned int segno, struct f2fs_summary_block *sum)
2896{
2897 struct f2fs_node *rn;
2898 struct f2fs_summary *sum_entry;
e05df3b1 2899 block_t addr;
9ecf4b80 2900 int i, idx, last_offset, nrpages;
e05df3b1
JK		 2901
2902 /* scan the node segment */
a60108f7 2903 last_offset = BLKS_PER_SEG(sbi);
e05df3b1
JK		 2904 addr = START_BLOCK(sbi, segno);
2905 sum_entry = &sum->entries[0];
2906
9ecf4b80 2907 for (i = 0; i < last_offset; i += nrpages, addr += nrpages) {
5f7136db 2908 nrpages = bio_max_segs(last_offset - i);
393ff91f 2909
e1c42045 2910 /* readahead node pages */
4d57b86d 2911 f2fs_ra_meta_pages(sbi, addr, nrpages, META_POR, true);
e05df3b1 2912
9ecf4b80 2913 for (idx = addr; idx < addr + nrpages; idx++) {
937d6a4d 2914 struct folio *folio = f2fs_get_tmp_folio(sbi, idx);
9af0ff1c 2915
937d6a4d
MWO		 2916 if (IS_ERR(folio))
2917 return PTR_ERR(folio);
7735730d 2918
8591db2a 2919 rn = F2FS_NODE(folio);
9ecf4b80
CY		 2920 sum_entry->nid = rn->footer.nid;
2921 sum_entry->version = 0;
2922 sum_entry->ofs_in_node = 0;
2923 sum_entry++;
937d6a4d 2924 f2fs_folio_put(folio, true);
9af0ff1c 2925 }
bac4eef6 2926
9ecf4b80 2927 invalidate_mapping_pages(META_MAPPING(sbi), addr,
bac4eef6 2928 addr + nrpages);
e05df3b1 2929 }
7735730d 2930 return 0;
e05df3b1
JK		 2931}
2932
aec71382 2933static void remove_nats_in_journal(struct f2fs_sb_info *sbi)
e05df3b1
JK		 2934{
2935 struct f2fs_nm_info *nm_i = NM_I(sbi);
2936 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
b7ad7512 2937 struct f2fs_journal *journal = curseg->journal;
e05df3b1 2938 int i;
40aa9e12 2939 bool init_dirty;
e05df3b1 2940
b7ad7512 2941 down_write(&curseg->journal_rwsem);
dfc08a12 2942 for (i = 0; i < nats_in_cursum(journal); i++) {
e05df3b1
JK		 2943 struct nat_entry *ne;
2944 struct f2fs_nat_entry raw_ne;
dfc08a12 2945 nid_t nid = le32_to_cpu(nid_in_journal(journal, i));
e05df3b1 2946
b862676e
CY		 2947 if (f2fs_check_nid_range(sbi, nid))
2948 continue;
2949
40aa9e12 2950 init_dirty = false;
2951
dfc08a12 2952 raw_ne = nat_in_journal(journal, i);
9be32d72 2953
0349b7f9 2954 ne = __lookup_nat_cache(nm_i, nid, true);
e05df3b1 2955 if (!ne) {
40aa9e12 2956 init_dirty = true;
32410577 2957 ne = __alloc_nat_entry(sbi, nid, true);
40aa9e12 2958 __init_nat_entry(nm_i, ne, &raw_ne, true, true);
e05df3b1 2959 }
04d47e67
CY		 2960
2961 /*
2962 * if a free nat in journal has not been used after last
2963 * checkpoint, we should remove it from available nids,
2964 * since later we will add it again.
2965 */
2966 if (!get_nat_flag(ne, IS_DIRTY) &&
2967 le32_to_cpu(raw_ne.block_addr) == NULL_ADDR) {
2968 spin_lock(&nm_i->nid_list_lock);
2969 nm_i->available_nids--;
2970 spin_unlock(&nm_i->nid_list_lock);
2971 }
2972
40aa9e12 2973 __set_nat_cache_dirty(nm_i, ne, init_dirty);
e05df3b1 2974 }
dfc08a12 2975 update_nats_in_cursum(journal, -i);
b7ad7512 2976 up_write(&curseg->journal_rwsem);
e05df3b1
JK		 2977}
2978
309cc2b6
JK		 2979static void __adjust_nat_entry_set(struct nat_entry_set *nes,
2980 struct list_head *head, int max)
e05df3b1 2981{
309cc2b6 2982 struct nat_entry_set *cur;
e05df3b1 2983
309cc2b6
JK		 2984 if (nes->entry_cnt >= max)
2985 goto add_out;
e05df3b1 2986
309cc2b6
JK		 2987 list_for_each_entry(cur, head, set_list) {
2988 if (cur->entry_cnt >= nes->entry_cnt) {
2989 list_add(&nes->set_list, cur->set_list.prev);
2990 return;
2991 }
aec71382 2992 }
309cc2b6
JK		 2993add_out:
2994 list_add_tail(&nes->set_list, head);
2995}
e05df3b1 2996
19426c49 2997static void __update_nat_bits(struct f2fs_sb_info *sbi, nid_t start_nid,
c07de755 2998 const struct f2fs_nat_block *nat_blk)
22ad0b6a
JK		 2999{
3000 struct f2fs_nm_info *nm_i = NM_I(sbi);
3001 unsigned int nat_index = start_nid / NAT_ENTRY_PER_BLOCK;
22ad0b6a 3002 int valid = 0;
37a0ab2a 3003 int i = 0;
22ad0b6a 3004
19426c49 3005 if (!enabled_nat_bits(sbi, NULL))
22ad0b6a
JK		 3006 return;
3007
37a0ab2a
FL		 3008 if (nat_index == 0) {
3009 valid = 1;
3010 i = 1;
3011 }
3012 for (; i < NAT_ENTRY_PER_BLOCK; i++) {
36af5f40 3013 if (le32_to_cpu(nat_blk->entries[i].block_addr) != NULL_ADDR)
22ad0b6a
JK		 3014 valid++;
3015 }
19426c49
CY		 3016 if (valid == 0) {
3017 __set_bit_le(nat_index, nm_i->empty_nat_bits);
3018 __clear_bit_le(nat_index, nm_i->full_nat_bits);
3019 return;
22ad0b6a
JK		 3020 }
3021
19426c49
CY		 3022 __clear_bit_le(nat_index, nm_i->empty_nat_bits);
3023 if (valid == NAT_ENTRY_PER_BLOCK)
3024 __set_bit_le(nat_index, nm_i->full_nat_bits);
3025 else
3026 __clear_bit_le(nat_index, nm_i->full_nat_bits);
22ad0b6a
JK		 3027}
3028
edc55aaf 3029static int __flush_nat_entry_set(struct f2fs_sb_info *sbi,
22ad0b6a 3030 struct nat_entry_set *set, struct cp_control *cpc)
309cc2b6
JK		 3031{
3032 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
b7ad7512 3033 struct f2fs_journal *journal = curseg->journal;
309cc2b6
JK		 3034 nid_t start_nid = set->set * NAT_ENTRY_PER_BLOCK;
3035 bool to_journal = true;
3036 struct f2fs_nat_block *nat_blk;
3037 struct nat_entry *ne, *cur;
3a19caf1 3038 struct folio *folio = NULL;
e05df3b1 3039
aec71382
CY		 3040 /*
3041 * there are two steps to flush nat entries:
3042 * #1, flush nat entries to journal in current hot data summary block.
3043 * #2, flush nat entries to nat page.
3044 */
19426c49 3045 if (enabled_nat_bits(sbi, cpc) ||
22ad0b6a 3046 !__has_cursum_space(journal, set->entry_cnt, NAT_JOURNAL))
309cc2b6
JK		 3047 to_journal = false;
3048
3049 if (to_journal) {
b7ad7512 3050 down_write(&curseg->journal_rwsem);
309cc2b6 3051 } else {
3a19caf1
MWO		 3052 folio = get_next_nat_folio(sbi, start_nid);
3053 if (IS_ERR(folio))
3054 return PTR_ERR(folio);
edc55aaf 3055
3a19caf1 3056 nat_blk = folio_address(folio);
309cc2b6
JK		 3057 f2fs_bug_on(sbi, !nat_blk);
3058 }
aec71382 3059
309cc2b6
JK		 3060 /* flush dirty nats in nat entry set */
3061 list_for_each_entry_safe(ne, cur, &set->entry_list, list) {
3062 struct f2fs_nat_entry *raw_ne;
3063 nid_t nid = nat_get_nid(ne);
3064 int offset;
3065
febeca6d 3066 f2fs_bug_on(sbi, nat_get_blkaddr(ne) == NEW_ADDR);
aec71382
CY		 3067
3068 if (to_journal) {
4d57b86d 3069 offset = f2fs_lookup_journal_in_cursum(journal,
309cc2b6
JK		 3070 NAT_JOURNAL, nid, 1);
3071 f2fs_bug_on(sbi, offset < 0);
dfc08a12
CY		 3072 raw_ne = &nat_in_journal(journal, offset);
3073 nid_in_journal(journal, offset) = cpu_to_le32(nid);
aec71382 3074 } else {
309cc2b6 3075 raw_ne = &nat_blk->entries[nid - start_nid];
e05df3b1 3076 }
309cc2b6 3077 raw_nat_from_node_info(raw_ne, &ne->ni);
309cc2b6 3078 nat_reset_flag(ne);
0b28b71e 3079 __clear_nat_cache_dirty(NM_I(sbi), set, ne);
04d47e67 3080 if (nat_get_blkaddr(ne) == NULL_ADDR) {
5921aaa1 3081 add_free_nid(sbi, nid, false, true);
4ac91242
CY		 3082 } else {
3083 spin_lock(&NM_I(sbi)->nid_list_lock);
346fe752 3084 update_free_nid_bitmap(sbi, nid, false, false);
04d47e67
CY		 3085 spin_unlock(&NM_I(sbi)->nid_list_lock);
3086 }
309cc2b6 3087 }
e05df3b1 3088
22ad0b6a 3089 if (to_journal) {
b7ad7512 3090 up_write(&curseg->journal_rwsem);
22ad0b6a 3091 } else {
c07de755 3092 __update_nat_bits(sbi, start_nid, nat_blk);
3a19caf1 3093 f2fs_folio_put(folio, true);
22ad0b6a 3094 }
aec71382 3095
59c9081b
YH		 3096 /* Allow dirty nats by node block allocation in write_begin */
3097 if (!set->entry_cnt) {
3098 radix_tree_delete(&NM_I(sbi)->nat_set_root, set->set);
3099 kmem_cache_free(nat_entry_set_slab, set);
3100 }
edc55aaf 3101 return 0;
309cc2b6 3102}
aec71382 3103
309cc2b6
JK		 3104/*
3105 * This function is called during the checkpointing process.
3106 */
edc55aaf 3107int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
309cc2b6
JK		 3108{
3109 struct f2fs_nm_info *nm_i = NM_I(sbi);
3110 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
b7ad7512 3111 struct f2fs_journal *journal = curseg->journal;
c31e4961 3112 struct nat_entry_set *setvec[NAT_VEC_SIZE];
309cc2b6
JK		 3113 struct nat_entry_set *set, *tmp;
3114 unsigned int found;
3115 nid_t set_idx = 0;
3116 LIST_HEAD(sets);
edc55aaf 3117 int err = 0;
309cc2b6 3118
a95ba66a
JK		 3119 /*
3120 * during unmount, let's flush nat_bits before checking
3121 * nat_cnt[DIRTY_NAT].
3122 */
19426c49 3123 if (enabled_nat_bits(sbi, cpc)) {
e4544b63 3124 f2fs_down_write(&nm_i->nat_tree_lock);
7f2ecdd8 3125 remove_nats_in_journal(sbi);
e4544b63 3126 f2fs_up_write(&nm_i->nat_tree_lock);
7f2ecdd8
JK		 3127 }
3128
a95ba66a 3129 if (!nm_i->nat_cnt[DIRTY_NAT])
edc55aaf 3130 return 0;
a5131193 3131
e4544b63 3132 f2fs_down_write(&nm_i->nat_tree_lock);
a5131193 3133
309cc2b6
JK		 3134 /*
3135 * if there are no enough space in journal to store dirty nat
3136 * entries, remove all entries from journal and merge them
3137 * into nat entry set.
3138 */
19426c49 3139 if (enabled_nat_bits(sbi, cpc) ||
a95ba66a
JK		 3140 !__has_cursum_space(journal,
3141 nm_i->nat_cnt[DIRTY_NAT], NAT_JOURNAL))
309cc2b6
JK		 3142 remove_nats_in_journal(sbi);
3143
309cc2b6 3144 while ((found = __gang_lookup_nat_set(nm_i,
c31e4961 3145 set_idx, NAT_VEC_SIZE, setvec))) {
309cc2b6 3146 unsigned idx;
5f029c04 3147
309cc2b6
JK		 3148 set_idx = setvec[found - 1]->set + 1;
3149 for (idx = 0; idx < found; idx++)
3150 __adjust_nat_entry_set(setvec[idx], &sets,
dfc08a12 3151 MAX_NAT_JENTRIES(journal));
e05df3b1 3152 }
aec71382 3153
309cc2b6 3154 /* flush dirty nats in nat entry set */
edc55aaf
JK		 3155 list_for_each_entry_safe(set, tmp, &sets, set_list) {
3156 err = __flush_nat_entry_set(sbi, set, cpc);
3157 if (err)
3158 break;
3159 }
309cc2b6 3160
e4544b63 3161 f2fs_up_write(&nm_i->nat_tree_lock);
59c9081b 3162 /* Allow dirty nats by node block allocation in write_begin */
edc55aaf
JK		 3163
3164 return err;
e05df3b1
JK		 3165}
3166
22ad0b6a
JK		 3167static int __get_nat_bitmaps(struct f2fs_sb_info *sbi)
3168{
3169 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3170 struct f2fs_nm_info *nm_i = NM_I(sbi);
3171 unsigned int nat_bits_bytes = nm_i->nat_blocks / BITS_PER_BYTE;
3172 unsigned int i;
3173 __u64 cp_ver = cur_cp_version(ckpt);
22ad0b6a
JK		 3174 block_t nat_bits_addr;
3175
19426c49
CY		 3176 if (!enabled_nat_bits(sbi, NULL))
3177 return 0;
3178
df033caf 3179 nm_i->nat_bits_blocks = F2FS_BLK_ALIGN((nat_bits_bytes << 1) + 8);
0b6d4ca0 3180 nm_i->nat_bits = f2fs_kvzalloc(sbi,
8fb9f319 3181 F2FS_BLK_TO_BYTES(nm_i->nat_bits_blocks), GFP_KERNEL);
22ad0b6a
JK		 3182 if (!nm_i->nat_bits)
3183 return -ENOMEM;
3184
a60108f7 3185 nat_bits_addr = __start_cp_addr(sbi) + BLKS_PER_SEG(sbi) -
22ad0b6a
JK		 3186 nm_i->nat_bits_blocks;
3187 for (i = 0; i < nm_i->nat_bits_blocks; i++) {
8a6fb4cc 3188 struct folio *folio;
7735730d 3189
8a6fb4cc
MWO		 3190 folio = f2fs_get_meta_folio(sbi, nat_bits_addr++);
3191 if (IS_ERR(folio))
3192 return PTR_ERR(folio);
22ad0b6a 3193
8fb9f319 3194 memcpy(nm_i->nat_bits + F2FS_BLK_TO_BYTES(i),
8a6fb4cc
MWO		 3195 folio_address(folio), F2FS_BLKSIZE);
3196 f2fs_folio_put(folio, true);
22ad0b6a
JK		 3197 }
3198
ced2c7ea 3199 cp_ver |= (cur_cp_crc(ckpt) << 32);
22ad0b6a 3200 if (cpu_to_le64(cp_ver) != *(__le64 *)nm_i->nat_bits) {
19426c49 3201 disable_nat_bits(sbi, true);
22ad0b6a
JK		 3202 return 0;
3203 }
3204
19426c49
CY		 3205 nm_i->full_nat_bits = nm_i->nat_bits + 8;
3206 nm_i->empty_nat_bits = nm_i->full_nat_bits + nat_bits_bytes;
3207
dcbb4c10 3208 f2fs_notice(sbi, "Found nat_bits in checkpoint");
22ad0b6a
JK		 3209 return 0;
3210}
3211
bd80a4b9 3212static inline void load_free_nid_bitmap(struct f2fs_sb_info *sbi)
7041d5d2
CY		 3213{
3214 struct f2fs_nm_info *nm_i = NM_I(sbi);
3215 unsigned int i = 0;
3216 nid_t nid, last_nid;
3217
19426c49 3218 if (!enabled_nat_bits(sbi, NULL))
7041d5d2
CY		 3219 return;
3220
3221 for (i = 0; i < nm_i->nat_blocks; i++) {
3222 i = find_next_bit_le(nm_i->empty_nat_bits, nm_i->nat_blocks, i);
3223 if (i >= nm_i->nat_blocks)
3224 break;
3225
3226 __set_bit_le(i, nm_i->nat_block_bitmap);
3227
3228 nid = i * NAT_ENTRY_PER_BLOCK;
f6986ede 3229 last_nid = nid + NAT_ENTRY_PER_BLOCK;
7041d5d2 3230
346fe752 3231 spin_lock(&NM_I(sbi)->nid_list_lock);
7041d5d2 3232 for (; nid < last_nid; nid++)
346fe752
CY		 3233 update_free_nid_bitmap(sbi, nid, true, true);
3234 spin_unlock(&NM_I(sbi)->nid_list_lock);
7041d5d2
CY		 3235 }
3236
3237 for (i = 0; i < nm_i->nat_blocks; i++) {
3238 i = find_next_bit_le(nm_i->full_nat_bits, nm_i->nat_blocks, i);
3239 if (i >= nm_i->nat_blocks)
3240 break;
3241
3242 __set_bit_le(i, nm_i->nat_block_bitmap);
3243 }
3244}
3245
e05df3b1
JK		 3246static int init_node_manager(struct f2fs_sb_info *sbi)
3247{
3248 struct f2fs_super_block *sb_raw = F2FS_RAW_SUPER(sbi);
3249 struct f2fs_nm_info *nm_i = NM_I(sbi);
3250 unsigned char *version_bitmap;
22ad0b6a
JK		 3251 unsigned int nat_segs;
3252 int err;
e05df3b1
JK		 3253
3254 nm_i->nat_blkaddr = le32_to_cpu(sb_raw->nat_blkaddr);
3255
3256 /* segment_count_nat includes pair segment so divide to 2. */
3257 nat_segs = le32_to_cpu(sb_raw->segment_count_nat) >> 1;
22ad0b6a
JK		 3258 nm_i->nat_blocks = nat_segs << le32_to_cpu(sb_raw->log_blocks_per_seg);
3259 nm_i->max_nid = NAT_ENTRY_PER_BLOCK * nm_i->nat_blocks;
7ee0eeab 3260
b63da15e 3261 /* not used nids: 0, node, meta, (and root counted as valid node) */
04d47e67 3262 nm_i->available_nids = nm_i->max_nid - sbi->total_valid_node_count -
27cae0bc 3263 F2FS_RESERVED_NODE_NUM;
9a4ffdf5
CY		 3264 nm_i->nid_cnt[FREE_NID] = 0;
3265 nm_i->nid_cnt[PREALLOC_NID] = 0;
cdfc41c1 3266 nm_i->ram_thresh = DEF_RAM_THRESHOLD;
ea1a29a0 3267 nm_i->ra_nid_pages = DEF_RA_NID_PAGES;
2304cb0c 3268 nm_i->dirty_nats_ratio = DEF_DIRTY_NAT_RATIO_THRESHOLD;
47c8ebcc 3269 nm_i->max_rf_node_blocks = DEF_RF_NODE_BLOCKS;
e05df3b1 3270
8a7ed66a 3271 INIT_RADIX_TREE(&nm_i->free_nid_root, GFP_ATOMIC);
9a4ffdf5 3272 INIT_LIST_HEAD(&nm_i->free_nid_list);
769ec6e5
JK		 3273 INIT_RADIX_TREE(&nm_i->nat_root, GFP_NOIO);
3274 INIT_RADIX_TREE(&nm_i->nat_set_root, GFP_NOIO);
e05df3b1 3275 INIT_LIST_HEAD(&nm_i->nat_entries);
22969158 3276 spin_lock_init(&nm_i->nat_list_lock);
e05df3b1
JK		 3277
3278 mutex_init(&nm_i->build_lock);
b8559dc2 3279 spin_lock_init(&nm_i->nid_list_lock);
e4544b63 3280 init_f2fs_rwsem(&nm_i->nat_tree_lock);
e05df3b1 3281
e05df3b1 3282 nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid);
79b5793b 3283 nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP);
e05df3b1 3284 version_bitmap = __bitmap_ptr(sbi, NAT_BITMAP);
79b5793b
AG		 3285 nm_i->nat_bitmap = kmemdup(version_bitmap, nm_i->bitmap_size,
3286 GFP_KERNEL);
3287 if (!nm_i->nat_bitmap)
3288 return -ENOMEM;
599a09b2 3289
c2ecba02
CY		 3290 if (!test_opt(sbi, NAT_BITS))
3291 disable_nat_bits(sbi, true);
3292
22ad0b6a
JK		 3293 err = __get_nat_bitmaps(sbi);
3294 if (err)
3295 return err;
3296
599a09b2
CY		 3297#ifdef CONFIG_F2FS_CHECK_FS
3298 nm_i->nat_bitmap_mir = kmemdup(version_bitmap, nm_i->bitmap_size,
3299 GFP_KERNEL);
3300 if (!nm_i->nat_bitmap_mir)
3301 return -ENOMEM;
3302#endif
3303
e05df3b1
JK		 3304 return 0;
3305}
3306
9f7e4a2c 3307static int init_free_nid_cache(struct f2fs_sb_info *sbi)
4ac91242
CY		 3308{
3309 struct f2fs_nm_info *nm_i = NM_I(sbi);
bb1105e4 3310 int i;
4ac91242 3311
026f0507 3312 nm_i->free_nid_bitmap =
0b6d4ca0
EB		 3313 f2fs_kvzalloc(sbi, array_size(sizeof(unsigned char *),
3314 nm_i->nat_blocks),
3315 GFP_KERNEL);
4ac91242
CY		 3316 if (!nm_i->free_nid_bitmap)
3317 return -ENOMEM;
3318
bb1105e4
JK		 3319 for (i = 0; i < nm_i->nat_blocks; i++) {
3320 nm_i->free_nid_bitmap[i] = f2fs_kvzalloc(sbi,
e15d54d5 3321 f2fs_bitmap_size(NAT_ENTRY_PER_BLOCK), GFP_KERNEL);
68c43a23 3322 if (!nm_i->free_nid_bitmap[i])
bb1105e4
JK		 3323 return -ENOMEM;
3324 }
3325
628b3d14 3326 nm_i->nat_block_bitmap = f2fs_kvzalloc(sbi, nm_i->nat_blocks / 8,
4ac91242
CY		 3327 GFP_KERNEL);
3328 if (!nm_i->nat_block_bitmap)
3329 return -ENOMEM;
586d1492 3330
9d2a789c
KC		 3331 nm_i->free_nid_count =
3332 f2fs_kvzalloc(sbi, array_size(sizeof(unsigned short),
3333 nm_i->nat_blocks),
3334 GFP_KERNEL);
586d1492
CY		 3335 if (!nm_i->free_nid_count)
3336 return -ENOMEM;
4ac91242
CY		 3337 return 0;
3338}
3339
4d57b86d 3340int f2fs_build_node_manager(struct f2fs_sb_info *sbi)
e05df3b1
JK		 3341{
3342 int err;
3343
acbf054d
CY		 3344 sbi->nm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_nm_info),
3345 GFP_KERNEL);
e05df3b1
JK		 3346 if (!sbi->nm_info)
3347 return -ENOMEM;
3348
3349 err = init_node_manager(sbi);
3350 if (err)
3351 return err;
3352
4ac91242
CY		 3353 err = init_free_nid_cache(sbi);
3354 if (err)
3355 return err;
3356
7041d5d2
CY		 3357 /* load free nid status from nat_bits table */
3358 load_free_nid_bitmap(sbi);
3359
e2374015 3360 return f2fs_build_free_nids(sbi, true, true);
e05df3b1
JK		 3361}
3362
4d57b86d 3363void f2fs_destroy_node_manager(struct f2fs_sb_info *sbi)
e05df3b1
JK		 3364{
3365 struct f2fs_nm_info *nm_i = NM_I(sbi);
3366 struct free_nid *i, *next_i;
c31e4961
CY		 3367 void *vec[NAT_VEC_SIZE];
3368 struct nat_entry **natvec = (struct nat_entry **)vec;
3369 struct nat_entry_set **setvec = (struct nat_entry_set **)vec;
e05df3b1
JK		 3370 nid_t nid = 0;
3371 unsigned int found;
3372
3373 if (!nm_i)
3374 return;
3375
3376 /* destroy free nid list */
b8559dc2 3377 spin_lock(&nm_i->nid_list_lock);
9a4ffdf5 3378 list_for_each_entry_safe(i, next_i, &nm_i->free_nid_list, list) {
a0761f63 3379 __remove_free_nid(sbi, i, FREE_NID);
b8559dc2 3380 spin_unlock(&nm_i->nid_list_lock);
cf0ee0f0 3381 kmem_cache_free(free_nid_slab, i);
b8559dc2 3382 spin_lock(&nm_i->nid_list_lock);
e05df3b1 3383 }
9a4ffdf5
CY		 3384 f2fs_bug_on(sbi, nm_i->nid_cnt[FREE_NID]);
3385 f2fs_bug_on(sbi, nm_i->nid_cnt[PREALLOC_NID]);
3386 f2fs_bug_on(sbi, !list_empty(&nm_i->free_nid_list));
b8559dc2 3387 spin_unlock(&nm_i->nid_list_lock);
e05df3b1
JK		 3388
3389 /* destroy nat cache */
e4544b63 3390 f2fs_down_write(&nm_i->nat_tree_lock);
e05df3b1 3391 while ((found = __gang_lookup_nat_cache(nm_i,
c31e4961 3392 nid, NAT_VEC_SIZE, natvec))) {
e05df3b1 3393 unsigned idx;
7aed0d45 3394
b6ce391e 3395 nid = nat_get_nid(natvec[found - 1]) + 1;
22969158
CY		 3396 for (idx = 0; idx < found; idx++) {
3397 spin_lock(&nm_i->nat_list_lock);
3398 list_del(&natvec[idx]->list);
3399 spin_unlock(&nm_i->nat_list_lock);
3400
b6ce391e 3401 __del_from_nat_cache(nm_i, natvec[idx]);
22969158 3402 }
e05df3b1 3403 }
a95ba66a 3404 f2fs_bug_on(sbi, nm_i->nat_cnt[TOTAL_NAT]);
7aed0d45
JK		 3405
3406 /* destroy nat set cache */
3407 nid = 0;
c31e4961 3408 memset(vec, 0, sizeof(void *) * NAT_VEC_SIZE);
7aed0d45 3409 while ((found = __gang_lookup_nat_set(nm_i,
c31e4961 3410 nid, NAT_VEC_SIZE, setvec))) {
7aed0d45
JK		 3411 unsigned idx;
3412
3413 nid = setvec[found - 1]->set + 1;
3414 for (idx = 0; idx < found; idx++) {
3415 /* entry_cnt is not zero, when cp_error was occurred */
3416 f2fs_bug_on(sbi, !list_empty(&setvec[idx]->entry_list));
3417 radix_tree_delete(&nm_i->nat_set_root, setvec[idx]->set);
3418 kmem_cache_free(nat_entry_set_slab, setvec[idx]);
3419 }
3420 }
e4544b63 3421 f2fs_up_write(&nm_i->nat_tree_lock);
e05df3b1 3422
4ac91242 3423 kvfree(nm_i->nat_block_bitmap);
bb1105e4
JK		 3424 if (nm_i->free_nid_bitmap) {
3425 int i;
3426
3427 for (i = 0; i < nm_i->nat_blocks; i++)
3428 kvfree(nm_i->free_nid_bitmap[i]);
5222595d 3429 kvfree(nm_i->free_nid_bitmap);
bb1105e4 3430 }
586d1492 3431 kvfree(nm_i->free_nid_count);
4ac91242 3432
e9705c61 3433 kfree(nm_i->nat_bitmap);
5222595d 3434 kvfree(nm_i->nat_bits);
599a09b2 3435#ifdef CONFIG_F2FS_CHECK_FS
e9705c61 3436 kfree(nm_i->nat_bitmap_mir);
599a09b2 3437#endif
e05df3b1 3438 sbi->nm_info = NULL;
c8eb7024 3439 kfree(nm_i);
e05df3b1
JK		 3440}
3441
4d57b86d 3442int __init f2fs_create_node_manager_caches(void)
e05df3b1 3443{
98510003 3444 nat_entry_slab = f2fs_kmem_cache_create("f2fs_nat_entry",
e8512d2e 3445 sizeof(struct nat_entry));
e05df3b1 3446 if (!nat_entry_slab)
aec71382 3447 goto fail;
e05df3b1 3448
98510003 3449 free_nid_slab = f2fs_kmem_cache_create("f2fs_free_nid",
e8512d2e 3450 sizeof(struct free_nid));
aec71382 3451 if (!free_nid_slab)
ce3e6d25 3452 goto destroy_nat_entry;
aec71382 3453
98510003 3454 nat_entry_set_slab = f2fs_kmem_cache_create("f2fs_nat_entry_set",
aec71382
CY		 3455 sizeof(struct nat_entry_set));
3456 if (!nat_entry_set_slab)
ce3e6d25 3457 goto destroy_free_nid;
50fa53ec 3458
98510003 3459 fsync_node_entry_slab = f2fs_kmem_cache_create("f2fs_fsync_node_entry",
50fa53ec
CY		 3460 sizeof(struct fsync_node_entry));
3461 if (!fsync_node_entry_slab)
3462 goto destroy_nat_entry_set;
e05df3b1 3463 return 0;
aec71382 3464
50fa53ec
CY		 3465destroy_nat_entry_set:
3466 kmem_cache_destroy(nat_entry_set_slab);
ce3e6d25 3467destroy_free_nid:
aec71382 3468 kmem_cache_destroy(free_nid_slab);
ce3e6d25 3469destroy_nat_entry:
aec71382
CY		 3470 kmem_cache_destroy(nat_entry_slab);
3471fail:
3472 return -ENOMEM;
e05df3b1
JK		 3473}
3474
4d57b86d 3475void f2fs_destroy_node_manager_caches(void)
e05df3b1 3476{
50fa53ec 3477 kmem_cache_destroy(fsync_node_entry_slab);
aec71382 3478 kmem_cache_destroy(nat_entry_set_slab);
e05df3b1
JK		 3479 kmem_cache_destroy(free_nid_slab);
3480 kmem_cache_destroy(nat_entry_slab);
3481}
Linux 6.x block layer and io_uring tree(s)