projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Btrfs: merge inode_list in __merge_refs
[linux-2.6-block.git] / fs / btrfs / delayed-inode.c
CommitLineData
16cdcec7
MX		 1/*
2 * Copyright (C) 2011 Fujitsu. All rights reserved.
3 * Written by Miao Xie <miaox@cn.fujitsu.com>
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public
7 * License v2 as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public
15 * License along with this program; if not, write to the
16 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
17 * Boston, MA 021110-1307, USA.
18 */
19
20#include <linux/slab.h>
21#include "delayed-inode.h"
22#include "disk-io.h"
23#include "transaction.h"
24
25#define BTRFS_DELAYED_WRITEBACK 400
26#define BTRFS_DELAYED_BACKGROUND 100
27
28static struct kmem_cache *delayed_node_cache;
29
30int __init btrfs_delayed_inode_init(void)
31{
837e1972 32 delayed_node_cache = kmem_cache_create("btrfs_delayed_node",
16cdcec7
MX		 33 sizeof(struct btrfs_delayed_node),
34 0,
35 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD,
36 NULL);
37 if (!delayed_node_cache)
38 return -ENOMEM;
39 return 0;
40}
41
42void btrfs_delayed_inode_exit(void)
43{
44 if (delayed_node_cache)
45 kmem_cache_destroy(delayed_node_cache);
46}
47
48static inline void btrfs_init_delayed_node(
49 struct btrfs_delayed_node *delayed_node,
50 struct btrfs_root *root, u64 inode_id)
51{
52 delayed_node->root = root;
53 delayed_node->inode_id = inode_id;
54 atomic_set(&delayed_node->refs, 0);
55 delayed_node->count = 0;
56 delayed_node->in_list = 0;
57 delayed_node->inode_dirty = 0;
58 delayed_node->ins_root = RB_ROOT;
59 delayed_node->del_root = RB_ROOT;
60 mutex_init(&delayed_node->mutex);
61 delayed_node->index_cnt = 0;
62 INIT_LIST_HEAD(&delayed_node->n_list);
63 INIT_LIST_HEAD(&delayed_node->p_list);
64 delayed_node->bytes_reserved = 0;
293f7e07 65 memset(&delayed_node->inode_item, 0, sizeof(delayed_node->inode_item));
16cdcec7
MX		 66}
67
68static inline int btrfs_is_continuous_delayed_item(
69 struct btrfs_delayed_item *item1,
70 struct btrfs_delayed_item *item2)
71{
72 if (item1->key.type == BTRFS_DIR_INDEX_KEY &&
73 item1->key.objectid == item2->key.objectid &&
74 item1->key.type == item2->key.type &&
75 item1->key.offset + 1 == item2->key.offset)
76 return 1;
77 return 0;
78}
79
80static inline struct btrfs_delayed_root *btrfs_get_delayed_root(
81 struct btrfs_root *root)
82{
83 return root->fs_info->delayed_root;
84}
85
2f7e33d4 86static struct btrfs_delayed_node *btrfs_get_delayed_node(struct inode *inode)
16cdcec7 87{
16cdcec7
MX		 88 struct btrfs_inode *btrfs_inode = BTRFS_I(inode);
89 struct btrfs_root *root = btrfs_inode->root;
0d0ca30f 90 u64 ino = btrfs_ino(inode);
2f7e33d4 91 struct btrfs_delayed_node *node;
16cdcec7 92
16cdcec7
MX		 93 node = ACCESS_ONCE(btrfs_inode->delayed_node);
94 if (node) {
2f7e33d4 95 atomic_inc(&node->refs);
16cdcec7
MX		 96 return node;
97 }
98
99 spin_lock(&root->inode_lock);
0d0ca30f 100 node = radix_tree_lookup(&root->delayed_nodes_tree, ino);
16cdcec7
MX		 101 if (node) {
102 if (btrfs_inode->delayed_node) {
2f7e33d4
MX		 103 atomic_inc(&node->refs); /* can be accessed */
104 BUG_ON(btrfs_inode->delayed_node != node);
16cdcec7 105 spin_unlock(&root->inode_lock);
2f7e33d4 106 return node;
16cdcec7
MX		 107 }
108 btrfs_inode->delayed_node = node;
109 atomic_inc(&node->refs); /* can be accessed */
110 atomic_inc(&node->refs); /* cached in the inode */
111 spin_unlock(&root->inode_lock);
112 return node;
113 }
114 spin_unlock(&root->inode_lock);
115
2f7e33d4
MX		 116 return NULL;
117}
118
79787eaa 119/* Will return either the node or PTR_ERR(-ENOMEM) */
2f7e33d4
MX		 120static struct btrfs_delayed_node *btrfs_get_or_create_delayed_node(
121 struct inode *inode)
122{
123 struct btrfs_delayed_node *node;
124 struct btrfs_inode *btrfs_inode = BTRFS_I(inode);
125 struct btrfs_root *root = btrfs_inode->root;
126 u64 ino = btrfs_ino(inode);
127 int ret;
128
129again:
130 node = btrfs_get_delayed_node(inode);
131 if (node)
132 return node;
133
16cdcec7
MX		 134 node = kmem_cache_alloc(delayed_node_cache, GFP_NOFS);
135 if (!node)
136 return ERR_PTR(-ENOMEM);
0d0ca30f 137 btrfs_init_delayed_node(node, root, ino);
16cdcec7
MX		 138
139 atomic_inc(&node->refs); /* cached in the btrfs inode */
140 atomic_inc(&node->refs); /* can be accessed */
141
142 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
143 if (ret) {
144 kmem_cache_free(delayed_node_cache, node);
145 return ERR_PTR(ret);
146 }
147
148 spin_lock(&root->inode_lock);
0d0ca30f 149 ret = radix_tree_insert(&root->delayed_nodes_tree, ino, node);
16cdcec7
MX		 150 if (ret == -EEXIST) {
151 kmem_cache_free(delayed_node_cache, node);
152 spin_unlock(&root->inode_lock);
153 radix_tree_preload_end();
154 goto again;
155 }
156 btrfs_inode->delayed_node = node;
157 spin_unlock(&root->inode_lock);
158 radix_tree_preload_end();
159
160 return node;
161}
162
163/*
164 * Call it when holding delayed_node->mutex
165 *
166 * If mod = 1, add this node into the prepared list.
167 */
168static void btrfs_queue_delayed_node(struct btrfs_delayed_root *root,
169 struct btrfs_delayed_node *node,
170 int mod)
171{
172 spin_lock(&root->lock);
173 if (node->in_list) {
174 if (!list_empty(&node->p_list))
175 list_move_tail(&node->p_list, &root->prepare_list);
176 else if (mod)
177 list_add_tail(&node->p_list, &root->prepare_list);
178 } else {
179 list_add_tail(&node->n_list, &root->node_list);
180 list_add_tail(&node->p_list, &root->prepare_list);
181 atomic_inc(&node->refs); /* inserted into list */
182 root->nodes++;
183 node->in_list = 1;
184 }
185 spin_unlock(&root->lock);
186}
187
188/* Call it when holding delayed_node->mutex */
189static void btrfs_dequeue_delayed_node(struct btrfs_delayed_root *root,
190 struct btrfs_delayed_node *node)
191{
192 spin_lock(&root->lock);
193 if (node->in_list) {
194 root->nodes--;
195 atomic_dec(&node->refs); /* not in the list */
196 list_del_init(&node->n_list);
197 if (!list_empty(&node->p_list))
198 list_del_init(&node->p_list);
199 node->in_list = 0;
200 }
201 spin_unlock(&root->lock);
202}
203
204struct btrfs_delayed_node *btrfs_first_delayed_node(
205 struct btrfs_delayed_root *delayed_root)
206{
207 struct list_head *p;
208 struct btrfs_delayed_node *node = NULL;
209
210 spin_lock(&delayed_root->lock);
211 if (list_empty(&delayed_root->node_list))
212 goto out;
213
214 p = delayed_root->node_list.next;
215 node = list_entry(p, struct btrfs_delayed_node, n_list);
216 atomic_inc(&node->refs);
217out:
218 spin_unlock(&delayed_root->lock);
219
220 return node;
221}
222
223struct btrfs_delayed_node *btrfs_next_delayed_node(
224 struct btrfs_delayed_node *node)
225{
226 struct btrfs_delayed_root *delayed_root;
227 struct list_head *p;
228 struct btrfs_delayed_node *next = NULL;
229
230 delayed_root = node->root->fs_info->delayed_root;
231 spin_lock(&delayed_root->lock);
232 if (!node->in_list) { /* not in the list */
233 if (list_empty(&delayed_root->node_list))
234 goto out;
235 p = delayed_root->node_list.next;
236 } else if (list_is_last(&node->n_list, &delayed_root->node_list))
237 goto out;
238 else
239 p = node->n_list.next;
240
241 next = list_entry(p, struct btrfs_delayed_node, n_list);
242 atomic_inc(&next->refs);
243out:
244 spin_unlock(&delayed_root->lock);
245
246 return next;
247}
248
249static void __btrfs_release_delayed_node(
250 struct btrfs_delayed_node *delayed_node,
251 int mod)
252{
253 struct btrfs_delayed_root *delayed_root;
254
255 if (!delayed_node)
256 return;
257
258 delayed_root = delayed_node->root->fs_info->delayed_root;
259
260 mutex_lock(&delayed_node->mutex);
261 if (delayed_node->count)
262 btrfs_queue_delayed_node(delayed_root, delayed_node, mod);
263 else
264 btrfs_dequeue_delayed_node(delayed_root, delayed_node);
265 mutex_unlock(&delayed_node->mutex);
266
267 if (atomic_dec_and_test(&delayed_node->refs)) {
268 struct btrfs_root *root = delayed_node->root;
269 spin_lock(&root->inode_lock);
270 if (atomic_read(&delayed_node->refs) == 0) {
271 radix_tree_delete(&root->delayed_nodes_tree,
272 delayed_node->inode_id);
273 kmem_cache_free(delayed_node_cache, delayed_node);
274 }
275 spin_unlock(&root->inode_lock);
276 }
277}
278
279static inline void btrfs_release_delayed_node(struct btrfs_delayed_node *node)
280{
281 __btrfs_release_delayed_node(node, 0);
282}
283
284struct btrfs_delayed_node *btrfs_first_prepared_delayed_node(
285 struct btrfs_delayed_root *delayed_root)
286{
287 struct list_head *p;
288 struct btrfs_delayed_node *node = NULL;
289
290 spin_lock(&delayed_root->lock);
291 if (list_empty(&delayed_root->prepare_list))
292 goto out;
293
294 p = delayed_root->prepare_list.next;
295 list_del_init(p);
296 node = list_entry(p, struct btrfs_delayed_node, p_list);
297 atomic_inc(&node->refs);
298out:
299 spin_unlock(&delayed_root->lock);
300
301 return node;
302}
303
304static inline void btrfs_release_prepared_delayed_node(
305 struct btrfs_delayed_node *node)
306{
307 __btrfs_release_delayed_node(node, 1);
308}
309
310struct btrfs_delayed_item *btrfs_alloc_delayed_item(u32 data_len)
311{
312 struct btrfs_delayed_item *item;
313 item = kmalloc(sizeof(*item) + data_len, GFP_NOFS);
314 if (item) {
315 item->data_len = data_len;
316 item->ins_or_del = 0;
317 item->bytes_reserved = 0;
16cdcec7
MX		 318 item->delayed_node = NULL;
319 atomic_set(&item->refs, 1);
320 }
321 return item;
322}
323
324/*
325 * __btrfs_lookup_delayed_item - look up the delayed item by key
326 * @delayed_node: pointer to the delayed node
327 * @key: the key to look up
328 * @prev: used to store the prev item if the right item isn't found
329 * @next: used to store the next item if the right item isn't found
330 *
331 * Note: if we don't find the right item, we will return the prev item and
332 * the next item.
333 */
334static struct btrfs_delayed_item *__btrfs_lookup_delayed_item(
335 struct rb_root *root,
336 struct btrfs_key *key,
337 struct btrfs_delayed_item **prev,
338 struct btrfs_delayed_item **next)
339{
340 struct rb_node *node, *prev_node = NULL;
341 struct btrfs_delayed_item *delayed_item = NULL;
342 int ret = 0;
343
344 node = root->rb_node;
345
346 while (node) {
347 delayed_item = rb_entry(node, struct btrfs_delayed_item,
348 rb_node);
349 prev_node = node;
350 ret = btrfs_comp_cpu_keys(&delayed_item->key, key);
351 if (ret < 0)
352 node = node->rb_right;
353 else if (ret > 0)
354 node = node->rb_left;
355 else
356 return delayed_item;
357 }
358
359 if (prev) {
360 if (!prev_node)
361 *prev = NULL;
362 else if (ret < 0)
363 *prev = delayed_item;
364 else if ((node = rb_prev(prev_node)) != NULL) {
365 *prev = rb_entry(node, struct btrfs_delayed_item,
366 rb_node);
367 } else
368 *prev = NULL;
369 }
370
371 if (next) {
372 if (!prev_node)
373 *next = NULL;
374 else if (ret > 0)
375 *next = delayed_item;
376 else if ((node = rb_next(prev_node)) != NULL) {
377 *next = rb_entry(node, struct btrfs_delayed_item,
378 rb_node);
379 } else
380 *next = NULL;
381 }
382 return NULL;
383}
384
385struct btrfs_delayed_item *__btrfs_lookup_delayed_insertion_item(
386 struct btrfs_delayed_node *delayed_node,
387 struct btrfs_key *key)
388{
389 struct btrfs_delayed_item *item;
390
391 item = __btrfs_lookup_delayed_item(&delayed_node->ins_root, key,
392 NULL, NULL);
393 return item;
394}
395
396struct btrfs_delayed_item *__btrfs_lookup_delayed_deletion_item(
397 struct btrfs_delayed_node *delayed_node,
398 struct btrfs_key *key)
399{
400 struct btrfs_delayed_item *item;
401
402 item = __btrfs_lookup_delayed_item(&delayed_node->del_root, key,
403 NULL, NULL);
404 return item;
405}
406
407struct btrfs_delayed_item *__btrfs_search_delayed_insertion_item(
408 struct btrfs_delayed_node *delayed_node,
409 struct btrfs_key *key)
410{
411 struct btrfs_delayed_item *item, *next;
412
413 item = __btrfs_lookup_delayed_item(&delayed_node->ins_root, key,
414 NULL, &next);
415 if (!item)
416 item = next;
417
418 return item;
419}
420
421struct btrfs_delayed_item *__btrfs_search_delayed_deletion_item(
422 struct btrfs_delayed_node *delayed_node,
423 struct btrfs_key *key)
424{
425 struct btrfs_delayed_item *item, *next;
426
427 item = __btrfs_lookup_delayed_item(&delayed_node->del_root, key,
428 NULL, &next);
429 if (!item)
430 item = next;
431
432 return item;
433}
434
435static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node,
436 struct btrfs_delayed_item *ins,
437 int action)
438{
439 struct rb_node **p, *node;
440 struct rb_node *parent_node = NULL;
441 struct rb_root *root;
442 struct btrfs_delayed_item *item;
443 int cmp;
444
445 if (action == BTRFS_DELAYED_INSERTION_ITEM)
446 root = &delayed_node->ins_root;
447 else if (action == BTRFS_DELAYED_DELETION_ITEM)
448 root = &delayed_node->del_root;
449 else
450 BUG();
451 p = &root->rb_node;
452 node = &ins->rb_node;
453
454 while (*p) {
455 parent_node = *p;
456 item = rb_entry(parent_node, struct btrfs_delayed_item,
457 rb_node);
458
459 cmp = btrfs_comp_cpu_keys(&item->key, &ins->key);
460 if (cmp < 0)
461 p = &(*p)->rb_right;
462 else if (cmp > 0)
463 p = &(*p)->rb_left;
464 else
465 return -EEXIST;
466 }
467
468 rb_link_node(node, parent_node, p);
469 rb_insert_color(node, root);
470 ins->delayed_node = delayed_node;
471 ins->ins_or_del = action;
472
473 if (ins->key.type == BTRFS_DIR_INDEX_KEY &&
474 action == BTRFS_DELAYED_INSERTION_ITEM &&
475 ins->key.offset >= delayed_node->index_cnt)
476 delayed_node->index_cnt = ins->key.offset + 1;
477
478 delayed_node->count++;
479 atomic_inc(&delayed_node->root->fs_info->delayed_root->items);
480 return 0;
481}
482
483static int __btrfs_add_delayed_insertion_item(struct btrfs_delayed_node *node,
484 struct btrfs_delayed_item *item)
485{
486 return __btrfs_add_delayed_item(node, item,
487 BTRFS_DELAYED_INSERTION_ITEM);
488}
489
490static int __btrfs_add_delayed_deletion_item(struct btrfs_delayed_node *node,
491 struct btrfs_delayed_item *item)
492{
493 return __btrfs_add_delayed_item(node, item,
494 BTRFS_DELAYED_DELETION_ITEM);
495}
496
497static void __btrfs_remove_delayed_item(struct btrfs_delayed_item *delayed_item)
498{
499 struct rb_root *root;
500 struct btrfs_delayed_root *delayed_root;
501
502 delayed_root = delayed_item->delayed_node->root->fs_info->delayed_root;
503
504 BUG_ON(!delayed_root);
505 BUG_ON(delayed_item->ins_or_del != BTRFS_DELAYED_DELETION_ITEM &&
506 delayed_item->ins_or_del != BTRFS_DELAYED_INSERTION_ITEM);
507
508 if (delayed_item->ins_or_del == BTRFS_DELAYED_INSERTION_ITEM)
509 root = &delayed_item->delayed_node->ins_root;
510 else
511 root = &delayed_item->delayed_node->del_root;
512
513 rb_erase(&delayed_item->rb_node, root);
514 delayed_item->delayed_node->count--;
66657b31
JB		 515 if (atomic_dec_return(&delayed_root->items) <
516 BTRFS_DELAYED_BACKGROUND &&
16cdcec7
MX		 517 waitqueue_active(&delayed_root->wait))
518 wake_up(&delayed_root->wait);
519}
520
521static void btrfs_release_delayed_item(struct btrfs_delayed_item *item)
522{
523 if (item) {
524 __btrfs_remove_delayed_item(item);
525 if (atomic_dec_and_test(&item->refs))
526 kfree(item);
527 }
528}
529
530struct btrfs_delayed_item *__btrfs_first_delayed_insertion_item(
531 struct btrfs_delayed_node *delayed_node)
532{
533 struct rb_node *p;
534 struct btrfs_delayed_item *item = NULL;
535
536 p = rb_first(&delayed_node->ins_root);
537 if (p)
538 item = rb_entry(p, struct btrfs_delayed_item, rb_node);
539
540 return item;
541}
542
543struct btrfs_delayed_item *__btrfs_first_delayed_deletion_item(
544 struct btrfs_delayed_node *delayed_node)
545{
546 struct rb_node *p;
547 struct btrfs_delayed_item *item = NULL;
548
549 p = rb_first(&delayed_node->del_root);
550 if (p)
551 item = rb_entry(p, struct btrfs_delayed_item, rb_node);
552
553 return item;
554}
555
556struct btrfs_delayed_item *__btrfs_next_delayed_item(
557 struct btrfs_delayed_item *item)
558{
559 struct rb_node *p;
560 struct btrfs_delayed_item *next = NULL;
561
562 p = rb_next(&item->rb_node);
563 if (p)
564 next = rb_entry(p, struct btrfs_delayed_item, rb_node);
565
566 return next;
567}
568
16cdcec7
MX		 569static inline struct btrfs_root *btrfs_get_fs_root(struct btrfs_root *root,
570 u64 root_id)
571{
572 struct btrfs_key root_key;
573
574 if (root->objectid == root_id)
575 return root;
576
577 root_key.objectid = root_id;
578 root_key.type = BTRFS_ROOT_ITEM_KEY;
579 root_key.offset = (u64)-1;
580 return btrfs_read_fs_root_no_name(root->fs_info, &root_key);
581}
582
583static int btrfs_delayed_item_reserve_metadata(struct btrfs_trans_handle *trans,
584 struct btrfs_root *root,
585 struct btrfs_delayed_item *item)
586{
587 struct btrfs_block_rsv *src_rsv;
588 struct btrfs_block_rsv *dst_rsv;
589 u64 num_bytes;
590 int ret;
591
592 if (!trans->bytes_reserved)
593 return 0;
594
595 src_rsv = trans->block_rsv;
6d668dda 596 dst_rsv = &root->fs_info->delayed_block_rsv;
16cdcec7
MX		 597
598 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
599 ret = btrfs_block_rsv_migrate(src_rsv, dst_rsv, num_bytes);
8c2a3ca2
JB		 600 if (!ret) {
601 trace_btrfs_space_reservation(root->fs_info, "delayed_item",
602 item->key.objectid,
603 num_bytes, 1);
16cdcec7 604 item->bytes_reserved = num_bytes;
8c2a3ca2 605 }
16cdcec7
MX		 606
607 return ret;
608}
609
610static void btrfs_delayed_item_release_metadata(struct btrfs_root *root,
611 struct btrfs_delayed_item *item)
612{
19fd2949
MX		 613 struct btrfs_block_rsv *rsv;
614
16cdcec7
MX		 615 if (!item->bytes_reserved)
616 return;
617
6d668dda 618 rsv = &root->fs_info->delayed_block_rsv;
8c2a3ca2
JB		 619 trace_btrfs_space_reservation(root->fs_info, "delayed_item",
620 item->key.objectid, item->bytes_reserved,
621 0);
19fd2949 622 btrfs_block_rsv_release(root, rsv,
16cdcec7
MX		 623 item->bytes_reserved);
624}
625
626static int btrfs_delayed_inode_reserve_metadata(
627 struct btrfs_trans_handle *trans,
628 struct btrfs_root *root,
7fd2ae21 629 struct inode *inode,
16cdcec7
MX		 630 struct btrfs_delayed_node *node)
631{
632 struct btrfs_block_rsv *src_rsv;
633 struct btrfs_block_rsv *dst_rsv;
634 u64 num_bytes;
635 int ret;
8c2a3ca2 636 bool release = false;
16cdcec7 637
16cdcec7 638 src_rsv = trans->block_rsv;
6d668dda 639 dst_rsv = &root->fs_info->delayed_block_rsv;
16cdcec7
MX		 640
641 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
c06a0e12
JB		 642
643 /*
644 * btrfs_dirty_inode will update the inode under btrfs_join_transaction
645 * which doesn't reserve space for speed. This is a problem since we
646 * still need to reserve space for this update, so try to reserve the
647 * space.
648 *
649 * Now if src_rsv == delalloc_block_rsv we'll let it just steal since
650 * we're accounted for.
651 */
e755d9ab 652 if (!src_rsv || (!trans->bytes_reserved &&
66d8f3dd 653 src_rsv->type != BTRFS_BLOCK_RSV_DELALLOC)) {
08e007d2
MX		 654 ret = btrfs_block_rsv_add(root, dst_rsv, num_bytes,
655 BTRFS_RESERVE_NO_FLUSH);
c06a0e12
JB		 656 /*
657 * Since we're under a transaction reserve_metadata_bytes could
658 * try to commit the transaction which will make it return
659 * EAGAIN to make us stop the transaction we have, so return
660 * ENOSPC instead so that btrfs_dirty_inode knows what to do.
661 */
662 if (ret == -EAGAIN)
663 ret = -ENOSPC;
8c2a3ca2 664 if (!ret) {
c06a0e12 665 node->bytes_reserved = num_bytes;
8c2a3ca2
JB		 666 trace_btrfs_space_reservation(root->fs_info,
667 "delayed_inode",
668 btrfs_ino(inode),
669 num_bytes, 1);
670 }
c06a0e12 671 return ret;
66d8f3dd 672 } else if (src_rsv->type == BTRFS_BLOCK_RSV_DELALLOC) {
7fd2ae21 673 spin_lock(&BTRFS_I(inode)->lock);
72ac3c0d
JB		 674 if (test_and_clear_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
675 &BTRFS_I(inode)->runtime_flags)) {
7fd2ae21
JB		 676 spin_unlock(&BTRFS_I(inode)->lock);
677 release = true;
678 goto migrate;
679 }
680 spin_unlock(&BTRFS_I(inode)->lock);
681
682 /* Ok we didn't have space pre-reserved. This shouldn't happen
683 * too often but it can happen if we do delalloc to an existing
684 * inode which gets dirtied because of the time update, and then
685 * isn't touched again until after the transaction commits and
686 * then we try to write out the data. First try to be nice and
687 * reserve something strictly for us. If not be a pain and try
688 * to steal from the delalloc block rsv.
689 */
08e007d2
MX		 690 ret = btrfs_block_rsv_add(root, dst_rsv, num_bytes,
691 BTRFS_RESERVE_NO_FLUSH);
7fd2ae21
JB		 692 if (!ret)
693 goto out;
694
695 ret = btrfs_block_rsv_migrate(src_rsv, dst_rsv, num_bytes);
696 if (!ret)
697 goto out;
698
699 /*
700 * Ok this is a problem, let's just steal from the global rsv
701 * since this really shouldn't happen that often.
702 */
703 WARN_ON(1);
704 ret = btrfs_block_rsv_migrate(&root->fs_info->global_block_rsv,
705 dst_rsv, num_bytes);
706 goto out;
c06a0e12
JB		 707 }
708
7fd2ae21 709migrate:
16cdcec7 710 ret = btrfs_block_rsv_migrate(src_rsv, dst_rsv, num_bytes);
7fd2ae21
JB		 711
712out:
713 /*
714 * Migrate only takes a reservation, it doesn't touch the size of the
715 * block_rsv. This is to simplify people who don't normally have things
716 * migrated from their block rsv. If they go to release their
717 * reservation, that will decrease the size as well, so if migrate
718 * reduced size we'd end up with a negative size. But for the
719 * delalloc_meta_reserved stuff we will only know to drop 1 reservation,
720 * but we could in fact do this reserve/migrate dance several times
721 * between the time we did the original reservation and we'd clean it
722 * up. So to take care of this, release the space for the meta
723 * reservation here. I think it may be time for a documentation page on
724 * how block rsvs. work.
725 */
8c2a3ca2
JB		 726 if (!ret) {
727 trace_btrfs_space_reservation(root->fs_info, "delayed_inode",
728 btrfs_ino(inode), num_bytes, 1);
16cdcec7 729 node->bytes_reserved = num_bytes;
8c2a3ca2 730 }
16cdcec7 731
8c2a3ca2
JB		 732 if (release) {
733 trace_btrfs_space_reservation(root->fs_info, "delalloc",
734 btrfs_ino(inode), num_bytes, 0);
7fd2ae21 735 btrfs_block_rsv_release(root, src_rsv, num_bytes);
8c2a3ca2 736 }
16cdcec7
MX		 737
738 return ret;
739}
740
741static void btrfs_delayed_inode_release_metadata(struct btrfs_root *root,
742 struct btrfs_delayed_node *node)
743{
744 struct btrfs_block_rsv *rsv;
745
746 if (!node->bytes_reserved)
747 return;
748
6d668dda 749 rsv = &root->fs_info->delayed_block_rsv;
8c2a3ca2
JB		 750 trace_btrfs_space_reservation(root->fs_info, "delayed_inode",
751 node->inode_id, node->bytes_reserved, 0);
16cdcec7
MX		 752 btrfs_block_rsv_release(root, rsv,
753 node->bytes_reserved);
754 node->bytes_reserved = 0;
755}
756
757/*
758 * This helper will insert some continuous items into the same leaf according
759 * to the free space of the leaf.
760 */
761static int btrfs_batch_insert_items(struct btrfs_trans_handle *trans,
762 struct btrfs_root *root,
763 struct btrfs_path *path,
764 struct btrfs_delayed_item *item)
765{
766 struct btrfs_delayed_item *curr, *next;
767 int free_space;
768 int total_data_size = 0, total_size = 0;
769 struct extent_buffer *leaf;
770 char *data_ptr;
771 struct btrfs_key *keys;
772 u32 *data_size;
773 struct list_head head;
774 int slot;
775 int nitems;
776 int i;
777 int ret = 0;
778
779 BUG_ON(!path->nodes[0]);
780
781 leaf = path->nodes[0];
782 free_space = btrfs_leaf_free_space(root, leaf);
783 INIT_LIST_HEAD(&head);
784
785 next = item;
17aca1c9 786 nitems = 0;
16cdcec7
MX		 787
788 /*
789 * count the number of the continuous items that we can insert in batch
790 */
791 while (total_size + next->data_len + sizeof(struct btrfs_item) <=
792 free_space) {
793 total_data_size += next->data_len;
794 total_size += next->data_len + sizeof(struct btrfs_item);
795 list_add_tail(&next->tree_list, &head);
796 nitems++;
797
798 curr = next;
799 next = __btrfs_next_delayed_item(curr);
800 if (!next)
801 break;
802
803 if (!btrfs_is_continuous_delayed_item(curr, next))
804 break;
805 }
806
807 if (!nitems) {
808 ret = 0;
809 goto out;
810 }
811
812 /*
813 * we need allocate some memory space, but it might cause the task
814 * to sleep, so we set all locked nodes in the path to blocking locks
815 * first.
816 */
817 btrfs_set_path_blocking(path);
818
819 keys = kmalloc(sizeof(struct btrfs_key) * nitems, GFP_NOFS);
820 if (!keys) {
821 ret = -ENOMEM;
822 goto out;
823 }
824
825 data_size = kmalloc(sizeof(u32) * nitems, GFP_NOFS);
826 if (!data_size) {
827 ret = -ENOMEM;
828 goto error;
829 }
830
831 /* get keys of all the delayed items */
832 i = 0;
833 list_for_each_entry(next, &head, tree_list) {
834 keys[i] = next->key;
835 data_size[i] = next->data_len;
836 i++;
837 }
838
839 /* reset all the locked nodes in the patch to spinning locks. */
bd681513 840 btrfs_clear_path_blocking(path, NULL, 0);
16cdcec7
MX		 841
842 /* insert the keys of the items */
143bede5
JM		 843 setup_items_for_insert(trans, root, path, keys, data_size,
844 total_data_size, total_size, nitems);
16cdcec7
MX		 845
846 /* insert the dir index items */
847 slot = path->slots[0];
848 list_for_each_entry_safe(curr, next, &head, tree_list) {
849 data_ptr = btrfs_item_ptr(leaf, slot, char);
850 write_extent_buffer(leaf, &curr->data,
851 (unsigned long)data_ptr,
852 curr->data_len);
853 slot++;
854
855 btrfs_delayed_item_release_metadata(root, curr);
856
857 list_del(&curr->tree_list);
858 btrfs_release_delayed_item(curr);
859 }
860
861error:
862 kfree(data_size);
863 kfree(keys);
864out:
865 return ret;
866}
867
868/*
869 * This helper can just do simple insertion that needn't extend item for new
870 * data, such as directory name index insertion, inode insertion.
871 */
872static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans,
873 struct btrfs_root *root,
874 struct btrfs_path *path,
875 struct btrfs_delayed_item *delayed_item)
876{
877 struct extent_buffer *leaf;
878 struct btrfs_item *item;
879 char *ptr;
880 int ret;
881
882 ret = btrfs_insert_empty_item(trans, root, path, &delayed_item->key,
883 delayed_item->data_len);
884 if (ret < 0 && ret != -EEXIST)
885 return ret;
886
887 leaf = path->nodes[0];
888
889 item = btrfs_item_nr(leaf, path->slots[0]);
890 ptr = btrfs_item_ptr(leaf, path->slots[0], char);
891
892 write_extent_buffer(leaf, delayed_item->data, (unsigned long)ptr,
893 delayed_item->data_len);
894 btrfs_mark_buffer_dirty(leaf);
895
896 btrfs_delayed_item_release_metadata(root, delayed_item);
897 return 0;
898}
899
900/*
901 * we insert an item first, then if there are some continuous items, we try
902 * to insert those items into the same leaf.
903 */
904static int btrfs_insert_delayed_items(struct btrfs_trans_handle *trans,
905 struct btrfs_path *path,
906 struct btrfs_root *root,
907 struct btrfs_delayed_node *node)
908{
909 struct btrfs_delayed_item *curr, *prev;
910 int ret = 0;
911
912do_again:
913 mutex_lock(&node->mutex);
914 curr = __btrfs_first_delayed_insertion_item(node);
915 if (!curr)
916 goto insert_end;
917
918 ret = btrfs_insert_delayed_item(trans, root, path, curr);
919 if (ret < 0) {
945d8962 920 btrfs_release_path(path);
16cdcec7
MX		 921 goto insert_end;
922 }
923
924 prev = curr;
925 curr = __btrfs_next_delayed_item(prev);
926 if (curr && btrfs_is_continuous_delayed_item(prev, curr)) {
927 /* insert the continuous items into the same leaf */
928 path->slots[0]++;
929 btrfs_batch_insert_items(trans, root, path, curr);
930 }
931 btrfs_release_delayed_item(prev);
932 btrfs_mark_buffer_dirty(path->nodes[0]);
933
945d8962 934 btrfs_release_path(path);
16cdcec7
MX		 935 mutex_unlock(&node->mutex);
936 goto do_again;
937
938insert_end:
939 mutex_unlock(&node->mutex);
940 return ret;
941}
942
943static int btrfs_batch_delete_items(struct btrfs_trans_handle *trans,
944 struct btrfs_root *root,
945 struct btrfs_path *path,
946 struct btrfs_delayed_item *item)
947{
948 struct btrfs_delayed_item *curr, *next;
949 struct extent_buffer *leaf;
950 struct btrfs_key key;
951 struct list_head head;
952 int nitems, i, last_item;
953 int ret = 0;
954
955 BUG_ON(!path->nodes[0]);
956
957 leaf = path->nodes[0];
958
959 i = path->slots[0];
960 last_item = btrfs_header_nritems(leaf) - 1;
961 if (i > last_item)
962 return -ENOENT; /* FIXME: Is errno suitable? */
963
964 next = item;
965 INIT_LIST_HEAD(&head);
966 btrfs_item_key_to_cpu(leaf, &key, i);
967 nitems = 0;
968 /*
969 * count the number of the dir index items that we can delete in batch
970 */
971 while (btrfs_comp_cpu_keys(&next->key, &key) == 0) {
972 list_add_tail(&next->tree_list, &head);
973 nitems++;
974
975 curr = next;
976 next = __btrfs_next_delayed_item(curr);
977 if (!next)
978 break;
979
980 if (!btrfs_is_continuous_delayed_item(curr, next))
981 break;
982
983 i++;
984 if (i > last_item)
985 break;
986 btrfs_item_key_to_cpu(leaf, &key, i);
987 }
988
989 if (!nitems)
990 return 0;
991
992 ret = btrfs_del_items(trans, root, path, path->slots[0], nitems);
993 if (ret)
994 goto out;
995
996 list_for_each_entry_safe(curr, next, &head, tree_list) {
997 btrfs_delayed_item_release_metadata(root, curr);
998 list_del(&curr->tree_list);
999 btrfs_release_delayed_item(curr);
1000 }
1001
1002out:
1003 return ret;
1004}
1005
1006static int btrfs_delete_delayed_items(struct btrfs_trans_handle *trans,
1007 struct btrfs_path *path,
1008 struct btrfs_root *root,
1009 struct btrfs_delayed_node *node)
1010{
1011 struct btrfs_delayed_item *curr, *prev;
1012 int ret = 0;
1013
1014do_again:
1015 mutex_lock(&node->mutex);
1016 curr = __btrfs_first_delayed_deletion_item(node);
1017 if (!curr)
1018 goto delete_fail;
1019
1020 ret = btrfs_search_slot(trans, root, &curr->key, path, -1, 1);
1021 if (ret < 0)
1022 goto delete_fail;
1023 else if (ret > 0) {
1024 /*
1025 * can't find the item which the node points to, so this node
1026 * is invalid, just drop it.
1027 */
1028 prev = curr;
1029 curr = __btrfs_next_delayed_item(prev);
1030 btrfs_release_delayed_item(prev);
1031 ret = 0;
945d8962 1032 btrfs_release_path(path);
62095265
FW		 1033 if (curr) {
1034 mutex_unlock(&node->mutex);
16cdcec7 1035 goto do_again;
62095265 1036 } else
16cdcec7
MX		 1037 goto delete_fail;
1038 }
1039
1040 btrfs_batch_delete_items(trans, root, path, curr);
945d8962 1041 btrfs_release_path(path);
16cdcec7
MX		 1042 mutex_unlock(&node->mutex);
1043 goto do_again;
1044
1045delete_fail:
945d8962 1046 btrfs_release_path(path);
16cdcec7
MX		 1047 mutex_unlock(&node->mutex);
1048 return ret;
1049}
1050
1051static void btrfs_release_delayed_inode(struct btrfs_delayed_node *delayed_node)
1052{
1053 struct btrfs_delayed_root *delayed_root;
1054
1055 if (delayed_node && delayed_node->inode_dirty) {
1056 BUG_ON(!delayed_node->root);
1057 delayed_node->inode_dirty = 0;
1058 delayed_node->count--;
1059
1060 delayed_root = delayed_node->root->fs_info->delayed_root;
66657b31 1061 if (atomic_dec_return(&delayed_root->items) <
16cdcec7
MX		 1062 BTRFS_DELAYED_BACKGROUND &&
1063 waitqueue_active(&delayed_root->wait))
1064 wake_up(&delayed_root->wait);
1065 }
1066}
1067
1068static int btrfs_update_delayed_inode(struct btrfs_trans_handle *trans,
1069 struct btrfs_root *root,
1070 struct btrfs_path *path,
1071 struct btrfs_delayed_node *node)
1072{
1073 struct btrfs_key key;
1074 struct btrfs_inode_item *inode_item;
1075 struct extent_buffer *leaf;
1076 int ret;
1077
1078 mutex_lock(&node->mutex);
1079 if (!node->inode_dirty) {
1080 mutex_unlock(&node->mutex);
1081 return 0;
1082 }
1083
1084 key.objectid = node->inode_id;
1085 btrfs_set_key_type(&key, BTRFS_INODE_ITEM_KEY);
1086 key.offset = 0;
1087 ret = btrfs_lookup_inode(trans, root, path, &key, 1);
1088 if (ret > 0) {
945d8962 1089 btrfs_release_path(path);
16cdcec7
MX		 1090 mutex_unlock(&node->mutex);
1091 return -ENOENT;
1092 } else if (ret < 0) {
1093 mutex_unlock(&node->mutex);
1094 return ret;
1095 }
1096
1097 btrfs_unlock_up_safe(path, 1);
1098 leaf = path->nodes[0];
1099 inode_item = btrfs_item_ptr(leaf, path->slots[0],
1100 struct btrfs_inode_item);
1101 write_extent_buffer(leaf, &node->inode_item, (unsigned long)inode_item,
1102 sizeof(struct btrfs_inode_item));
1103 btrfs_mark_buffer_dirty(leaf);
945d8962 1104 btrfs_release_path(path);
16cdcec7
MX		 1105
1106 btrfs_delayed_inode_release_metadata(root, node);
1107 btrfs_release_delayed_inode(node);
1108 mutex_unlock(&node->mutex);
1109
1110 return 0;
1111}
1112
79787eaa
JM		 1113/*
1114 * Called when committing the transaction.
1115 * Returns 0 on success.
1116 * Returns < 0 on error and returns with an aborted transaction with any
1117 * outstanding delayed items cleaned up.
1118 */
96c3f433
JB		 1119static int __btrfs_run_delayed_items(struct btrfs_trans_handle *trans,
1120 struct btrfs_root *root, int nr)
16cdcec7 1121{
79787eaa 1122 struct btrfs_root *curr_root = root;
16cdcec7
MX		 1123 struct btrfs_delayed_root *delayed_root;
1124 struct btrfs_delayed_node *curr_node, *prev_node;
1125 struct btrfs_path *path;
19fd2949 1126 struct btrfs_block_rsv *block_rsv;
16cdcec7 1127 int ret = 0;
96c3f433 1128 bool count = (nr > 0);
16cdcec7 1129
79787eaa
JM		 1130 if (trans->aborted)
1131 return -EIO;
1132
16cdcec7
MX		 1133 path = btrfs_alloc_path();
1134 if (!path)
1135 return -ENOMEM;
1136 path->leave_spinning = 1;
1137
19fd2949 1138 block_rsv = trans->block_rsv;
6d668dda 1139 trans->block_rsv = &root->fs_info->delayed_block_rsv;
19fd2949 1140
16cdcec7
MX		 1141 delayed_root = btrfs_get_delayed_root(root);
1142
1143 curr_node = btrfs_first_delayed_node(delayed_root);
96c3f433 1144 while (curr_node && (!count || (count && nr--))) {
79787eaa
JM		 1145 curr_root = curr_node->root;
1146 ret = btrfs_insert_delayed_items(trans, path, curr_root,
16cdcec7
MX		 1147 curr_node);
1148 if (!ret)
79787eaa
JM		 1149 ret = btrfs_delete_delayed_items(trans, path,
1150 curr_root, curr_node);
16cdcec7 1151 if (!ret)
79787eaa
JM		 1152 ret = btrfs_update_delayed_inode(trans, curr_root,
1153 path, curr_node);
16cdcec7
MX		 1154 if (ret) {
1155 btrfs_release_delayed_node(curr_node);
96c3f433 1156 curr_node = NULL;
79787eaa 1157 btrfs_abort_transaction(trans, root, ret);
16cdcec7
MX		 1158 break;
1159 }
1160
1161 prev_node = curr_node;
1162 curr_node = btrfs_next_delayed_node(curr_node);
1163 btrfs_release_delayed_node(prev_node);
1164 }
1165
96c3f433
JB		 1166 if (curr_node)
1167 btrfs_release_delayed_node(curr_node);
16cdcec7 1168 btrfs_free_path(path);
19fd2949 1169 trans->block_rsv = block_rsv;
79787eaa 1170
16cdcec7
MX		 1171 return ret;
1172}
1173
96c3f433
JB		 1174int btrfs_run_delayed_items(struct btrfs_trans_handle *trans,
1175 struct btrfs_root *root)
1176{
1177 return __btrfs_run_delayed_items(trans, root, -1);
1178}
1179
1180int btrfs_run_delayed_items_nr(struct btrfs_trans_handle *trans,
1181 struct btrfs_root *root, int nr)
1182{
1183 return __btrfs_run_delayed_items(trans, root, nr);
1184}
1185
16cdcec7
MX		 1186static int __btrfs_commit_inode_delayed_items(struct btrfs_trans_handle *trans,
1187 struct btrfs_delayed_node *node)
1188{
1189 struct btrfs_path *path;
19fd2949 1190 struct btrfs_block_rsv *block_rsv;
16cdcec7
MX		 1191 int ret;
1192
1193 path = btrfs_alloc_path();
1194 if (!path)
1195 return -ENOMEM;
1196 path->leave_spinning = 1;
1197
19fd2949 1198 block_rsv = trans->block_rsv;
6d668dda 1199 trans->block_rsv = &node->root->fs_info->delayed_block_rsv;
19fd2949 1200
16cdcec7
MX		 1201 ret = btrfs_insert_delayed_items(trans, path, node->root, node);
1202 if (!ret)
1203 ret = btrfs_delete_delayed_items(trans, path, node->root, node);
1204 if (!ret)
1205 ret = btrfs_update_delayed_inode(trans, node->root, path, node);
1206 btrfs_free_path(path);
1207
19fd2949 1208 trans->block_rsv = block_rsv;
16cdcec7
MX		 1209 return ret;
1210}
1211
1212int btrfs_commit_inode_delayed_items(struct btrfs_trans_handle *trans,
1213 struct inode *inode)
1214{
1215 struct btrfs_delayed_node *delayed_node = btrfs_get_delayed_node(inode);
1216 int ret;
1217
1218 if (!delayed_node)
1219 return 0;
1220
1221 mutex_lock(&delayed_node->mutex);
1222 if (!delayed_node->count) {
1223 mutex_unlock(&delayed_node->mutex);
1224 btrfs_release_delayed_node(delayed_node);
1225 return 0;
1226 }
1227 mutex_unlock(&delayed_node->mutex);
1228
1229 ret = __btrfs_commit_inode_delayed_items(trans, delayed_node);
1230 btrfs_release_delayed_node(delayed_node);
1231 return ret;
1232}
1233
1234void btrfs_remove_delayed_node(struct inode *inode)
1235{
1236 struct btrfs_delayed_node *delayed_node;
1237
1238 delayed_node = ACCESS_ONCE(BTRFS_I(inode)->delayed_node);
1239 if (!delayed_node)
1240 return;
1241
1242 BTRFS_I(inode)->delayed_node = NULL;
1243 btrfs_release_delayed_node(delayed_node);
1244}
1245
1246struct btrfs_async_delayed_node {
1247 struct btrfs_root *root;
1248 struct btrfs_delayed_node *delayed_node;
1249 struct btrfs_work work;
1250};
1251
1252static void btrfs_async_run_delayed_node_done(struct btrfs_work *work)
1253{
1254 struct btrfs_async_delayed_node *async_node;
1255 struct btrfs_trans_handle *trans;
1256 struct btrfs_path *path;
1257 struct btrfs_delayed_node *delayed_node = NULL;
1258 struct btrfs_root *root;
19fd2949 1259 struct btrfs_block_rsv *block_rsv;
16cdcec7
MX		 1260 unsigned long nr = 0;
1261 int need_requeue = 0;
1262 int ret;
1263
1264 async_node = container_of(work, struct btrfs_async_delayed_node, work);
1265
1266 path = btrfs_alloc_path();
1267 if (!path)
1268 goto out;
1269 path->leave_spinning = 1;
1270
1271 delayed_node = async_node->delayed_node;
1272 root = delayed_node->root;
1273
ff5714cc 1274 trans = btrfs_join_transaction(root);
16cdcec7
MX		 1275 if (IS_ERR(trans))
1276 goto free_path;
1277
19fd2949 1278 block_rsv = trans->block_rsv;
6d668dda 1279 trans->block_rsv = &root->fs_info->delayed_block_rsv;
19fd2949 1280
16cdcec7
MX		 1281 ret = btrfs_insert_delayed_items(trans, path, root, delayed_node);
1282 if (!ret)
1283 ret = btrfs_delete_delayed_items(trans, path, root,
1284 delayed_node);
1285
1286 if (!ret)
1287 btrfs_update_delayed_inode(trans, root, path, delayed_node);
1288
1289 /*
1290 * Maybe new delayed items have been inserted, so we need requeue
1291 * the work. Besides that, we must dequeue the empty delayed nodes
1292 * to avoid the race between delayed items balance and the worker.
1293 * The race like this:
1294 * Task1 Worker thread
1295 * count == 0, needn't requeue
1296 * also needn't insert the
1297 * delayed node into prepare
1298 * list again.
1299 * add lots of delayed items
1300 * queue the delayed node
1301 * already in the list,
1302 * and not in the prepare
1303 * list, it means the delayed
1304 * node is being dealt with
1305 * by the worker.
1306 * do delayed items balance
1307 * the delayed node is being
1308 * dealt with by the worker
1309 * now, just wait.
1310 * the worker goto idle.
1311 * Task1 will sleep until the transaction is commited.
1312 */
1313 mutex_lock(&delayed_node->mutex);
1314 if (delayed_node->count)
1315 need_requeue = 1;
1316 else
1317 btrfs_dequeue_delayed_node(root->fs_info->delayed_root,
1318 delayed_node);
1319 mutex_unlock(&delayed_node->mutex);
1320
1321 nr = trans->blocks_used;
1322
19fd2949 1323 trans->block_rsv = block_rsv;
16cdcec7
MX		 1324 btrfs_end_transaction_dmeta(trans, root);
1325 __btrfs_btree_balance_dirty(root, nr);
1326free_path:
1327 btrfs_free_path(path);
1328out:
1329 if (need_requeue)
1330 btrfs_requeue_work(&async_node->work);
1331 else {
1332 btrfs_release_prepared_delayed_node(delayed_node);
1333 kfree(async_node);
1334 }
1335}
1336
1337static int btrfs_wq_run_delayed_node(struct btrfs_delayed_root *delayed_root,
1338 struct btrfs_root *root, int all)
1339{
1340 struct btrfs_async_delayed_node *async_node;
1341 struct btrfs_delayed_node *curr;
1342 int count = 0;
1343
1344again:
1345 curr = btrfs_first_prepared_delayed_node(delayed_root);
1346 if (!curr)
1347 return 0;
1348
1349 async_node = kmalloc(sizeof(*async_node), GFP_NOFS);
1350 if (!async_node) {
1351 btrfs_release_prepared_delayed_node(curr);
1352 return -ENOMEM;
1353 }
1354
1355 async_node->root = root;
1356 async_node->delayed_node = curr;
1357
1358 async_node->work.func = btrfs_async_run_delayed_node_done;
1359 async_node->work.flags = 0;
1360
1361 btrfs_queue_worker(&root->fs_info->delayed_workers, &async_node->work);
1362 count++;
1363
1364 if (all || count < 4)
1365 goto again;
1366
1367 return 0;
1368}
1369
e999376f
CM		 1370void btrfs_assert_delayed_root_empty(struct btrfs_root *root)
1371{
1372 struct btrfs_delayed_root *delayed_root;
1373 delayed_root = btrfs_get_delayed_root(root);
1374 WARN_ON(btrfs_first_delayed_node(delayed_root));
1375}
1376
16cdcec7
MX		 1377void btrfs_balance_delayed_items(struct btrfs_root *root)
1378{
1379 struct btrfs_delayed_root *delayed_root;
1380
1381 delayed_root = btrfs_get_delayed_root(root);
1382
1383 if (atomic_read(&delayed_root->items) < BTRFS_DELAYED_BACKGROUND)
1384 return;
1385
1386 if (atomic_read(&delayed_root->items) >= BTRFS_DELAYED_WRITEBACK) {
1387 int ret;
1388 ret = btrfs_wq_run_delayed_node(delayed_root, root, 1);
1389 if (ret)
1390 return;
1391
1392 wait_event_interruptible_timeout(
1393 delayed_root->wait,
1394 (atomic_read(&delayed_root->items) <
1395 BTRFS_DELAYED_BACKGROUND),
1396 HZ);
1397 return;
1398 }
1399
1400 btrfs_wq_run_delayed_node(delayed_root, root, 0);
1401}
1402
79787eaa 1403/* Will return 0 or -ENOMEM */
16cdcec7
MX		 1404int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans,
1405 struct btrfs_root *root, const char *name,
1406 int name_len, struct inode *dir,
1407 struct btrfs_disk_key *disk_key, u8 type,
1408 u64 index)
1409{
1410 struct btrfs_delayed_node *delayed_node;
1411 struct btrfs_delayed_item *delayed_item;
1412 struct btrfs_dir_item *dir_item;
1413 int ret;
1414
1415 delayed_node = btrfs_get_or_create_delayed_node(dir);
1416 if (IS_ERR(delayed_node))
1417 return PTR_ERR(delayed_node);
1418
1419 delayed_item = btrfs_alloc_delayed_item(sizeof(*dir_item) + name_len);
1420 if (!delayed_item) {
1421 ret = -ENOMEM;
1422 goto release_node;
1423 }
1424
0d0ca30f 1425 delayed_item->key.objectid = btrfs_ino(dir);
16cdcec7
MX		 1426 btrfs_set_key_type(&delayed_item->key, BTRFS_DIR_INDEX_KEY);
1427 delayed_item->key.offset = index;
1428
1429 dir_item = (struct btrfs_dir_item *)delayed_item->data;
1430 dir_item->location = *disk_key;
1431 dir_item->transid = cpu_to_le64(trans->transid);
1432 dir_item->data_len = 0;
1433 dir_item->name_len = cpu_to_le16(name_len);
1434 dir_item->type = type;
1435 memcpy((char *)(dir_item + 1), name, name_len);
1436
8c2a3ca2
JB		 1437 ret = btrfs_delayed_item_reserve_metadata(trans, root, delayed_item);
1438 /*
1439 * we have reserved enough space when we start a new transaction,
1440 * so reserving metadata failure is impossible
1441 */
1442 BUG_ON(ret);
1443
1444
16cdcec7
MX		 1445 mutex_lock(&delayed_node->mutex);
1446 ret = __btrfs_add_delayed_insertion_item(delayed_node, delayed_item);
1447 if (unlikely(ret)) {
1448 printk(KERN_ERR "err add delayed dir index item(name: %s) into "
1449 "the insertion tree of the delayed node"
1450 "(root id: %llu, inode id: %llu, errno: %d)\n",
1451 name,
1452 (unsigned long long)delayed_node->root->objectid,
1453 (unsigned long long)delayed_node->inode_id,
1454 ret);
1455 BUG();
1456 }
1457 mutex_unlock(&delayed_node->mutex);
1458
1459release_node:
1460 btrfs_release_delayed_node(delayed_node);
1461 return ret;
1462}
1463
1464static int btrfs_delete_delayed_insertion_item(struct btrfs_root *root,
1465 struct btrfs_delayed_node *node,
1466 struct btrfs_key *key)
1467{
1468 struct btrfs_delayed_item *item;
1469
1470 mutex_lock(&node->mutex);
1471 item = __btrfs_lookup_delayed_insertion_item(node, key);
1472 if (!item) {
1473 mutex_unlock(&node->mutex);
1474 return 1;
1475 }
1476
1477 btrfs_delayed_item_release_metadata(root, item);
1478 btrfs_release_delayed_item(item);
1479 mutex_unlock(&node->mutex);
1480 return 0;
1481}
1482
1483int btrfs_delete_delayed_dir_index(struct btrfs_trans_handle *trans,
1484 struct btrfs_root *root, struct inode *dir,
1485 u64 index)
1486{
1487 struct btrfs_delayed_node *node;
1488 struct btrfs_delayed_item *item;
1489 struct btrfs_key item_key;
1490 int ret;
1491
1492 node = btrfs_get_or_create_delayed_node(dir);
1493 if (IS_ERR(node))
1494 return PTR_ERR(node);
1495
0d0ca30f 1496 item_key.objectid = btrfs_ino(dir);
16cdcec7
MX		 1497 btrfs_set_key_type(&item_key, BTRFS_DIR_INDEX_KEY);
1498 item_key.offset = index;
1499
1500 ret = btrfs_delete_delayed_insertion_item(root, node, &item_key);
1501 if (!ret)
1502 goto end;
1503
1504 item = btrfs_alloc_delayed_item(0);
1505 if (!item) {
1506 ret = -ENOMEM;
1507 goto end;
1508 }
1509
1510 item->key = item_key;
1511
1512 ret = btrfs_delayed_item_reserve_metadata(trans, root, item);
1513 /*
1514 * we have reserved enough space when we start a new transaction,
1515 * so reserving metadata failure is impossible.
1516 */
1517 BUG_ON(ret);
1518
1519 mutex_lock(&node->mutex);
1520 ret = __btrfs_add_delayed_deletion_item(node, item);
1521 if (unlikely(ret)) {
1522 printk(KERN_ERR "err add delayed dir index item(index: %llu) "
1523 "into the deletion tree of the delayed node"
1524 "(root id: %llu, inode id: %llu, errno: %d)\n",
1525 (unsigned long long)index,
1526 (unsigned long long)node->root->objectid,
1527 (unsigned long long)node->inode_id,
1528 ret);
1529 BUG();
1530 }
1531 mutex_unlock(&node->mutex);
1532end:
1533 btrfs_release_delayed_node(node);
1534 return ret;
1535}
1536
1537int btrfs_inode_delayed_dir_index_count(struct inode *inode)
1538{
2f7e33d4 1539 struct btrfs_delayed_node *delayed_node = btrfs_get_delayed_node(inode);
16cdcec7
MX		 1540
1541 if (!delayed_node)
1542 return -ENOENT;
1543
1544 /*
1545 * Since we have held i_mutex of this directory, it is impossible that
1546 * a new directory index is added into the delayed node and index_cnt
1547 * is updated now. So we needn't lock the delayed node.
1548 */
2f7e33d4
MX		 1549 if (!delayed_node->index_cnt) {
1550 btrfs_release_delayed_node(delayed_node);
16cdcec7 1551 return -EINVAL;
2f7e33d4 1552 }
16cdcec7
MX		 1553
1554 BTRFS_I(inode)->index_cnt = delayed_node->index_cnt;
2f7e33d4
MX		 1555 btrfs_release_delayed_node(delayed_node);
1556 return 0;
16cdcec7
MX		 1557}
1558
1559void btrfs_get_delayed_items(struct inode *inode, struct list_head *ins_list,
1560 struct list_head *del_list)
1561{
1562 struct btrfs_delayed_node *delayed_node;
1563 struct btrfs_delayed_item *item;
1564
1565 delayed_node = btrfs_get_delayed_node(inode);
1566 if (!delayed_node)
1567 return;
1568
1569 mutex_lock(&delayed_node->mutex);
1570 item = __btrfs_first_delayed_insertion_item(delayed_node);
1571 while (item) {
1572 atomic_inc(&item->refs);
1573 list_add_tail(&item->readdir_list, ins_list);
1574 item = __btrfs_next_delayed_item(item);
1575 }
1576
1577 item = __btrfs_first_delayed_deletion_item(delayed_node);
1578 while (item) {
1579 atomic_inc(&item->refs);
1580 list_add_tail(&item->readdir_list, del_list);
1581 item = __btrfs_next_delayed_item(item);
1582 }
1583 mutex_unlock(&delayed_node->mutex);
1584 /*
1585 * This delayed node is still cached in the btrfs inode, so refs
1586 * must be > 1 now, and we needn't check it is going to be freed
1587 * or not.
1588 *
1589 * Besides that, this function is used to read dir, we do not
1590 * insert/delete delayed items in this period. So we also needn't
1591 * requeue or dequeue this delayed node.
1592 */
1593 atomic_dec(&delayed_node->refs);
1594}
1595
1596void btrfs_put_delayed_items(struct list_head *ins_list,
1597 struct list_head *del_list)
1598{
1599 struct btrfs_delayed_item *curr, *next;
1600
1601 list_for_each_entry_safe(curr, next, ins_list, readdir_list) {
1602 list_del(&curr->readdir_list);
1603 if (atomic_dec_and_test(&curr->refs))
1604 kfree(curr);
1605 }
1606
1607 list_for_each_entry_safe(curr, next, del_list, readdir_list) {
1608 list_del(&curr->readdir_list);
1609 if (atomic_dec_and_test(&curr->refs))
1610 kfree(curr);
1611 }
1612}
1613
1614int btrfs_should_delete_dir_index(struct list_head *del_list,
1615 u64 index)
1616{
1617 struct btrfs_delayed_item *curr, *next;
1618 int ret;
1619
1620 if (list_empty(del_list))
1621 return 0;
1622
1623 list_for_each_entry_safe(curr, next, del_list, readdir_list) {
1624 if (curr->key.offset > index)
1625 break;
1626
1627 list_del(&curr->readdir_list);
1628 ret = (curr->key.offset == index);
1629
1630 if (atomic_dec_and_test(&curr->refs))
1631 kfree(curr);
1632
1633 if (ret)
1634 return 1;
1635 else
1636 continue;
1637 }
1638 return 0;
1639}
1640
1641/*
1642 * btrfs_readdir_delayed_dir_index - read dir info stored in the delayed tree
1643 *
1644 */
1645int btrfs_readdir_delayed_dir_index(struct file *filp, void *dirent,
1646 filldir_t filldir,
1647 struct list_head *ins_list)
1648{
1649 struct btrfs_dir_item *di;
1650 struct btrfs_delayed_item *curr, *next;
1651 struct btrfs_key location;
1652 char *name;
1653 int name_len;
1654 int over = 0;
1655 unsigned char d_type;
1656
1657 if (list_empty(ins_list))
1658 return 0;
1659
1660 /*
1661 * Changing the data of the delayed item is impossible. So
1662 * we needn't lock them. And we have held i_mutex of the
1663 * directory, nobody can delete any directory indexes now.
1664 */
1665 list_for_each_entry_safe(curr, next, ins_list, readdir_list) {
1666 list_del(&curr->readdir_list);
1667
1668 if (curr->key.offset < filp->f_pos) {
1669 if (atomic_dec_and_test(&curr->refs))
1670 kfree(curr);
1671 continue;
1672 }
1673
1674 filp->f_pos = curr->key.offset;
1675
1676 di = (struct btrfs_dir_item *)curr->data;
1677 name = (char *)(di + 1);
1678 name_len = le16_to_cpu(di->name_len);
1679
1680 d_type = btrfs_filetype_table[di->type];
1681 btrfs_disk_key_to_cpu(&location, &di->location);
1682
1683 over = filldir(dirent, name, name_len, curr->key.offset,
1684 location.objectid, d_type);
1685
1686 if (atomic_dec_and_test(&curr->refs))
1687 kfree(curr);
1688
1689 if (over)
1690 return 1;
1691 }
1692 return 0;
1693}
1694
1695BTRFS_SETGET_STACK_FUNCS(stack_inode_generation, struct btrfs_inode_item,
1696 generation, 64);
1697BTRFS_SETGET_STACK_FUNCS(stack_inode_sequence, struct btrfs_inode_item,
1698 sequence, 64);
1699BTRFS_SETGET_STACK_FUNCS(stack_inode_transid, struct btrfs_inode_item,
1700 transid, 64);
1701BTRFS_SETGET_STACK_FUNCS(stack_inode_size, struct btrfs_inode_item, size, 64);
1702BTRFS_SETGET_STACK_FUNCS(stack_inode_nbytes, struct btrfs_inode_item,
1703 nbytes, 64);
1704BTRFS_SETGET_STACK_FUNCS(stack_inode_block_group, struct btrfs_inode_item,
1705 block_group, 64);
1706BTRFS_SETGET_STACK_FUNCS(stack_inode_nlink, struct btrfs_inode_item, nlink, 32);
1707BTRFS_SETGET_STACK_FUNCS(stack_inode_uid, struct btrfs_inode_item, uid, 32);
1708BTRFS_SETGET_STACK_FUNCS(stack_inode_gid, struct btrfs_inode_item, gid, 32);
1709BTRFS_SETGET_STACK_FUNCS(stack_inode_mode, struct btrfs_inode_item, mode, 32);
1710BTRFS_SETGET_STACK_FUNCS(stack_inode_rdev, struct btrfs_inode_item, rdev, 64);
1711BTRFS_SETGET_STACK_FUNCS(stack_inode_flags, struct btrfs_inode_item, flags, 64);
1712
1713BTRFS_SETGET_STACK_FUNCS(stack_timespec_sec, struct btrfs_timespec, sec, 64);
1714BTRFS_SETGET_STACK_FUNCS(stack_timespec_nsec, struct btrfs_timespec, nsec, 32);
1715
1716static void fill_stack_inode_item(struct btrfs_trans_handle *trans,
1717 struct btrfs_inode_item *inode_item,
1718 struct inode *inode)
1719{
2f2f43d3
EB		 1720 btrfs_set_stack_inode_uid(inode_item, i_uid_read(inode));
1721 btrfs_set_stack_inode_gid(inode_item, i_gid_read(inode));
16cdcec7
MX		 1722 btrfs_set_stack_inode_size(inode_item, BTRFS_I(inode)->disk_i_size);
1723 btrfs_set_stack_inode_mode(inode_item, inode->i_mode);
1724 btrfs_set_stack_inode_nlink(inode_item, inode->i_nlink);
1725 btrfs_set_stack_inode_nbytes(inode_item, inode_get_bytes(inode));
1726 btrfs_set_stack_inode_generation(inode_item,
1727 BTRFS_I(inode)->generation);
0c4d2d95 1728 btrfs_set_stack_inode_sequence(inode_item, inode->i_version);
16cdcec7
MX		 1729 btrfs_set_stack_inode_transid(inode_item, trans->transid);
1730 btrfs_set_stack_inode_rdev(inode_item, inode->i_rdev);
1731 btrfs_set_stack_inode_flags(inode_item, BTRFS_I(inode)->flags);
ff5714cc 1732 btrfs_set_stack_inode_block_group(inode_item, 0);
16cdcec7
MX		 1733
1734 btrfs_set_stack_timespec_sec(btrfs_inode_atime(inode_item),
1735 inode->i_atime.tv_sec);
1736 btrfs_set_stack_timespec_nsec(btrfs_inode_atime(inode_item),
1737 inode->i_atime.tv_nsec);
1738
1739 btrfs_set_stack_timespec_sec(btrfs_inode_mtime(inode_item),
1740 inode->i_mtime.tv_sec);
1741 btrfs_set_stack_timespec_nsec(btrfs_inode_mtime(inode_item),
1742 inode->i_mtime.tv_nsec);
1743
1744 btrfs_set_stack_timespec_sec(btrfs_inode_ctime(inode_item),
1745 inode->i_ctime.tv_sec);
1746 btrfs_set_stack_timespec_nsec(btrfs_inode_ctime(inode_item),
1747 inode->i_ctime.tv_nsec);
1748}
1749
2f7e33d4
MX		 1750int btrfs_fill_inode(struct inode *inode, u32 *rdev)
1751{
1752 struct btrfs_delayed_node *delayed_node;
1753 struct btrfs_inode_item *inode_item;
1754 struct btrfs_timespec *tspec;
1755
1756 delayed_node = btrfs_get_delayed_node(inode);
1757 if (!delayed_node)
1758 return -ENOENT;
1759
1760 mutex_lock(&delayed_node->mutex);
1761 if (!delayed_node->inode_dirty) {
1762 mutex_unlock(&delayed_node->mutex);
1763 btrfs_release_delayed_node(delayed_node);
1764 return -ENOENT;
1765 }
1766
1767 inode_item = &delayed_node->inode_item;
1768
2f2f43d3
EB		 1769 i_uid_write(inode, btrfs_stack_inode_uid(inode_item));
1770 i_gid_write(inode, btrfs_stack_inode_gid(inode_item));
2f7e33d4
MX		 1771 btrfs_i_size_write(inode, btrfs_stack_inode_size(inode_item));
1772 inode->i_mode = btrfs_stack_inode_mode(inode_item);
bfe86848 1773 set_nlink(inode, btrfs_stack_inode_nlink(inode_item));
2f7e33d4
MX		 1774 inode_set_bytes(inode, btrfs_stack_inode_nbytes(inode_item));
1775 BTRFS_I(inode)->generation = btrfs_stack_inode_generation(inode_item);
0c4d2d95 1776 inode->i_version = btrfs_stack_inode_sequence(inode_item);
2f7e33d4
MX		 1777 inode->i_rdev = 0;
1778 *rdev = btrfs_stack_inode_rdev(inode_item);
1779 BTRFS_I(inode)->flags = btrfs_stack_inode_flags(inode_item);
1780
1781 tspec = btrfs_inode_atime(inode_item);
1782 inode->i_atime.tv_sec = btrfs_stack_timespec_sec(tspec);
1783 inode->i_atime.tv_nsec = btrfs_stack_timespec_nsec(tspec);
1784
1785 tspec = btrfs_inode_mtime(inode_item);
1786 inode->i_mtime.tv_sec = btrfs_stack_timespec_sec(tspec);
1787 inode->i_mtime.tv_nsec = btrfs_stack_timespec_nsec(tspec);
1788
1789 tspec = btrfs_inode_ctime(inode_item);
1790 inode->i_ctime.tv_sec = btrfs_stack_timespec_sec(tspec);
1791 inode->i_ctime.tv_nsec = btrfs_stack_timespec_nsec(tspec);
1792
1793 inode->i_generation = BTRFS_I(inode)->generation;
1794 BTRFS_I(inode)->index_cnt = (u64)-1;
1795
1796 mutex_unlock(&delayed_node->mutex);
1797 btrfs_release_delayed_node(delayed_node);
1798 return 0;
1799}
1800
16cdcec7
MX		 1801int btrfs_delayed_update_inode(struct btrfs_trans_handle *trans,
1802 struct btrfs_root *root, struct inode *inode)
1803{
1804 struct btrfs_delayed_node *delayed_node;
aa0467d8 1805 int ret = 0;
16cdcec7
MX		 1806
1807 delayed_node = btrfs_get_or_create_delayed_node(inode);
1808 if (IS_ERR(delayed_node))
1809 return PTR_ERR(delayed_node);
1810
1811 mutex_lock(&delayed_node->mutex);
1812 if (delayed_node->inode_dirty) {
1813 fill_stack_inode_item(trans, &delayed_node->inode_item, inode);
1814 goto release_node;
1815 }
1816
7fd2ae21
JB		 1817 ret = btrfs_delayed_inode_reserve_metadata(trans, root, inode,
1818 delayed_node);
c06a0e12
JB		 1819 if (ret)
1820 goto release_node;
16cdcec7
MX		 1821
1822 fill_stack_inode_item(trans, &delayed_node->inode_item, inode);
1823 delayed_node->inode_dirty = 1;
1824 delayed_node->count++;
1825 atomic_inc(&root->fs_info->delayed_root->items);
1826release_node:
1827 mutex_unlock(&delayed_node->mutex);
1828 btrfs_release_delayed_node(delayed_node);
1829 return ret;
1830}
1831
1832static void __btrfs_kill_delayed_node(struct btrfs_delayed_node *delayed_node)
1833{
1834 struct btrfs_root *root = delayed_node->root;
1835 struct btrfs_delayed_item *curr_item, *prev_item;
1836
1837 mutex_lock(&delayed_node->mutex);
1838 curr_item = __btrfs_first_delayed_insertion_item(delayed_node);
1839 while (curr_item) {
1840 btrfs_delayed_item_release_metadata(root, curr_item);
1841 prev_item = curr_item;
1842 curr_item = __btrfs_next_delayed_item(prev_item);
1843 btrfs_release_delayed_item(prev_item);
1844 }
1845
1846 curr_item = __btrfs_first_delayed_deletion_item(delayed_node);
1847 while (curr_item) {
1848 btrfs_delayed_item_release_metadata(root, curr_item);
1849 prev_item = curr_item;
1850 curr_item = __btrfs_next_delayed_item(prev_item);
1851 btrfs_release_delayed_item(prev_item);
1852 }
1853
1854 if (delayed_node->inode_dirty) {
1855 btrfs_delayed_inode_release_metadata(root, delayed_node);
1856 btrfs_release_delayed_inode(delayed_node);
1857 }
1858 mutex_unlock(&delayed_node->mutex);
1859}
1860
1861void btrfs_kill_delayed_inode_items(struct inode *inode)
1862{
1863 struct btrfs_delayed_node *delayed_node;
1864
1865 delayed_node = btrfs_get_delayed_node(inode);
1866 if (!delayed_node)
1867 return;
1868
1869 __btrfs_kill_delayed_node(delayed_node);
1870 btrfs_release_delayed_node(delayed_node);
1871}
1872
1873void btrfs_kill_all_delayed_nodes(struct btrfs_root *root)
1874{
1875 u64 inode_id = 0;
1876 struct btrfs_delayed_node *delayed_nodes[8];
1877 int i, n;
1878
1879 while (1) {
1880 spin_lock(&root->inode_lock);
1881 n = radix_tree_gang_lookup(&root->delayed_nodes_tree,
1882 (void **)delayed_nodes, inode_id,
1883 ARRAY_SIZE(delayed_nodes));
1884 if (!n) {
1885 spin_unlock(&root->inode_lock);
1886 break;
1887 }
1888
1889 inode_id = delayed_nodes[n - 1]->inode_id + 1;
1890
1891 for (i = 0; i < n; i++)
1892 atomic_inc(&delayed_nodes[i]->refs);
1893 spin_unlock(&root->inode_lock);
1894
1895 for (i = 0; i < n; i++) {
1896 __btrfs_kill_delayed_node(delayed_nodes[i]);
1897 btrfs_release_delayed_node(delayed_nodes[i]);
1898 }
1899 }
1900}
67cde344
MX		 1901
1902void btrfs_destroy_delayed_inodes(struct btrfs_root *root)
1903{
1904 struct btrfs_delayed_root *delayed_root;
1905 struct btrfs_delayed_node *curr_node, *prev_node;
1906
1907 delayed_root = btrfs_get_delayed_root(root);
1908
1909 curr_node = btrfs_first_delayed_node(delayed_root);
1910 while (curr_node) {
1911 __btrfs_kill_delayed_node(curr_node);
1912
1913 prev_node = curr_node;
1914 curr_node = btrfs_next_delayed_node(curr_node);
1915 btrfs_release_delayed_node(prev_node);
1916 }
1917}
1918
Linux 6.x block layer and io_uring tree(s)