1 #include <linux/bitops.h>
2 #include <linux/slab.h>
6 #include <linux/pagemap.h>
7 #include <linux/page-flags.h>
8 #include <linux/module.h>
9 #include <linux/spinlock.h>
10 #include <linux/blkdev.h>
11 #include <linux/swap.h>
12 #include <linux/version.h>
13 #include <linux/writeback.h>
14 #include <linux/pagevec.h>
15 #include "extent_io.h"
16 #include "extent_map.h"
18 /* temporary define until extent_map moves out of btrfs */
19 struct kmem_cache *btrfs_cache_create(const char *name, size_t size,
20 unsigned long extra_flags,
21 void (*ctor)(void *, struct kmem_cache *,
24 static struct kmem_cache *extent_state_cache;
25 static struct kmem_cache *extent_buffer_cache;
27 static LIST_HEAD(buffers);
28 static LIST_HEAD(states);
29 static spinlock_t leak_lock = SPIN_LOCK_UNLOCKED;
31 #define BUFFER_LRU_MAX 64
36 struct rb_node rb_node;
39 struct extent_page_data {
41 struct extent_io_tree *tree;
42 get_extent_t *get_extent;
45 int __init extent_io_init(void)
47 extent_state_cache = btrfs_cache_create("extent_state",
48 sizeof(struct extent_state), 0,
50 if (!extent_state_cache)
53 extent_buffer_cache = btrfs_cache_create("extent_buffers",
54 sizeof(struct extent_buffer), 0,
56 if (!extent_buffer_cache)
57 goto free_state_cache;
61 kmem_cache_destroy(extent_state_cache);
65 void extent_io_exit(void)
67 struct extent_state *state;
68 struct extent_buffer *eb;
70 while (!list_empty(&states)) {
71 state = list_entry(states.next, struct extent_state, leak_list);
72 printk("state leak: start %Lu end %Lu state %lu in tree %p refs %d\n", state->start, state->end, state->state, state->tree, atomic_read(&state->refs));
73 list_del(&state->leak_list);
74 kmem_cache_free(extent_state_cache, state);
78 while (!list_empty(&buffers)) {
79 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
80 printk("buffer leak start %Lu len %lu refs %d\n", eb->start, eb->len, atomic_read(&eb->refs));
81 list_del(&eb->leak_list);
82 kmem_cache_free(extent_buffer_cache, eb);
84 if (extent_state_cache)
85 kmem_cache_destroy(extent_state_cache);
86 if (extent_buffer_cache)
87 kmem_cache_destroy(extent_buffer_cache);
90 void extent_io_tree_init(struct extent_io_tree *tree,
91 struct address_space *mapping, gfp_t mask)
93 tree->state.rb_node = NULL;
95 tree->dirty_bytes = 0;
96 spin_lock_init(&tree->lock);
97 spin_lock_init(&tree->lru_lock);
98 tree->mapping = mapping;
99 INIT_LIST_HEAD(&tree->buffer_lru);
103 EXPORT_SYMBOL(extent_io_tree_init);
105 void extent_io_tree_empty_lru(struct extent_io_tree *tree)
107 struct extent_buffer *eb;
108 while(!list_empty(&tree->buffer_lru)) {
109 eb = list_entry(tree->buffer_lru.next, struct extent_buffer,
111 list_del_init(&eb->lru);
112 free_extent_buffer(eb);
115 EXPORT_SYMBOL(extent_io_tree_empty_lru);
117 struct extent_state *alloc_extent_state(gfp_t mask)
119 struct extent_state *state;
122 state = kmem_cache_alloc(extent_state_cache, mask);
128 spin_lock_irqsave(&leak_lock, flags);
129 list_add(&state->leak_list, &states);
130 spin_unlock_irqrestore(&leak_lock, flags);
132 atomic_set(&state->refs, 1);
133 init_waitqueue_head(&state->wq);
136 EXPORT_SYMBOL(alloc_extent_state);
138 void free_extent_state(struct extent_state *state)
142 if (atomic_dec_and_test(&state->refs)) {
144 WARN_ON(state->tree);
145 spin_lock_irqsave(&leak_lock, flags);
146 list_del(&state->leak_list);
147 spin_unlock_irqrestore(&leak_lock, flags);
148 kmem_cache_free(extent_state_cache, state);
151 EXPORT_SYMBOL(free_extent_state);
153 static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
154 struct rb_node *node)
156 struct rb_node ** p = &root->rb_node;
157 struct rb_node * parent = NULL;
158 struct tree_entry *entry;
162 entry = rb_entry(parent, struct tree_entry, rb_node);
164 if (offset < entry->start)
166 else if (offset > entry->end)
172 entry = rb_entry(node, struct tree_entry, rb_node);
173 rb_link_node(node, parent, p);
174 rb_insert_color(node, root);
178 static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
179 struct rb_node **prev_ret,
180 struct rb_node **next_ret)
182 struct rb_root *root = &tree->state;
183 struct rb_node * n = root->rb_node;
184 struct rb_node *prev = NULL;
185 struct rb_node *orig_prev = NULL;
186 struct tree_entry *entry;
187 struct tree_entry *prev_entry = NULL;
190 struct extent_state *state;
192 if (state->start <= offset && offset <= state->end)
193 return &tree->last->rb_node;
196 entry = rb_entry(n, struct tree_entry, rb_node);
200 if (offset < entry->start)
202 else if (offset > entry->end)
205 tree->last = rb_entry(n, struct extent_state, rb_node);
212 while(prev && offset > prev_entry->end) {
213 prev = rb_next(prev);
214 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
221 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
222 while(prev && offset < prev_entry->start) {
223 prev = rb_prev(prev);
224 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
231 static inline struct rb_node *tree_search(struct extent_io_tree *tree,
234 struct rb_node *prev = NULL;
237 ret = __etree_search(tree, offset, &prev, NULL);
240 tree->last = rb_entry(prev, struct extent_state,
249 * utility function to look for merge candidates inside a given range.
250 * Any extents with matching state are merged together into a single
251 * extent in the tree. Extents with EXTENT_IO in their state field
252 * are not merged because the end_io handlers need to be able to do
253 * operations on them without sleeping (or doing allocations/splits).
255 * This should be called with the tree lock held.
257 static int merge_state(struct extent_io_tree *tree,
258 struct extent_state *state)
260 struct extent_state *other;
261 struct rb_node *other_node;
263 if (state->state & EXTENT_IOBITS)
266 other_node = rb_prev(&state->rb_node);
268 other = rb_entry(other_node, struct extent_state, rb_node);
269 if (other->end == state->start - 1 &&
270 other->state == state->state) {
271 state->start = other->start;
273 if (tree->last == other)
275 rb_erase(&other->rb_node, &tree->state);
276 free_extent_state(other);
279 other_node = rb_next(&state->rb_node);
281 other = rb_entry(other_node, struct extent_state, rb_node);
282 if (other->start == state->end + 1 &&
283 other->state == state->state) {
284 other->start = state->start;
286 if (tree->last == state)
288 rb_erase(&state->rb_node, &tree->state);
289 free_extent_state(state);
295 static void set_state_cb(struct extent_io_tree *tree,
296 struct extent_state *state,
299 if (tree->ops && tree->ops->set_bit_hook) {
300 tree->ops->set_bit_hook(tree->mapping->host, state->start,
301 state->end, state->state, bits);
305 static void clear_state_cb(struct extent_io_tree *tree,
306 struct extent_state *state,
309 if (tree->ops && tree->ops->set_bit_hook) {
310 tree->ops->clear_bit_hook(tree->mapping->host, state->start,
311 state->end, state->state, bits);
316 * insert an extent_state struct into the tree. 'bits' are set on the
317 * struct before it is inserted.
319 * This may return -EEXIST if the extent is already there, in which case the
320 * state struct is freed.
322 * The tree lock is not taken internally. This is a utility function and
323 * probably isn't what you want to call (see set/clear_extent_bit).
325 static int insert_state(struct extent_io_tree *tree,
326 struct extent_state *state, u64 start, u64 end,
329 struct rb_node *node;
332 printk("end < start %Lu %Lu\n", end, start);
335 if (bits & EXTENT_DIRTY)
336 tree->dirty_bytes += end - start + 1;
337 set_state_cb(tree, state, bits);
338 state->state |= bits;
339 state->start = start;
341 node = tree_insert(&tree->state, end, &state->rb_node);
343 struct extent_state *found;
344 found = rb_entry(node, struct extent_state, rb_node);
345 printk("found node %Lu %Lu on insert of %Lu %Lu\n", found->start, found->end, start, end);
346 free_extent_state(state);
351 merge_state(tree, state);
356 * split a given extent state struct in two, inserting the preallocated
357 * struct 'prealloc' as the newly created second half. 'split' indicates an
358 * offset inside 'orig' where it should be split.
361 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
362 * are two extent state structs in the tree:
363 * prealloc: [orig->start, split - 1]
364 * orig: [ split, orig->end ]
366 * The tree locks are not taken by this function. They need to be held
369 static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
370 struct extent_state *prealloc, u64 split)
372 struct rb_node *node;
373 prealloc->start = orig->start;
374 prealloc->end = split - 1;
375 prealloc->state = orig->state;
378 node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node);
380 struct extent_state *found;
381 found = rb_entry(node, struct extent_state, rb_node);
382 printk("found node %Lu %Lu on insert of %Lu %Lu\n", found->start, found->end, prealloc->start, prealloc->end);
383 free_extent_state(prealloc);
386 prealloc->tree = tree;
391 * utility function to clear some bits in an extent state struct.
392 * it will optionally wake up any one waiting on this state (wake == 1), or
393 * forcibly remove the state from the tree (delete == 1).
395 * If no bits are set on the state struct after clearing things, the
396 * struct is freed and removed from the tree
398 static int clear_state_bit(struct extent_io_tree *tree,
399 struct extent_state *state, int bits, int wake,
402 int ret = state->state & bits;
404 if ((bits & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
405 u64 range = state->end - state->start + 1;
406 WARN_ON(range > tree->dirty_bytes);
407 tree->dirty_bytes -= range;
409 clear_state_cb(tree, state, bits);
410 state->state &= ~bits;
413 if (delete || state->state == 0) {
415 clear_state_cb(tree, state, state->state);
416 if (tree->last == state) {
417 tree->last = extent_state_next(state);
419 rb_erase(&state->rb_node, &tree->state);
421 free_extent_state(state);
426 merge_state(tree, state);
432 * clear some bits on a range in the tree. This may require splitting
433 * or inserting elements in the tree, so the gfp mask is used to
434 * indicate which allocations or sleeping are allowed.
436 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
437 * the given range from the tree regardless of state (ie for truncate).
439 * the range [start, end] is inclusive.
441 * This takes the tree lock, and returns < 0 on error, > 0 if any of the
442 * bits were already set, or zero if none of the bits were already set.
444 int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
445 int bits, int wake, int delete, gfp_t mask)
447 struct extent_state *state;
448 struct extent_state *prealloc = NULL;
449 struct rb_node *node;
455 if (!prealloc && (mask & __GFP_WAIT)) {
456 prealloc = alloc_extent_state(mask);
461 spin_lock_irqsave(&tree->lock, flags);
463 * this search will find the extents that end after
466 node = tree_search(tree, start);
469 state = rb_entry(node, struct extent_state, rb_node);
470 if (state->start > end)
472 WARN_ON(state->end < start);
475 * | ---- desired range ---- |
477 * | ------------- state -------------- |
479 * We need to split the extent we found, and may flip
480 * bits on second half.
482 * If the extent we found extends past our range, we
483 * just split and search again. It'll get split again
484 * the next time though.
486 * If the extent we found is inside our range, we clear
487 * the desired bit on it.
490 if (state->start < start) {
492 prealloc = alloc_extent_state(GFP_ATOMIC);
493 err = split_state(tree, state, prealloc, start);
494 BUG_ON(err == -EEXIST);
498 if (state->end <= end) {
499 start = state->end + 1;
500 set |= clear_state_bit(tree, state, bits,
503 start = state->start;
508 * | ---- desired range ---- |
510 * We need to split the extent, and clear the bit
513 if (state->start <= end && state->end > end) {
515 prealloc = alloc_extent_state(GFP_ATOMIC);
516 err = split_state(tree, state, prealloc, end + 1);
517 BUG_ON(err == -EEXIST);
521 set |= clear_state_bit(tree, prealloc, bits,
527 start = state->end + 1;
528 set |= clear_state_bit(tree, state, bits, wake, delete);
532 spin_unlock_irqrestore(&tree->lock, flags);
534 free_extent_state(prealloc);
541 spin_unlock_irqrestore(&tree->lock, flags);
542 if (mask & __GFP_WAIT)
546 EXPORT_SYMBOL(clear_extent_bit);
548 static int wait_on_state(struct extent_io_tree *tree,
549 struct extent_state *state)
552 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
553 spin_unlock_irq(&tree->lock);
555 spin_lock_irq(&tree->lock);
556 finish_wait(&state->wq, &wait);
561 * waits for one or more bits to clear on a range in the state tree.
562 * The range [start, end] is inclusive.
563 * The tree lock is taken by this function
565 int wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits)
567 struct extent_state *state;
568 struct rb_node *node;
570 spin_lock_irq(&tree->lock);
574 * this search will find all the extents that end after
577 node = tree_search(tree, start);
581 state = rb_entry(node, struct extent_state, rb_node);
583 if (state->start > end)
586 if (state->state & bits) {
587 start = state->start;
588 atomic_inc(&state->refs);
589 wait_on_state(tree, state);
590 free_extent_state(state);
593 start = state->end + 1;
598 if (need_resched()) {
599 spin_unlock_irq(&tree->lock);
601 spin_lock_irq(&tree->lock);
605 spin_unlock_irq(&tree->lock);
608 EXPORT_SYMBOL(wait_extent_bit);
610 static void set_state_bits(struct extent_io_tree *tree,
611 struct extent_state *state,
614 if ((bits & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
615 u64 range = state->end - state->start + 1;
616 tree->dirty_bytes += range;
618 set_state_cb(tree, state, bits);
619 state->state |= bits;
623 * set some bits on a range in the tree. This may require allocations
624 * or sleeping, so the gfp mask is used to indicate what is allowed.
626 * If 'exclusive' == 1, this will fail with -EEXIST if some part of the
627 * range already has the desired bits set. The start of the existing
628 * range is returned in failed_start in this case.
630 * [start, end] is inclusive
631 * This takes the tree lock.
633 int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits,
634 int exclusive, u64 *failed_start, gfp_t mask)
636 struct extent_state *state;
637 struct extent_state *prealloc = NULL;
638 struct rb_node *node;
645 if (!prealloc && (mask & __GFP_WAIT)) {
646 prealloc = alloc_extent_state(mask);
651 spin_lock_irqsave(&tree->lock, flags);
653 * this search will find all the extents that end after
656 node = tree_search(tree, start);
658 err = insert_state(tree, prealloc, start, end, bits);
660 BUG_ON(err == -EEXIST);
664 state = rb_entry(node, struct extent_state, rb_node);
665 last_start = state->start;
666 last_end = state->end;
669 * | ---- desired range ---- |
672 * Just lock what we found and keep going
674 if (state->start == start && state->end <= end) {
675 set = state->state & bits;
676 if (set && exclusive) {
677 *failed_start = state->start;
681 set_state_bits(tree, state, bits);
682 start = state->end + 1;
683 merge_state(tree, state);
688 * | ---- desired range ---- |
691 * | ------------- state -------------- |
693 * We need to split the extent we found, and may flip bits on
696 * If the extent we found extends past our
697 * range, we just split and search again. It'll get split
698 * again the next time though.
700 * If the extent we found is inside our range, we set the
703 if (state->start < start) {
704 set = state->state & bits;
705 if (exclusive && set) {
706 *failed_start = start;
710 err = split_state(tree, state, prealloc, start);
711 BUG_ON(err == -EEXIST);
715 if (state->end <= end) {
716 set_state_bits(tree, state, bits);
717 start = state->end + 1;
718 merge_state(tree, state);
720 start = state->start;
725 * | ---- desired range ---- |
726 * | state | or | state |
728 * There's a hole, we need to insert something in it and
729 * ignore the extent we found.
731 if (state->start > start) {
733 if (end < last_start)
736 this_end = last_start -1;
737 err = insert_state(tree, prealloc, start, this_end,
740 BUG_ON(err == -EEXIST);
743 start = this_end + 1;
747 * | ---- desired range ---- |
749 * We need to split the extent, and set the bit
752 if (state->start <= end && state->end > end) {
753 set = state->state & bits;
754 if (exclusive && set) {
755 *failed_start = start;
759 err = split_state(tree, state, prealloc, end + 1);
760 BUG_ON(err == -EEXIST);
762 set_state_bits(tree, prealloc, bits);
763 merge_state(tree, prealloc);
771 spin_unlock_irqrestore(&tree->lock, flags);
773 free_extent_state(prealloc);
780 spin_unlock_irqrestore(&tree->lock, flags);
781 if (mask & __GFP_WAIT)
785 EXPORT_SYMBOL(set_extent_bit);
787 /* wrappers around set/clear extent bit */
788 int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
791 return set_extent_bit(tree, start, end, EXTENT_DIRTY, 0, NULL,
794 EXPORT_SYMBOL(set_extent_dirty);
796 int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
797 int bits, gfp_t mask)
799 return set_extent_bit(tree, start, end, bits, 0, NULL,
802 EXPORT_SYMBOL(set_extent_bits);
804 int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
805 int bits, gfp_t mask)
807 return clear_extent_bit(tree, start, end, bits, 0, 0, mask);
809 EXPORT_SYMBOL(clear_extent_bits);
811 int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
814 return set_extent_bit(tree, start, end,
815 EXTENT_DELALLOC | EXTENT_DIRTY, 0, NULL,
818 EXPORT_SYMBOL(set_extent_delalloc);
820 int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
823 return clear_extent_bit(tree, start, end,
824 EXTENT_DIRTY | EXTENT_DELALLOC, 0, 0, mask);
826 EXPORT_SYMBOL(clear_extent_dirty);
828 int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
831 return set_extent_bit(tree, start, end, EXTENT_NEW, 0, NULL,
834 EXPORT_SYMBOL(set_extent_new);
836 int clear_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
839 return clear_extent_bit(tree, start, end, EXTENT_NEW, 0, 0, mask);
841 EXPORT_SYMBOL(clear_extent_new);
843 int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
846 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, NULL,
849 EXPORT_SYMBOL(set_extent_uptodate);
851 int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
854 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0, mask);
856 EXPORT_SYMBOL(clear_extent_uptodate);
858 int set_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end,
861 return set_extent_bit(tree, start, end, EXTENT_WRITEBACK,
864 EXPORT_SYMBOL(set_extent_writeback);
866 int clear_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end,
869 return clear_extent_bit(tree, start, end, EXTENT_WRITEBACK, 1, 0, mask);
871 EXPORT_SYMBOL(clear_extent_writeback);
873 int wait_on_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end)
875 return wait_extent_bit(tree, start, end, EXTENT_WRITEBACK);
877 EXPORT_SYMBOL(wait_on_extent_writeback);
879 int lock_extent(struct extent_io_tree *tree, u64 start, u64 end, gfp_t mask)
884 err = set_extent_bit(tree, start, end, EXTENT_LOCKED, 1,
885 &failed_start, mask);
886 if (err == -EEXIST && (mask & __GFP_WAIT)) {
887 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
888 start = failed_start;
892 WARN_ON(start > end);
896 EXPORT_SYMBOL(lock_extent);
898 int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end,
901 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, mask);
903 EXPORT_SYMBOL(unlock_extent);
906 * helper function to set pages and extents in the tree dirty
908 int set_range_dirty(struct extent_io_tree *tree, u64 start, u64 end)
910 unsigned long index = start >> PAGE_CACHE_SHIFT;
911 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
914 while (index <= end_index) {
915 page = find_get_page(tree->mapping, index);
917 __set_page_dirty_nobuffers(page);
918 page_cache_release(page);
921 set_extent_dirty(tree, start, end, GFP_NOFS);
924 EXPORT_SYMBOL(set_range_dirty);
927 * helper function to set both pages and extents in the tree writeback
929 int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
931 unsigned long index = start >> PAGE_CACHE_SHIFT;
932 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
935 while (index <= end_index) {
936 page = find_get_page(tree->mapping, index);
938 set_page_writeback(page);
939 page_cache_release(page);
942 set_extent_writeback(tree, start, end, GFP_NOFS);
945 EXPORT_SYMBOL(set_range_writeback);
947 int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
948 u64 *start_ret, u64 *end_ret, int bits)
950 struct rb_node *node;
951 struct extent_state *state;
954 spin_lock_irq(&tree->lock);
956 * this search will find all the extents that end after
959 node = tree_search(tree, start);
965 state = rb_entry(node, struct extent_state, rb_node);
966 if (state->end >= start && (state->state & bits)) {
967 *start_ret = state->start;
968 *end_ret = state->end;
972 node = rb_next(node);
977 spin_unlock_irq(&tree->lock);
980 EXPORT_SYMBOL(find_first_extent_bit);
982 struct extent_state *find_first_extent_bit_state(struct extent_io_tree *tree,
985 struct rb_node *node;
986 struct extent_state *state;
989 * this search will find all the extents that end after
992 node = tree_search(tree, start);
998 state = rb_entry(node, struct extent_state, rb_node);
999 if (state->end >= start && (state->state & bits)) {
1002 node = rb_next(node);
1009 EXPORT_SYMBOL(find_first_extent_bit_state);
1011 u64 find_lock_delalloc_range(struct extent_io_tree *tree,
1012 u64 *start, u64 *end, u64 max_bytes)
1014 struct rb_node *node;
1015 struct extent_state *state;
1016 u64 cur_start = *start;
1018 u64 total_bytes = 0;
1020 spin_lock_irq(&tree->lock);
1022 * this search will find all the extents that end after
1026 node = tree_search(tree, cur_start);
1033 state = rb_entry(node, struct extent_state, rb_node);
1034 if (found && state->start != cur_start) {
1037 if (!(state->state & EXTENT_DELALLOC)) {
1043 struct extent_state *prev_state;
1044 struct rb_node *prev_node = node;
1046 prev_node = rb_prev(prev_node);
1049 prev_state = rb_entry(prev_node,
1050 struct extent_state,
1052 if (!(prev_state->state & EXTENT_DELALLOC))
1058 if (state->state & EXTENT_LOCKED) {
1060 atomic_inc(&state->refs);
1061 prepare_to_wait(&state->wq, &wait,
1062 TASK_UNINTERRUPTIBLE);
1063 spin_unlock_irq(&tree->lock);
1065 spin_lock_irq(&tree->lock);
1066 finish_wait(&state->wq, &wait);
1067 free_extent_state(state);
1070 set_state_cb(tree, state, EXTENT_LOCKED);
1071 state->state |= EXTENT_LOCKED;
1073 *start = state->start;
1076 cur_start = state->end + 1;
1077 node = rb_next(node);
1080 total_bytes += state->end - state->start + 1;
1081 if (total_bytes >= max_bytes)
1085 spin_unlock_irq(&tree->lock);
1089 u64 count_range_bits(struct extent_io_tree *tree,
1090 u64 *start, u64 search_end, u64 max_bytes,
1093 struct rb_node *node;
1094 struct extent_state *state;
1095 u64 cur_start = *start;
1096 u64 total_bytes = 0;
1099 if (search_end <= cur_start) {
1100 printk("search_end %Lu start %Lu\n", search_end, cur_start);
1105 spin_lock_irq(&tree->lock);
1106 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1107 total_bytes = tree->dirty_bytes;
1111 * this search will find all the extents that end after
1114 node = tree_search(tree, cur_start);
1120 state = rb_entry(node, struct extent_state, rb_node);
1121 if (state->start > search_end)
1123 if (state->end >= cur_start && (state->state & bits)) {
1124 total_bytes += min(search_end, state->end) + 1 -
1125 max(cur_start, state->start);
1126 if (total_bytes >= max_bytes)
1129 *start = state->start;
1133 node = rb_next(node);
1138 spin_unlock_irq(&tree->lock);
1142 * helper function to lock both pages and extents in the tree.
1143 * pages must be locked first.
1145 int lock_range(struct extent_io_tree *tree, u64 start, u64 end)
1147 unsigned long index = start >> PAGE_CACHE_SHIFT;
1148 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1152 while (index <= end_index) {
1153 page = grab_cache_page(tree->mapping, index);
1159 err = PTR_ERR(page);
1164 lock_extent(tree, start, end, GFP_NOFS);
1169 * we failed above in getting the page at 'index', so we undo here
1170 * up to but not including the page at 'index'
1173 index = start >> PAGE_CACHE_SHIFT;
1174 while (index < end_index) {
1175 page = find_get_page(tree->mapping, index);
1177 page_cache_release(page);
1182 EXPORT_SYMBOL(lock_range);
1185 * helper function to unlock both pages and extents in the tree.
1187 int unlock_range(struct extent_io_tree *tree, u64 start, u64 end)
1189 unsigned long index = start >> PAGE_CACHE_SHIFT;
1190 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1193 while (index <= end_index) {
1194 page = find_get_page(tree->mapping, index);
1196 page_cache_release(page);
1199 unlock_extent(tree, start, end, GFP_NOFS);
1202 EXPORT_SYMBOL(unlock_range);
1204 int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
1206 struct rb_node *node;
1207 struct extent_state *state;
1210 spin_lock_irq(&tree->lock);
1212 * this search will find all the extents that end after
1215 node = tree_search(tree, start);
1220 state = rb_entry(node, struct extent_state, rb_node);
1221 if (state->start != start) {
1225 state->private = private;
1227 spin_unlock_irq(&tree->lock);
1231 int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
1233 struct rb_node *node;
1234 struct extent_state *state;
1237 spin_lock_irq(&tree->lock);
1239 * this search will find all the extents that end after
1242 node = tree_search(tree, start);
1247 state = rb_entry(node, struct extent_state, rb_node);
1248 if (state->start != start) {
1252 *private = state->private;
1254 spin_unlock_irq(&tree->lock);
1259 * searches a range in the state tree for a given mask.
1260 * If 'filled' == 1, this returns 1 only if every extent in the tree
1261 * has the bits set. Otherwise, 1 is returned if any bit in the
1262 * range is found set.
1264 int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
1265 int bits, int filled)
1267 struct extent_state *state = NULL;
1268 struct rb_node *node;
1270 unsigned long flags;
1272 spin_lock_irqsave(&tree->lock, flags);
1273 node = tree_search(tree, start);
1274 while (node && start <= end) {
1275 state = rb_entry(node, struct extent_state, rb_node);
1277 if (filled && state->start > start) {
1282 if (state->start > end)
1285 if (state->state & bits) {
1289 } else if (filled) {
1293 start = state->end + 1;
1296 node = rb_next(node);
1303 spin_unlock_irqrestore(&tree->lock, flags);
1306 EXPORT_SYMBOL(test_range_bit);
1309 * helper function to set a given page up to date if all the
1310 * extents in the tree for that page are up to date
1312 static int check_page_uptodate(struct extent_io_tree *tree,
1315 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1316 u64 end = start + PAGE_CACHE_SIZE - 1;
1317 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1))
1318 SetPageUptodate(page);
1323 * helper function to unlock a page if all the extents in the tree
1324 * for that page are unlocked
1326 static int check_page_locked(struct extent_io_tree *tree,
1329 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1330 u64 end = start + PAGE_CACHE_SIZE - 1;
1331 if (!test_range_bit(tree, start, end, EXTENT_LOCKED, 0))
1337 * helper function to end page writeback if all the extents
1338 * in the tree for that page are done with writeback
1340 static int check_page_writeback(struct extent_io_tree *tree,
1343 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1344 u64 end = start + PAGE_CACHE_SIZE - 1;
1345 if (!test_range_bit(tree, start, end, EXTENT_WRITEBACK, 0))
1346 end_page_writeback(page);
1350 /* lots and lots of room for performance fixes in the end_bio funcs */
1353 * after a writepage IO is done, we need to:
1354 * clear the uptodate bits on error
1355 * clear the writeback bits in the extent tree for this IO
1356 * end_page_writeback if the page has no more pending IO
1358 * Scheduling is not allowed, so the extent state tree is expected
1359 * to have one and only one object corresponding to this IO.
1361 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
1362 static void end_bio_extent_writepage(struct bio *bio, int err)
1364 static int end_bio_extent_writepage(struct bio *bio,
1365 unsigned int bytes_done, int err)
1368 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1369 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1370 struct extent_state *state = bio->bi_private;
1371 struct extent_io_tree *tree = state->tree;
1372 struct rb_node *node;
1377 unsigned long flags;
1379 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
1384 struct page *page = bvec->bv_page;
1385 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
1387 end = start + bvec->bv_len - 1;
1389 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
1394 if (--bvec >= bio->bi_io_vec)
1395 prefetchw(&bvec->bv_page->flags);
1398 clear_extent_uptodate(tree, start, end, GFP_ATOMIC);
1399 ClearPageUptodate(page);
1403 if (tree->ops && tree->ops->writepage_end_io_hook) {
1404 tree->ops->writepage_end_io_hook(page, start, end,
1409 * bios can get merged in funny ways, and so we need to
1410 * be careful with the state variable. We know the
1411 * state won't be merged with others because it has
1412 * WRITEBACK set, but we can't be sure each biovec is
1413 * sequential in the file. So, if our cached state
1414 * doesn't match the expected end, search the tree
1415 * for the correct one.
1418 spin_lock_irqsave(&tree->lock, flags);
1419 if (!state || state->end != end) {
1421 node = __etree_search(tree, start, NULL, NULL);
1423 state = rb_entry(node, struct extent_state,
1425 if (state->end != end ||
1426 !(state->state & EXTENT_WRITEBACK))
1430 spin_unlock_irqrestore(&tree->lock, flags);
1431 clear_extent_writeback(tree, start,
1438 struct extent_state *clear = state;
1440 node = rb_prev(&state->rb_node);
1442 state = rb_entry(node,
1443 struct extent_state,
1449 clear_state_bit(tree, clear, EXTENT_WRITEBACK,
1460 /* before releasing the lock, make sure the next state
1461 * variable has the expected bits set and corresponds
1462 * to the correct offsets in the file
1464 if (state && (state->end + 1 != start ||
1465 !(state->state & EXTENT_WRITEBACK))) {
1468 spin_unlock_irqrestore(&tree->lock, flags);
1472 end_page_writeback(page);
1474 check_page_writeback(tree, page);
1475 } while (bvec >= bio->bi_io_vec);
1477 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
1483 * after a readpage IO is done, we need to:
1484 * clear the uptodate bits on error
1485 * set the uptodate bits if things worked
1486 * set the page up to date if all extents in the tree are uptodate
1487 * clear the lock bit in the extent tree
1488 * unlock the page if there are no other extents locked for it
1490 * Scheduling is not allowed, so the extent state tree is expected
1491 * to have one and only one object corresponding to this IO.
1493 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
1494 static void end_bio_extent_readpage(struct bio *bio, int err)
1496 static int end_bio_extent_readpage(struct bio *bio,
1497 unsigned int bytes_done, int err)
1500 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1501 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1502 struct extent_state *state = bio->bi_private;
1503 struct extent_io_tree *tree = state->tree;
1504 struct rb_node *node;
1508 unsigned long flags;
1512 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
1518 struct page *page = bvec->bv_page;
1519 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
1521 end = start + bvec->bv_len - 1;
1523 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
1528 if (--bvec >= bio->bi_io_vec)
1529 prefetchw(&bvec->bv_page->flags);
1531 if (uptodate && tree->ops && tree->ops->readpage_end_io_hook) {
1532 ret = tree->ops->readpage_end_io_hook(page, start, end,
1538 spin_lock_irqsave(&tree->lock, flags);
1539 if (!state || state->end != end) {
1541 node = __etree_search(tree, start, NULL, NULL);
1543 state = rb_entry(node, struct extent_state,
1545 if (state->end != end ||
1546 !(state->state & EXTENT_LOCKED))
1550 spin_unlock_irqrestore(&tree->lock, flags);
1551 set_extent_uptodate(tree, start, end,
1553 unlock_extent(tree, start, end, GFP_ATOMIC);
1560 struct extent_state *clear = state;
1562 node = rb_prev(&state->rb_node);
1564 state = rb_entry(node,
1565 struct extent_state,
1570 set_state_cb(tree, clear, EXTENT_UPTODATE);
1571 clear->state |= EXTENT_UPTODATE;
1572 clear_state_bit(tree, clear, EXTENT_LOCKED,
1583 /* before releasing the lock, make sure the next state
1584 * variable has the expected bits set and corresponds
1585 * to the correct offsets in the file
1587 if (state && (state->end + 1 != start ||
1588 !(state->state & EXTENT_LOCKED))) {
1591 spin_unlock_irqrestore(&tree->lock, flags);
1595 SetPageUptodate(page);
1597 ClearPageUptodate(page);
1603 check_page_uptodate(tree, page);
1605 ClearPageUptodate(page);
1608 check_page_locked(tree, page);
1610 } while (bvec >= bio->bi_io_vec);
1613 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
1619 * IO done from prepare_write is pretty simple, we just unlock
1620 * the structs in the extent tree when done, and set the uptodate bits
1623 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
1624 static void end_bio_extent_preparewrite(struct bio *bio, int err)
1626 static int end_bio_extent_preparewrite(struct bio *bio,
1627 unsigned int bytes_done, int err)
1630 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1631 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1632 struct extent_state *state = bio->bi_private;
1633 struct extent_io_tree *tree = state->tree;
1637 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
1643 struct page *page = bvec->bv_page;
1644 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
1646 end = start + bvec->bv_len - 1;
1648 if (--bvec >= bio->bi_io_vec)
1649 prefetchw(&bvec->bv_page->flags);
1652 set_extent_uptodate(tree, start, end, GFP_ATOMIC);
1654 ClearPageUptodate(page);
1658 unlock_extent(tree, start, end, GFP_ATOMIC);
1660 } while (bvec >= bio->bi_io_vec);
1663 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
1669 extent_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
1674 bio = bio_alloc(gfp_flags, nr_vecs);
1676 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
1677 while (!bio && (nr_vecs /= 2))
1678 bio = bio_alloc(gfp_flags, nr_vecs);
1682 bio->bi_bdev = bdev;
1683 bio->bi_sector = first_sector;
1688 static int submit_one_bio(int rw, struct bio *bio)
1692 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1693 struct page *page = bvec->bv_page;
1694 struct extent_io_tree *tree = bio->bi_private;
1695 struct rb_node *node;
1696 struct extent_state *state;
1700 start = ((u64)page->index << PAGE_CACHE_SHIFT) + bvec->bv_offset;
1701 end = start + bvec->bv_len - 1;
1703 spin_lock_irq(&tree->lock);
1704 node = __etree_search(tree, start, NULL, NULL);
1706 state = rb_entry(node, struct extent_state, rb_node);
1707 while(state->end < end) {
1708 node = rb_next(node);
1709 state = rb_entry(node, struct extent_state, rb_node);
1711 BUG_ON(state->end != end);
1712 spin_unlock_irq(&tree->lock);
1714 bio->bi_private = state;
1718 maxsector = bio->bi_bdev->bd_inode->i_size >> 9;
1719 if (maxsector < bio->bi_sector) {
1720 printk("sector too large max %Lu got %llu\n", maxsector,
1721 (unsigned long long)bio->bi_sector);
1724 if (tree->ops && tree->ops->submit_bio_hook)
1725 tree->ops->submit_bio_hook(page->mapping->host, rw, bio);
1727 submit_bio(rw, bio);
1728 if (bio_flagged(bio, BIO_EOPNOTSUPP))
1734 static int submit_extent_page(int rw, struct extent_io_tree *tree,
1735 struct page *page, sector_t sector,
1736 size_t size, unsigned long offset,
1737 struct block_device *bdev,
1738 struct bio **bio_ret,
1739 unsigned long max_pages,
1740 bio_end_io_t end_io_func)
1746 if (bio_ret && *bio_ret) {
1748 if (bio->bi_sector + (bio->bi_size >> 9) != sector ||
1749 (tree->ops && tree->ops->merge_bio_hook &&
1750 tree->ops->merge_bio_hook(page, offset, size, bio)) ||
1751 bio_add_page(bio, page, size, offset) < size) {
1752 ret = submit_one_bio(rw, bio);
1758 nr = bio_get_nr_vecs(bdev);
1759 bio = extent_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
1761 printk("failed to allocate bio nr %d\n", nr);
1765 bio_add_page(bio, page, size, offset);
1766 bio->bi_end_io = end_io_func;
1767 bio->bi_private = tree;
1772 ret = submit_one_bio(rw, bio);
1778 void set_page_extent_mapped(struct page *page)
1780 if (!PagePrivate(page)) {
1781 SetPagePrivate(page);
1782 WARN_ON(!page->mapping->a_ops->invalidatepage);
1783 set_page_private(page, EXTENT_PAGE_PRIVATE);
1784 page_cache_get(page);
1788 void set_page_extent_head(struct page *page, unsigned long len)
1790 set_page_private(page, EXTENT_PAGE_PRIVATE_FIRST_PAGE | len << 2);
1794 * basic readpage implementation. Locked extent state structs are inserted
1795 * into the tree that are removed when the IO is done (by the end_io
1798 static int __extent_read_full_page(struct extent_io_tree *tree,
1800 get_extent_t *get_extent,
1803 struct inode *inode = page->mapping->host;
1804 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1805 u64 page_end = start + PAGE_CACHE_SIZE - 1;
1809 u64 last_byte = i_size_read(inode);
1813 struct extent_map *em;
1814 struct block_device *bdev;
1817 size_t page_offset = 0;
1819 size_t blocksize = inode->i_sb->s_blocksize;
1821 set_page_extent_mapped(page);
1824 lock_extent(tree, start, end, GFP_NOFS);
1826 while (cur <= end) {
1827 if (cur >= last_byte) {
1829 iosize = PAGE_CACHE_SIZE - page_offset;
1830 userpage = kmap_atomic(page, KM_USER0);
1831 memset(userpage + page_offset, 0, iosize);
1832 flush_dcache_page(page);
1833 kunmap_atomic(userpage, KM_USER0);
1834 set_extent_uptodate(tree, cur, cur + iosize - 1,
1836 unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
1839 em = get_extent(inode, page, page_offset, cur,
1841 if (IS_ERR(em) || !em) {
1843 unlock_extent(tree, cur, end, GFP_NOFS);
1847 extent_offset = cur - em->start;
1848 BUG_ON(extent_map_end(em) <= cur);
1851 iosize = min(extent_map_end(em) - cur, end - cur + 1);
1852 cur_end = min(extent_map_end(em) - 1, end);
1853 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
1854 sector = (em->block_start + extent_offset) >> 9;
1856 block_start = em->block_start;
1857 free_extent_map(em);
1860 /* we've found a hole, just zero and go on */
1861 if (block_start == EXTENT_MAP_HOLE) {
1863 userpage = kmap_atomic(page, KM_USER0);
1864 memset(userpage + page_offset, 0, iosize);
1865 flush_dcache_page(page);
1866 kunmap_atomic(userpage, KM_USER0);
1868 set_extent_uptodate(tree, cur, cur + iosize - 1,
1870 unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
1872 page_offset += iosize;
1875 /* the get_extent function already copied into the page */
1876 if (test_range_bit(tree, cur, cur_end, EXTENT_UPTODATE, 1)) {
1877 unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
1879 page_offset += iosize;
1882 /* we have an inline extent but it didn't get marked up
1883 * to date. Error out
1885 if (block_start == EXTENT_MAP_INLINE) {
1887 unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
1889 page_offset += iosize;
1894 if (tree->ops && tree->ops->readpage_io_hook) {
1895 ret = tree->ops->readpage_io_hook(page, cur,
1899 unsigned long nr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
1901 ret = submit_extent_page(READ, tree, page,
1902 sector, iosize, page_offset,
1904 end_bio_extent_readpage);
1909 page_offset += iosize;
1913 if (!PageError(page))
1914 SetPageUptodate(page);
1920 int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
1921 get_extent_t *get_extent)
1923 struct bio *bio = NULL;
1926 ret = __extent_read_full_page(tree, page, get_extent, &bio);
1928 submit_one_bio(READ, bio);
1931 EXPORT_SYMBOL(extent_read_full_page);
1934 * the writepage semantics are similar to regular writepage. extent
1935 * records are inserted to lock ranges in the tree, and as dirty areas
1936 * are found, they are marked writeback. Then the lock bits are removed
1937 * and the end_io handler clears the writeback ranges
1939 static int __extent_writepage(struct page *page, struct writeback_control *wbc,
1942 struct inode *inode = page->mapping->host;
1943 struct extent_page_data *epd = data;
1944 struct extent_io_tree *tree = epd->tree;
1945 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1947 u64 page_end = start + PAGE_CACHE_SIZE - 1;
1951 u64 last_byte = i_size_read(inode);
1955 struct extent_map *em;
1956 struct block_device *bdev;
1959 size_t page_offset = 0;
1961 loff_t i_size = i_size_read(inode);
1962 unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
1966 WARN_ON(!PageLocked(page));
1967 if (page->index > end_index) {
1968 clear_extent_dirty(tree, start, page_end, GFP_NOFS);
1973 if (page->index == end_index) {
1976 size_t offset = i_size & (PAGE_CACHE_SIZE - 1);
1978 userpage = kmap_atomic(page, KM_USER0);
1979 memset(userpage + offset, 0, PAGE_CACHE_SIZE - offset);
1980 flush_dcache_page(page);
1981 kunmap_atomic(userpage, KM_USER0);
1984 set_page_extent_mapped(page);
1986 delalloc_start = start;
1988 while(delalloc_end < page_end) {
1989 nr_delalloc = find_lock_delalloc_range(tree, &delalloc_start,
1992 if (nr_delalloc == 0) {
1993 delalloc_start = delalloc_end + 1;
1996 tree->ops->fill_delalloc(inode, delalloc_start,
1998 clear_extent_bit(tree, delalloc_start,
2000 EXTENT_LOCKED | EXTENT_DELALLOC,
2002 delalloc_start = delalloc_end + 1;
2004 lock_extent(tree, start, page_end, GFP_NOFS);
2007 if (test_range_bit(tree, start, page_end, EXTENT_DELALLOC, 0)) {
2008 printk("found delalloc bits after lock_extent\n");
2011 if (last_byte <= start) {
2012 clear_extent_dirty(tree, start, page_end, GFP_NOFS);
2016 set_extent_uptodate(tree, start, page_end, GFP_NOFS);
2017 blocksize = inode->i_sb->s_blocksize;
2019 while (cur <= end) {
2020 if (cur >= last_byte) {
2021 clear_extent_dirty(tree, cur, page_end, GFP_NOFS);
2024 em = epd->get_extent(inode, page, page_offset, cur,
2026 if (IS_ERR(em) || !em) {
2031 extent_offset = cur - em->start;
2032 BUG_ON(extent_map_end(em) <= cur);
2034 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2035 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
2036 sector = (em->block_start + extent_offset) >> 9;
2038 block_start = em->block_start;
2039 free_extent_map(em);
2042 if (block_start == EXTENT_MAP_HOLE ||
2043 block_start == EXTENT_MAP_INLINE) {
2044 clear_extent_dirty(tree, cur,
2045 cur + iosize - 1, GFP_NOFS);
2047 page_offset += iosize;
2051 /* leave this out until we have a page_mkwrite call */
2052 if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
2055 page_offset += iosize;
2058 clear_extent_dirty(tree, cur, cur + iosize - 1, GFP_NOFS);
2059 if (tree->ops && tree->ops->writepage_io_hook) {
2060 ret = tree->ops->writepage_io_hook(page, cur,
2068 unsigned long max_nr = end_index + 1;
2069 set_range_writeback(tree, cur, cur + iosize - 1);
2070 if (!PageWriteback(page)) {
2071 printk("warning page %lu not writeback, "
2072 "cur %llu end %llu\n", page->index,
2073 (unsigned long long)cur,
2074 (unsigned long long)end);
2077 ret = submit_extent_page(WRITE, tree, page, sector,
2078 iosize, page_offset, bdev,
2080 end_bio_extent_writepage);
2085 page_offset += iosize;
2090 /* make sure the mapping tag for page dirty gets cleared */
2091 set_page_writeback(page);
2092 end_page_writeback(page);
2094 unlock_extent(tree, start, page_end, GFP_NOFS);
2099 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,18)
2101 /* Taken directly from 2.6.23 for 2.6.18 back port */
2102 typedef int (*writepage_t)(struct page *page, struct writeback_control *wbc,
2106 * write_cache_pages - walk the list of dirty pages of the given address space
2107 * and write all of them.
2108 * @mapping: address space structure to write
2109 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
2110 * @writepage: function called for each page
2111 * @data: data passed to writepage function
2113 * If a page is already under I/O, write_cache_pages() skips it, even
2114 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
2115 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
2116 * and msync() need to guarantee that all the data which was dirty at the time
2117 * the call was made get new I/O started against them. If wbc->sync_mode is
2118 * WB_SYNC_ALL then we were called for data integrity and we must wait for
2119 * existing IO to complete.
2121 static int write_cache_pages(struct address_space *mapping,
2122 struct writeback_control *wbc, writepage_t writepage,
2125 struct backing_dev_info *bdi = mapping->backing_dev_info;
2128 struct pagevec pvec;
2131 pgoff_t end; /* Inclusive */
2133 int range_whole = 0;
2135 if (wbc->nonblocking && bdi_write_congested(bdi)) {
2136 wbc->encountered_congestion = 1;
2140 pagevec_init(&pvec, 0);
2141 if (wbc->range_cyclic) {
2142 index = mapping->writeback_index; /* Start from prev offset */
2145 index = wbc->range_start >> PAGE_CACHE_SHIFT;
2146 end = wbc->range_end >> PAGE_CACHE_SHIFT;
2147 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2152 while (!done && (index <= end) &&
2153 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
2154 PAGECACHE_TAG_DIRTY,
2155 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
2159 for (i = 0; i < nr_pages; i++) {
2160 struct page *page = pvec.pages[i];
2163 * At this point we hold neither mapping->tree_lock nor
2164 * lock on the page itself: the page may be truncated or
2165 * invalidated (changing page->mapping to NULL), or even
2166 * swizzled back from swapper_space to tmpfs file
2171 if (unlikely(page->mapping != mapping)) {
2176 if (!wbc->range_cyclic && page->index > end) {
2182 if (wbc->sync_mode != WB_SYNC_NONE)
2183 wait_on_page_writeback(page);
2185 if (PageWriteback(page) ||
2186 !clear_page_dirty_for_io(page)) {
2191 ret = (*writepage)(page, wbc, data);
2193 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
2197 if (ret || (--(wbc->nr_to_write) <= 0))
2199 if (wbc->nonblocking && bdi_write_congested(bdi)) {
2200 wbc->encountered_congestion = 1;
2204 pagevec_release(&pvec);
2207 if (!scanned && !done) {
2209 * We hit the last page and there is more work to be done: wrap
2210 * back to the start of the file
2216 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2217 mapping->writeback_index = index;
2222 int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
2223 get_extent_t *get_extent,
2224 struct writeback_control *wbc)
2227 struct address_space *mapping = page->mapping;
2228 struct extent_page_data epd = {
2231 .get_extent = get_extent,
2233 struct writeback_control wbc_writepages = {
2235 .sync_mode = WB_SYNC_NONE,
2236 .older_than_this = NULL,
2238 .range_start = page_offset(page) + PAGE_CACHE_SIZE,
2239 .range_end = (loff_t)-1,
2243 ret = __extent_writepage(page, wbc, &epd);
2245 write_cache_pages(mapping, &wbc_writepages, __extent_writepage, &epd);
2247 submit_one_bio(WRITE, epd.bio);
2251 EXPORT_SYMBOL(extent_write_full_page);
2254 int extent_writepages(struct extent_io_tree *tree,
2255 struct address_space *mapping,
2256 get_extent_t *get_extent,
2257 struct writeback_control *wbc)
2260 struct extent_page_data epd = {
2263 .get_extent = get_extent,
2266 ret = write_cache_pages(mapping, wbc, __extent_writepage, &epd);
2268 submit_one_bio(WRITE, epd.bio);
2272 EXPORT_SYMBOL(extent_writepages);
2274 int extent_readpages(struct extent_io_tree *tree,
2275 struct address_space *mapping,
2276 struct list_head *pages, unsigned nr_pages,
2277 get_extent_t get_extent)
2279 struct bio *bio = NULL;
2281 struct pagevec pvec;
2283 pagevec_init(&pvec, 0);
2284 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
2285 struct page *page = list_entry(pages->prev, struct page, lru);
2287 prefetchw(&page->flags);
2288 list_del(&page->lru);
2290 * what we want to do here is call add_to_page_cache_lru,
2291 * but that isn't exported, so we reproduce it here
2293 if (!add_to_page_cache(page, mapping,
2294 page->index, GFP_KERNEL)) {
2296 /* open coding of lru_cache_add, also not exported */
2297 page_cache_get(page);
2298 if (!pagevec_add(&pvec, page))
2299 __pagevec_lru_add(&pvec);
2300 __extent_read_full_page(tree, page, get_extent, &bio);
2302 page_cache_release(page);
2304 if (pagevec_count(&pvec))
2305 __pagevec_lru_add(&pvec);
2306 BUG_ON(!list_empty(pages));
2308 submit_one_bio(READ, bio);
2311 EXPORT_SYMBOL(extent_readpages);
2314 * basic invalidatepage code, this waits on any locked or writeback
2315 * ranges corresponding to the page, and then deletes any extent state
2316 * records from the tree
2318 int extent_invalidatepage(struct extent_io_tree *tree,
2319 struct page *page, unsigned long offset)
2321 u64 start = ((u64)page->index << PAGE_CACHE_SHIFT);
2322 u64 end = start + PAGE_CACHE_SIZE - 1;
2323 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
2325 start += (offset + blocksize -1) & ~(blocksize - 1);
2329 lock_extent(tree, start, end, GFP_NOFS);
2330 wait_on_extent_writeback(tree, start, end);
2331 clear_extent_bit(tree, start, end,
2332 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC,
2336 EXPORT_SYMBOL(extent_invalidatepage);
2339 * simple commit_write call, set_range_dirty is used to mark both
2340 * the pages and the extent records as dirty
2342 int extent_commit_write(struct extent_io_tree *tree,
2343 struct inode *inode, struct page *page,
2344 unsigned from, unsigned to)
2346 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2348 set_page_extent_mapped(page);
2349 set_page_dirty(page);
2351 if (pos > inode->i_size) {
2352 i_size_write(inode, pos);
2353 mark_inode_dirty(inode);
2357 EXPORT_SYMBOL(extent_commit_write);
2359 int extent_prepare_write(struct extent_io_tree *tree,
2360 struct inode *inode, struct page *page,
2361 unsigned from, unsigned to, get_extent_t *get_extent)
2363 u64 page_start = (u64)page->index << PAGE_CACHE_SHIFT;
2364 u64 page_end = page_start + PAGE_CACHE_SIZE - 1;
2366 u64 orig_block_start;
2369 struct extent_map *em;
2370 unsigned blocksize = 1 << inode->i_blkbits;
2371 size_t page_offset = 0;
2372 size_t block_off_start;
2373 size_t block_off_end;
2379 set_page_extent_mapped(page);
2381 block_start = (page_start + from) & ~((u64)blocksize - 1);
2382 block_end = (page_start + to - 1) | (blocksize - 1);
2383 orig_block_start = block_start;
2385 lock_extent(tree, page_start, page_end, GFP_NOFS);
2386 while(block_start <= block_end) {
2387 em = get_extent(inode, page, page_offset, block_start,
2388 block_end - block_start + 1, 1);
2389 if (IS_ERR(em) || !em) {
2392 cur_end = min(block_end, extent_map_end(em) - 1);
2393 block_off_start = block_start & (PAGE_CACHE_SIZE - 1);
2394 block_off_end = block_off_start + blocksize;
2395 isnew = clear_extent_new(tree, block_start, cur_end, GFP_NOFS);
2397 if (!PageUptodate(page) && isnew &&
2398 (block_off_end > to || block_off_start < from)) {
2401 kaddr = kmap_atomic(page, KM_USER0);
2402 if (block_off_end > to)
2403 memset(kaddr + to, 0, block_off_end - to);
2404 if (block_off_start < from)
2405 memset(kaddr + block_off_start, 0,
2406 from - block_off_start);
2407 flush_dcache_page(page);
2408 kunmap_atomic(kaddr, KM_USER0);
2410 if ((em->block_start != EXTENT_MAP_HOLE &&
2411 em->block_start != EXTENT_MAP_INLINE) &&
2412 !isnew && !PageUptodate(page) &&
2413 (block_off_end > to || block_off_start < from) &&
2414 !test_range_bit(tree, block_start, cur_end,
2415 EXTENT_UPTODATE, 1)) {
2417 u64 extent_offset = block_start - em->start;
2419 sector = (em->block_start + extent_offset) >> 9;
2420 iosize = (cur_end - block_start + blocksize) &
2421 ~((u64)blocksize - 1);
2423 * we've already got the extent locked, but we
2424 * need to split the state such that our end_bio
2425 * handler can clear the lock.
2427 set_extent_bit(tree, block_start,
2428 block_start + iosize - 1,
2429 EXTENT_LOCKED, 0, NULL, GFP_NOFS);
2430 ret = submit_extent_page(READ, tree, page,
2431 sector, iosize, page_offset, em->bdev,
2433 end_bio_extent_preparewrite);
2435 block_start = block_start + iosize;
2437 set_extent_uptodate(tree, block_start, cur_end,
2439 unlock_extent(tree, block_start, cur_end, GFP_NOFS);
2440 block_start = cur_end + 1;
2442 page_offset = block_start & (PAGE_CACHE_SIZE - 1);
2443 free_extent_map(em);
2446 wait_extent_bit(tree, orig_block_start,
2447 block_end, EXTENT_LOCKED);
2449 check_page_uptodate(tree, page);
2451 /* FIXME, zero out newly allocated blocks on error */
2454 EXPORT_SYMBOL(extent_prepare_write);
2457 * a helper for releasepage. As long as there are no locked extents
2458 * in the range corresponding to the page, both state records and extent
2459 * map records are removed
2461 int try_release_extent_mapping(struct extent_map_tree *map,
2462 struct extent_io_tree *tree, struct page *page,
2465 struct extent_map *em;
2466 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2467 u64 end = start + PAGE_CACHE_SIZE - 1;
2468 u64 orig_start = start;
2470 if ((mask & __GFP_WAIT) &&
2471 page->mapping->host->i_size > 16 * 1024 * 1024) {
2473 while (start <= end) {
2474 len = end - start + 1;
2475 spin_lock(&map->lock);
2476 em = lookup_extent_mapping(map, start, len);
2477 if (!em || IS_ERR(em)) {
2478 spin_unlock(&map->lock);
2481 if (em->start != start) {
2482 spin_unlock(&map->lock);
2483 free_extent_map(em);
2486 if (!test_range_bit(tree, em->start,
2487 extent_map_end(em) - 1,
2488 EXTENT_LOCKED, 0)) {
2489 remove_extent_mapping(map, em);
2490 /* once for the rb tree */
2491 free_extent_map(em);
2493 start = extent_map_end(em);
2494 spin_unlock(&map->lock);
2497 free_extent_map(em);
2500 if (test_range_bit(tree, orig_start, end, EXTENT_IOBITS, 0))
2503 if ((mask & GFP_NOFS) == GFP_NOFS)
2505 clear_extent_bit(tree, orig_start, end, EXTENT_UPTODATE,
2510 EXPORT_SYMBOL(try_release_extent_mapping);
2512 sector_t extent_bmap(struct address_space *mapping, sector_t iblock,
2513 get_extent_t *get_extent)
2515 struct inode *inode = mapping->host;
2516 u64 start = iblock << inode->i_blkbits;
2517 sector_t sector = 0;
2518 struct extent_map *em;
2520 em = get_extent(inode, NULL, 0, start, (1 << inode->i_blkbits), 0);
2521 if (!em || IS_ERR(em))
2524 if (em->block_start == EXTENT_MAP_INLINE ||
2525 em->block_start == EXTENT_MAP_HOLE)
2528 sector = (em->block_start + start - em->start) >> inode->i_blkbits;
2530 free_extent_map(em);
2534 static int add_lru(struct extent_io_tree *tree, struct extent_buffer *eb)
2536 if (list_empty(&eb->lru)) {
2537 extent_buffer_get(eb);
2538 list_add(&eb->lru, &tree->buffer_lru);
2540 if (tree->lru_size >= BUFFER_LRU_MAX) {
2541 struct extent_buffer *rm;
2542 rm = list_entry(tree->buffer_lru.prev,
2543 struct extent_buffer, lru);
2545 list_del_init(&rm->lru);
2546 free_extent_buffer(rm);
2549 list_move(&eb->lru, &tree->buffer_lru);
2552 static struct extent_buffer *find_lru(struct extent_io_tree *tree,
2553 u64 start, unsigned long len)
2555 struct list_head *lru = &tree->buffer_lru;
2556 struct list_head *cur = lru->next;
2557 struct extent_buffer *eb;
2559 if (list_empty(lru))
2563 eb = list_entry(cur, struct extent_buffer, lru);
2564 if (eb->start == start && eb->len == len) {
2565 extent_buffer_get(eb);
2569 } while (cur != lru);
2573 static inline unsigned long num_extent_pages(u64 start, u64 len)
2575 return ((start + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT) -
2576 (start >> PAGE_CACHE_SHIFT);
2579 static inline struct page *extent_buffer_page(struct extent_buffer *eb,
2583 struct address_space *mapping;
2586 return eb->first_page;
2587 i += eb->start >> PAGE_CACHE_SHIFT;
2588 mapping = eb->first_page->mapping;
2589 read_lock_irq(&mapping->tree_lock);
2590 p = radix_tree_lookup(&mapping->page_tree, i);
2591 read_unlock_irq(&mapping->tree_lock);
2595 int release_extent_buffer_tail_pages(struct extent_buffer *eb)
2597 unsigned long num_pages = num_extent_pages(eb->start, eb->len);
2603 for (i = 1; i < num_pages; i++) {
2604 page = extent_buffer_page(eb, i);
2605 page_cache_release(page);
2611 int invalidate_extent_lru(struct extent_io_tree *tree, u64 start,
2614 struct list_head *lru = &tree->buffer_lru;
2615 struct list_head *cur = lru->next;
2616 struct extent_buffer *eb;
2619 spin_lock(&tree->lru_lock);
2620 if (list_empty(lru))
2624 eb = list_entry(cur, struct extent_buffer, lru);
2625 if (eb->start <= start && eb->start + eb->len > start) {
2626 eb->flags &= ~EXTENT_UPTODATE;
2629 } while (cur != lru);
2631 spin_unlock(&tree->lru_lock);
2635 static struct extent_buffer *__alloc_extent_buffer(struct extent_io_tree *tree,
2640 struct extent_buffer *eb = NULL;
2641 unsigned long flags;
2643 spin_lock(&tree->lru_lock);
2644 eb = find_lru(tree, start, len);
2645 spin_unlock(&tree->lru_lock);
2650 eb = kmem_cache_zalloc(extent_buffer_cache, mask);
2651 INIT_LIST_HEAD(&eb->lru);
2654 spin_lock_irqsave(&leak_lock, flags);
2655 list_add(&eb->leak_list, &buffers);
2656 spin_unlock_irqrestore(&leak_lock, flags);
2657 atomic_set(&eb->refs, 1);
2662 static void __free_extent_buffer(struct extent_buffer *eb)
2664 unsigned long flags;
2665 spin_lock_irqsave(&leak_lock, flags);
2666 list_del(&eb->leak_list);
2667 spin_unlock_irqrestore(&leak_lock, flags);
2668 kmem_cache_free(extent_buffer_cache, eb);
2671 struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
2672 u64 start, unsigned long len,
2676 unsigned long num_pages = num_extent_pages(start, len);
2678 unsigned long index = start >> PAGE_CACHE_SHIFT;
2679 struct extent_buffer *eb;
2681 struct address_space *mapping = tree->mapping;
2684 eb = __alloc_extent_buffer(tree, start, len, mask);
2688 if (eb->flags & EXTENT_BUFFER_FILLED)
2692 eb->first_page = page0;
2695 page_cache_get(page0);
2696 mark_page_accessed(page0);
2697 set_page_extent_mapped(page0);
2698 set_page_extent_head(page0, len);
2702 for (; i < num_pages; i++, index++) {
2703 p = find_or_create_page(mapping, index, mask | __GFP_HIGHMEM);
2708 set_page_extent_mapped(p);
2709 mark_page_accessed(p);
2712 set_page_extent_head(p, len);
2714 set_page_private(p, EXTENT_PAGE_PRIVATE);
2716 if (!PageUptodate(p))
2721 eb->flags |= EXTENT_UPTODATE;
2722 eb->flags |= EXTENT_BUFFER_FILLED;
2725 spin_lock(&tree->lru_lock);
2727 spin_unlock(&tree->lru_lock);
2731 spin_lock(&tree->lru_lock);
2732 list_del_init(&eb->lru);
2733 spin_unlock(&tree->lru_lock);
2734 if (!atomic_dec_and_test(&eb->refs))
2736 for (index = 1; index < i; index++) {
2737 page_cache_release(extent_buffer_page(eb, index));
2740 page_cache_release(extent_buffer_page(eb, 0));
2741 __free_extent_buffer(eb);
2744 EXPORT_SYMBOL(alloc_extent_buffer);
2746 struct extent_buffer *find_extent_buffer(struct extent_io_tree *tree,
2747 u64 start, unsigned long len,
2750 unsigned long num_pages = num_extent_pages(start, len);
2752 unsigned long index = start >> PAGE_CACHE_SHIFT;
2753 struct extent_buffer *eb;
2755 struct address_space *mapping = tree->mapping;
2758 eb = __alloc_extent_buffer(tree, start, len, mask);
2762 if (eb->flags & EXTENT_BUFFER_FILLED)
2765 for (i = 0; i < num_pages; i++, index++) {
2766 p = find_lock_page(mapping, index);
2770 set_page_extent_mapped(p);
2771 mark_page_accessed(p);
2775 set_page_extent_head(p, len);
2777 set_page_private(p, EXTENT_PAGE_PRIVATE);
2780 if (!PageUptodate(p))
2785 eb->flags |= EXTENT_UPTODATE;
2786 eb->flags |= EXTENT_BUFFER_FILLED;
2789 spin_lock(&tree->lru_lock);
2791 spin_unlock(&tree->lru_lock);
2794 spin_lock(&tree->lru_lock);
2795 list_del_init(&eb->lru);
2796 spin_unlock(&tree->lru_lock);
2797 if (!atomic_dec_and_test(&eb->refs))
2799 for (index = 1; index < i; index++) {
2800 page_cache_release(extent_buffer_page(eb, index));
2803 page_cache_release(extent_buffer_page(eb, 0));
2804 __free_extent_buffer(eb);
2807 EXPORT_SYMBOL(find_extent_buffer);
2809 void free_extent_buffer(struct extent_buffer *eb)
2812 unsigned long num_pages;
2817 if (!atomic_dec_and_test(&eb->refs))
2820 WARN_ON(!list_empty(&eb->lru));
2821 num_pages = num_extent_pages(eb->start, eb->len);
2823 for (i = 1; i < num_pages; i++) {
2824 page_cache_release(extent_buffer_page(eb, i));
2826 page_cache_release(extent_buffer_page(eb, 0));
2827 __free_extent_buffer(eb);
2829 EXPORT_SYMBOL(free_extent_buffer);
2831 int clear_extent_buffer_dirty(struct extent_io_tree *tree,
2832 struct extent_buffer *eb)
2836 unsigned long num_pages;
2839 u64 start = eb->start;
2840 u64 end = start + eb->len - 1;
2842 set = clear_extent_dirty(tree, start, end, GFP_NOFS);
2843 num_pages = num_extent_pages(eb->start, eb->len);
2845 for (i = 0; i < num_pages; i++) {
2846 page = extent_buffer_page(eb, i);
2849 set_page_extent_head(page, eb->len);
2851 set_page_private(page, EXTENT_PAGE_PRIVATE);
2854 * if we're on the last page or the first page and the
2855 * block isn't aligned on a page boundary, do extra checks
2856 * to make sure we don't clean page that is partially dirty
2858 if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) ||
2859 ((i == num_pages - 1) &&
2860 ((eb->start + eb->len) & (PAGE_CACHE_SIZE - 1)))) {
2861 start = (u64)page->index << PAGE_CACHE_SHIFT;
2862 end = start + PAGE_CACHE_SIZE - 1;
2863 if (test_range_bit(tree, start, end,
2869 clear_page_dirty_for_io(page);
2870 read_lock_irq(&page->mapping->tree_lock);
2871 if (!PageDirty(page)) {
2872 radix_tree_tag_clear(&page->mapping->page_tree,
2874 PAGECACHE_TAG_DIRTY);
2876 read_unlock_irq(&page->mapping->tree_lock);
2881 EXPORT_SYMBOL(clear_extent_buffer_dirty);
2883 int wait_on_extent_buffer_writeback(struct extent_io_tree *tree,
2884 struct extent_buffer *eb)
2886 return wait_on_extent_writeback(tree, eb->start,
2887 eb->start + eb->len - 1);
2889 EXPORT_SYMBOL(wait_on_extent_buffer_writeback);
2891 int set_extent_buffer_dirty(struct extent_io_tree *tree,
2892 struct extent_buffer *eb)
2895 unsigned long num_pages;
2897 num_pages = num_extent_pages(eb->start, eb->len);
2898 for (i = 0; i < num_pages; i++) {
2899 struct page *page = extent_buffer_page(eb, i);
2900 /* writepage may need to do something special for the
2901 * first page, we have to make sure page->private is
2902 * properly set. releasepage may drop page->private
2903 * on us if the page isn't already dirty.
2907 set_page_extent_head(page, eb->len);
2908 } else if (PagePrivate(page) &&
2909 page->private != EXTENT_PAGE_PRIVATE) {
2911 set_page_extent_mapped(page);
2914 __set_page_dirty_nobuffers(extent_buffer_page(eb, i));
2918 return set_extent_dirty(tree, eb->start,
2919 eb->start + eb->len - 1, GFP_NOFS);
2921 EXPORT_SYMBOL(set_extent_buffer_dirty);
2923 int set_extent_buffer_uptodate(struct extent_io_tree *tree,
2924 struct extent_buffer *eb)
2928 unsigned long num_pages;
2930 num_pages = num_extent_pages(eb->start, eb->len);
2932 set_extent_uptodate(tree, eb->start, eb->start + eb->len - 1,
2934 for (i = 0; i < num_pages; i++) {
2935 page = extent_buffer_page(eb, i);
2936 if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) ||
2937 ((i == num_pages - 1) &&
2938 ((eb->start + eb->len) & (PAGE_CACHE_SIZE - 1)))) {
2939 check_page_uptodate(tree, page);
2942 SetPageUptodate(page);
2946 EXPORT_SYMBOL(set_extent_buffer_uptodate);
2948 int extent_range_uptodate(struct extent_io_tree *tree,
2953 int pg_uptodate = 1;
2955 unsigned long index;
2957 ret = test_range_bit(tree, start, end, EXTENT_UPTODATE, 1);
2960 while(start <= end) {
2961 index = start >> PAGE_CACHE_SHIFT;
2962 page = find_get_page(tree->mapping, index);
2963 uptodate = PageUptodate(page);
2964 page_cache_release(page);
2969 start += PAGE_CACHE_SIZE;
2974 int extent_buffer_uptodate(struct extent_io_tree *tree,
2975 struct extent_buffer *eb)
2979 unsigned long num_pages;
2982 int pg_uptodate = 1;
2984 if (eb->flags & EXTENT_UPTODATE)
2987 ret2 = test_range_bit(tree, eb->start, eb->start + eb->len - 1,
2988 EXTENT_UPTODATE, 1);
2990 num_pages = num_extent_pages(eb->start, eb->len);
2991 for (i = 0; i < num_pages; i++) {
2992 page = extent_buffer_page(eb, i);
2993 if (!PageUptodate(page)) {
2998 if ((ret || ret2) && !pg_uptodate) {
2999 printk("uptodate error2 eb %Lu ret %d ret2 %d pg_uptodate %d\n", eb->start, ret, ret2, pg_uptodate);
3002 return (ret || ret2);
3004 EXPORT_SYMBOL(extent_buffer_uptodate);
3006 int read_extent_buffer_pages(struct extent_io_tree *tree,
3007 struct extent_buffer *eb,
3008 u64 start, int wait,
3009 get_extent_t *get_extent)
3012 unsigned long start_i;
3016 int locked_pages = 0;
3017 int all_uptodate = 1;
3018 int inc_all_pages = 0;
3019 unsigned long num_pages;
3020 struct bio *bio = NULL;
3022 if (eb->flags & EXTENT_UPTODATE)
3025 if (test_range_bit(tree, eb->start, eb->start + eb->len - 1,
3026 EXTENT_UPTODATE, 1)) {
3031 WARN_ON(start < eb->start);
3032 start_i = (start >> PAGE_CACHE_SHIFT) -
3033 (eb->start >> PAGE_CACHE_SHIFT);
3038 num_pages = num_extent_pages(eb->start, eb->len);
3039 for (i = start_i; i < num_pages; i++) {
3040 page = extent_buffer_page(eb, i);
3042 if (TestSetPageLocked(page))
3048 if (!PageUptodate(page)) {
3054 eb->flags |= EXTENT_UPTODATE;
3058 for (i = start_i; i < num_pages; i++) {
3059 page = extent_buffer_page(eb, i);
3061 page_cache_get(page);
3062 if (!PageUptodate(page)) {
3065 err = __extent_read_full_page(tree, page,
3076 submit_one_bio(READ, bio);
3081 for (i = start_i; i < num_pages; i++) {
3082 page = extent_buffer_page(eb, i);
3083 wait_on_page_locked(page);
3084 if (!PageUptodate(page)) {
3089 eb->flags |= EXTENT_UPTODATE;
3094 while(locked_pages > 0) {
3095 page = extent_buffer_page(eb, i);
3102 EXPORT_SYMBOL(read_extent_buffer_pages);
3104 void read_extent_buffer(struct extent_buffer *eb, void *dstv,
3105 unsigned long start,
3112 char *dst = (char *)dstv;
3113 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
3114 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
3116 WARN_ON(start > eb->len);
3117 WARN_ON(start + len > eb->start + eb->len);
3119 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
3122 page = extent_buffer_page(eb, i);
3124 cur = min(len, (PAGE_CACHE_SIZE - offset));
3125 kaddr = kmap_atomic(page, KM_USER1);
3126 memcpy(dst, kaddr + offset, cur);
3127 kunmap_atomic(kaddr, KM_USER1);
3135 EXPORT_SYMBOL(read_extent_buffer);
3137 int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
3138 unsigned long min_len, char **token, char **map,
3139 unsigned long *map_start,
3140 unsigned long *map_len, int km)
3142 size_t offset = start & (PAGE_CACHE_SIZE - 1);
3145 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
3146 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
3147 unsigned long end_i = (start_offset + start + min_len - 1) >>
3154 offset = start_offset;
3158 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
3160 if (start + min_len > eb->len) {
3161 printk("bad mapping eb start %Lu len %lu, wanted %lu %lu\n", eb->start, eb->len, start, min_len);
3165 p = extent_buffer_page(eb, i);
3166 kaddr = kmap_atomic(p, km);
3168 *map = kaddr + offset;
3169 *map_len = PAGE_CACHE_SIZE - offset;
3172 EXPORT_SYMBOL(map_private_extent_buffer);
3174 int map_extent_buffer(struct extent_buffer *eb, unsigned long start,
3175 unsigned long min_len,
3176 char **token, char **map,
3177 unsigned long *map_start,
3178 unsigned long *map_len, int km)
3182 if (eb->map_token) {
3183 unmap_extent_buffer(eb, eb->map_token, km);
3184 eb->map_token = NULL;
3187 err = map_private_extent_buffer(eb, start, min_len, token, map,
3188 map_start, map_len, km);
3190 eb->map_token = *token;
3192 eb->map_start = *map_start;
3193 eb->map_len = *map_len;
3197 EXPORT_SYMBOL(map_extent_buffer);
3199 void unmap_extent_buffer(struct extent_buffer *eb, char *token, int km)
3201 kunmap_atomic(token, km);
3203 EXPORT_SYMBOL(unmap_extent_buffer);
3205 int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
3206 unsigned long start,
3213 char *ptr = (char *)ptrv;
3214 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
3215 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
3218 WARN_ON(start > eb->len);
3219 WARN_ON(start + len > eb->start + eb->len);
3221 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
3224 page = extent_buffer_page(eb, i);
3226 cur = min(len, (PAGE_CACHE_SIZE - offset));
3228 kaddr = kmap_atomic(page, KM_USER0);
3229 ret = memcmp(ptr, kaddr + offset, cur);
3230 kunmap_atomic(kaddr, KM_USER0);
3241 EXPORT_SYMBOL(memcmp_extent_buffer);
3243 void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
3244 unsigned long start, unsigned long len)
3250 char *src = (char *)srcv;
3251 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
3252 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
3254 WARN_ON(start > eb->len);
3255 WARN_ON(start + len > eb->start + eb->len);
3257 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
3260 page = extent_buffer_page(eb, i);
3261 WARN_ON(!PageUptodate(page));
3263 cur = min(len, PAGE_CACHE_SIZE - offset);
3264 kaddr = kmap_atomic(page, KM_USER1);
3265 memcpy(kaddr + offset, src, cur);
3266 kunmap_atomic(kaddr, KM_USER1);
3274 EXPORT_SYMBOL(write_extent_buffer);
3276 void memset_extent_buffer(struct extent_buffer *eb, char c,
3277 unsigned long start, unsigned long len)
3283 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
3284 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
3286 WARN_ON(start > eb->len);
3287 WARN_ON(start + len > eb->start + eb->len);
3289 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
3292 page = extent_buffer_page(eb, i);
3293 WARN_ON(!PageUptodate(page));
3295 cur = min(len, PAGE_CACHE_SIZE - offset);
3296 kaddr = kmap_atomic(page, KM_USER0);
3297 memset(kaddr + offset, c, cur);
3298 kunmap_atomic(kaddr, KM_USER0);
3305 EXPORT_SYMBOL(memset_extent_buffer);
3307 void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
3308 unsigned long dst_offset, unsigned long src_offset,
3311 u64 dst_len = dst->len;
3316 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
3317 unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
3319 WARN_ON(src->len != dst_len);
3321 offset = (start_offset + dst_offset) &
3322 ((unsigned long)PAGE_CACHE_SIZE - 1);
3325 page = extent_buffer_page(dst, i);
3326 WARN_ON(!PageUptodate(page));
3328 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
3330 kaddr = kmap_atomic(page, KM_USER0);
3331 read_extent_buffer(src, kaddr + offset, src_offset, cur);
3332 kunmap_atomic(kaddr, KM_USER0);
3340 EXPORT_SYMBOL(copy_extent_buffer);
3342 static void move_pages(struct page *dst_page, struct page *src_page,
3343 unsigned long dst_off, unsigned long src_off,
3346 char *dst_kaddr = kmap_atomic(dst_page, KM_USER0);
3347 if (dst_page == src_page) {
3348 memmove(dst_kaddr + dst_off, dst_kaddr + src_off, len);
3350 char *src_kaddr = kmap_atomic(src_page, KM_USER1);
3351 char *p = dst_kaddr + dst_off + len;
3352 char *s = src_kaddr + src_off + len;
3357 kunmap_atomic(src_kaddr, KM_USER1);
3359 kunmap_atomic(dst_kaddr, KM_USER0);
3362 static void copy_pages(struct page *dst_page, struct page *src_page,
3363 unsigned long dst_off, unsigned long src_off,
3366 char *dst_kaddr = kmap_atomic(dst_page, KM_USER0);
3369 if (dst_page != src_page)
3370 src_kaddr = kmap_atomic(src_page, KM_USER1);
3372 src_kaddr = dst_kaddr;
3374 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
3375 kunmap_atomic(dst_kaddr, KM_USER0);
3376 if (dst_page != src_page)
3377 kunmap_atomic(src_kaddr, KM_USER1);
3380 void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
3381 unsigned long src_offset, unsigned long len)
3384 size_t dst_off_in_page;
3385 size_t src_off_in_page;
3386 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
3387 unsigned long dst_i;
3388 unsigned long src_i;
3390 if (src_offset + len > dst->len) {
3391 printk("memmove bogus src_offset %lu move len %lu len %lu\n",
3392 src_offset, len, dst->len);
3395 if (dst_offset + len > dst->len) {
3396 printk("memmove bogus dst_offset %lu move len %lu len %lu\n",
3397 dst_offset, len, dst->len);
3402 dst_off_in_page = (start_offset + dst_offset) &
3403 ((unsigned long)PAGE_CACHE_SIZE - 1);
3404 src_off_in_page = (start_offset + src_offset) &
3405 ((unsigned long)PAGE_CACHE_SIZE - 1);
3407 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
3408 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
3410 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
3412 cur = min_t(unsigned long, cur,
3413 (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
3415 copy_pages(extent_buffer_page(dst, dst_i),
3416 extent_buffer_page(dst, src_i),
3417 dst_off_in_page, src_off_in_page, cur);
3424 EXPORT_SYMBOL(memcpy_extent_buffer);
3426 void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
3427 unsigned long src_offset, unsigned long len)
3430 size_t dst_off_in_page;
3431 size_t src_off_in_page;
3432 unsigned long dst_end = dst_offset + len - 1;
3433 unsigned long src_end = src_offset + len - 1;
3434 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
3435 unsigned long dst_i;
3436 unsigned long src_i;
3438 if (src_offset + len > dst->len) {
3439 printk("memmove bogus src_offset %lu move len %lu len %lu\n",
3440 src_offset, len, dst->len);
3443 if (dst_offset + len > dst->len) {
3444 printk("memmove bogus dst_offset %lu move len %lu len %lu\n",
3445 dst_offset, len, dst->len);
3448 if (dst_offset < src_offset) {
3449 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
3453 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
3454 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
3456 dst_off_in_page = (start_offset + dst_end) &
3457 ((unsigned long)PAGE_CACHE_SIZE - 1);
3458 src_off_in_page = (start_offset + src_end) &
3459 ((unsigned long)PAGE_CACHE_SIZE - 1);
3461 cur = min_t(unsigned long, len, src_off_in_page + 1);
3462 cur = min(cur, dst_off_in_page + 1);
3463 move_pages(extent_buffer_page(dst, dst_i),
3464 extent_buffer_page(dst, src_i),
3465 dst_off_in_page - cur + 1,
3466 src_off_in_page - cur + 1, cur);
3473 EXPORT_SYMBOL(memmove_extent_buffer);