1 #include <linux/bitops.h>
2 #include <linux/slab.h>
6 #include <linux/pagemap.h>
7 #include <linux/page-flags.h>
8 #include <linux/module.h>
9 #include <linux/spinlock.h>
10 #include <linux/blkdev.h>
11 #include <linux/swap.h>
12 #include <linux/version.h>
13 #include <linux/writeback.h>
14 #include <linux/pagevec.h>
15 #include "extent_map.h"
17 /* temporary define until extent_map moves out of btrfs */
18 struct kmem_cache *btrfs_cache_create(const char *name, size_t size,
19 unsigned long extra_flags,
20 void (*ctor)(void *, struct kmem_cache *,
23 static struct kmem_cache *extent_map_cache;
24 static struct kmem_cache *extent_state_cache;
25 static struct kmem_cache *extent_buffer_cache;
27 static LIST_HEAD(buffers);
28 static LIST_HEAD(states);
30 static spinlock_t state_lock = SPIN_LOCK_UNLOCKED;
31 #define BUFFER_LRU_MAX 64
37 struct rb_node rb_node;
40 struct extent_page_data {
42 struct extent_map_tree *tree;
43 get_extent_t *get_extent;
46 void __init extent_map_init(void)
48 extent_map_cache = btrfs_cache_create("extent_map",
49 sizeof(struct extent_map), 0,
51 extent_state_cache = btrfs_cache_create("extent_state",
52 sizeof(struct extent_state), 0,
54 extent_buffer_cache = btrfs_cache_create("extent_buffers",
55 sizeof(struct extent_buffer), 0,
59 void __exit extent_map_exit(void)
61 struct extent_state *state;
63 while (!list_empty(&states)) {
64 state = list_entry(states.next, struct extent_state, list);
65 printk("state leak: start %Lu end %Lu state %lu in tree %d refs %d\n", state->start, state->end, state->state, state->in_tree, atomic_read(&state->refs));
66 list_del(&state->list);
67 kmem_cache_free(extent_state_cache, state);
72 kmem_cache_destroy(extent_map_cache);
73 if (extent_state_cache)
74 kmem_cache_destroy(extent_state_cache);
75 if (extent_buffer_cache)
76 kmem_cache_destroy(extent_buffer_cache);
79 void extent_map_tree_init(struct extent_map_tree *tree,
80 struct address_space *mapping, gfp_t mask)
82 tree->map.rb_node = NULL;
83 tree->state.rb_node = NULL;
85 rwlock_init(&tree->lock);
86 spin_lock_init(&tree->lru_lock);
87 tree->mapping = mapping;
88 INIT_LIST_HEAD(&tree->buffer_lru);
91 EXPORT_SYMBOL(extent_map_tree_init);
93 void extent_map_tree_empty_lru(struct extent_map_tree *tree)
95 struct extent_buffer *eb;
96 while(!list_empty(&tree->buffer_lru)) {
97 eb = list_entry(tree->buffer_lru.next, struct extent_buffer,
99 list_del_init(&eb->lru);
100 free_extent_buffer(eb);
103 EXPORT_SYMBOL(extent_map_tree_empty_lru);
105 struct extent_map *alloc_extent_map(gfp_t mask)
107 struct extent_map *em;
108 em = kmem_cache_alloc(extent_map_cache, mask);
109 if (!em || IS_ERR(em))
112 atomic_set(&em->refs, 1);
115 EXPORT_SYMBOL(alloc_extent_map);
117 void free_extent_map(struct extent_map *em)
121 if (atomic_dec_and_test(&em->refs)) {
122 WARN_ON(em->in_tree);
123 kmem_cache_free(extent_map_cache, em);
126 EXPORT_SYMBOL(free_extent_map);
129 struct extent_state *alloc_extent_state(gfp_t mask)
131 struct extent_state *state;
134 state = kmem_cache_alloc(extent_state_cache, mask);
135 if (!state || IS_ERR(state))
141 spin_lock_irqsave(&state_lock, flags);
142 list_add(&state->list, &states);
143 spin_unlock_irqrestore(&state_lock, flags);
145 atomic_set(&state->refs, 1);
146 init_waitqueue_head(&state->wq);
149 EXPORT_SYMBOL(alloc_extent_state);
151 void free_extent_state(struct extent_state *state)
156 if (atomic_dec_and_test(&state->refs)) {
157 WARN_ON(state->in_tree);
158 spin_lock_irqsave(&state_lock, flags);
159 list_del(&state->list);
160 spin_unlock_irqrestore(&state_lock, flags);
161 kmem_cache_free(extent_state_cache, state);
164 EXPORT_SYMBOL(free_extent_state);
166 static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
167 struct rb_node *node)
169 struct rb_node ** p = &root->rb_node;
170 struct rb_node * parent = NULL;
171 struct tree_entry *entry;
175 entry = rb_entry(parent, struct tree_entry, rb_node);
177 if (offset < entry->start)
179 else if (offset > entry->end)
185 entry = rb_entry(node, struct tree_entry, rb_node);
187 rb_link_node(node, parent, p);
188 rb_insert_color(node, root);
192 static struct rb_node *__tree_search(struct rb_root *root, u64 offset,
193 struct rb_node **prev_ret)
195 struct rb_node * n = root->rb_node;
196 struct rb_node *prev = NULL;
197 struct tree_entry *entry;
198 struct tree_entry *prev_entry = NULL;
201 entry = rb_entry(n, struct tree_entry, rb_node);
205 if (offset < entry->start)
207 else if (offset > entry->end)
214 while(prev && offset > prev_entry->end) {
215 prev = rb_next(prev);
216 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
222 static inline struct rb_node *tree_search(struct rb_root *root, u64 offset)
224 struct rb_node *prev;
226 ret = __tree_search(root, offset, &prev);
232 static int tree_delete(struct rb_root *root, u64 offset)
234 struct rb_node *node;
235 struct tree_entry *entry;
237 node = __tree_search(root, offset, NULL);
240 entry = rb_entry(node, struct tree_entry, rb_node);
242 rb_erase(node, root);
247 * add_extent_mapping tries a simple backward merge with existing
248 * mappings. The extent_map struct passed in will be inserted into
249 * the tree directly (no copies made, just a reference taken).
251 int add_extent_mapping(struct extent_map_tree *tree,
252 struct extent_map *em)
255 struct extent_map *prev = NULL;
258 write_lock_irq(&tree->lock);
259 rb = tree_insert(&tree->map, em->end, &em->rb_node);
261 prev = rb_entry(rb, struct extent_map, rb_node);
262 printk("found extent map %Lu %Lu on insert of %Lu %Lu\n", prev->start, prev->end, em->start, em->end);
266 atomic_inc(&em->refs);
267 if (em->start != 0) {
268 rb = rb_prev(&em->rb_node);
270 prev = rb_entry(rb, struct extent_map, rb_node);
271 if (prev && prev->end + 1 == em->start &&
272 ((em->block_start == EXTENT_MAP_HOLE &&
273 prev->block_start == EXTENT_MAP_HOLE) ||
274 (em->block_start == EXTENT_MAP_INLINE &&
275 prev->block_start == EXTENT_MAP_INLINE) ||
276 (em->block_start == EXTENT_MAP_DELALLOC &&
277 prev->block_start == EXTENT_MAP_DELALLOC) ||
278 (em->block_start < EXTENT_MAP_DELALLOC - 1 &&
279 em->block_start == prev->block_end + 1))) {
280 em->start = prev->start;
281 em->block_start = prev->block_start;
282 rb_erase(&prev->rb_node, &tree->map);
284 free_extent_map(prev);
288 write_unlock_irq(&tree->lock);
291 EXPORT_SYMBOL(add_extent_mapping);
294 * lookup_extent_mapping returns the first extent_map struct in the
295 * tree that intersects the [start, end] (inclusive) range. There may
296 * be additional objects in the tree that intersect, so check the object
297 * returned carefully to make sure you don't need additional lookups.
299 struct extent_map *lookup_extent_mapping(struct extent_map_tree *tree,
302 struct extent_map *em;
303 struct rb_node *rb_node;
305 read_lock_irq(&tree->lock);
306 rb_node = tree_search(&tree->map, start);
311 if (IS_ERR(rb_node)) {
312 em = ERR_PTR(PTR_ERR(rb_node));
315 em = rb_entry(rb_node, struct extent_map, rb_node);
316 if (em->end < start || em->start > end) {
320 atomic_inc(&em->refs);
322 read_unlock_irq(&tree->lock);
325 EXPORT_SYMBOL(lookup_extent_mapping);
328 * removes an extent_map struct from the tree. No reference counts are
329 * dropped, and no checks are done to see if the range is in use
331 int remove_extent_mapping(struct extent_map_tree *tree, struct extent_map *em)
335 write_lock_irq(&tree->lock);
336 ret = tree_delete(&tree->map, em->end);
337 write_unlock_irq(&tree->lock);
340 EXPORT_SYMBOL(remove_extent_mapping);
343 * utility function to look for merge candidates inside a given range.
344 * Any extents with matching state are merged together into a single
345 * extent in the tree. Extents with EXTENT_IO in their state field
346 * are not merged because the end_io handlers need to be able to do
347 * operations on them without sleeping (or doing allocations/splits).
349 * This should be called with the tree lock held.
351 static int merge_state(struct extent_map_tree *tree,
352 struct extent_state *state)
354 struct extent_state *other;
355 struct rb_node *other_node;
357 if (state->state & EXTENT_IOBITS)
360 other_node = rb_prev(&state->rb_node);
362 other = rb_entry(other_node, struct extent_state, rb_node);
363 if (other->end == state->start - 1 &&
364 other->state == state->state) {
365 state->start = other->start;
367 rb_erase(&other->rb_node, &tree->state);
368 free_extent_state(other);
371 other_node = rb_next(&state->rb_node);
373 other = rb_entry(other_node, struct extent_state, rb_node);
374 if (other->start == state->end + 1 &&
375 other->state == state->state) {
376 other->start = state->start;
378 rb_erase(&state->rb_node, &tree->state);
379 free_extent_state(state);
386 * insert an extent_state struct into the tree. 'bits' are set on the
387 * struct before it is inserted.
389 * This may return -EEXIST if the extent is already there, in which case the
390 * state struct is freed.
392 * The tree lock is not taken internally. This is a utility function and
393 * probably isn't what you want to call (see set/clear_extent_bit).
395 static int insert_state(struct extent_map_tree *tree,
396 struct extent_state *state, u64 start, u64 end,
399 struct rb_node *node;
402 printk("end < start %Lu %Lu\n", end, start);
405 state->state |= bits;
406 state->start = start;
408 node = tree_insert(&tree->state, end, &state->rb_node);
410 struct extent_state *found;
411 found = rb_entry(node, struct extent_state, rb_node);
412 printk("found node %Lu %Lu on insert of %Lu %Lu\n", found->start, found->end, start, end);
413 free_extent_state(state);
416 merge_state(tree, state);
421 * split a given extent state struct in two, inserting the preallocated
422 * struct 'prealloc' as the newly created second half. 'split' indicates an
423 * offset inside 'orig' where it should be split.
426 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
427 * are two extent state structs in the tree:
428 * prealloc: [orig->start, split - 1]
429 * orig: [ split, orig->end ]
431 * The tree locks are not taken by this function. They need to be held
434 static int split_state(struct extent_map_tree *tree, struct extent_state *orig,
435 struct extent_state *prealloc, u64 split)
437 struct rb_node *node;
438 prealloc->start = orig->start;
439 prealloc->end = split - 1;
440 prealloc->state = orig->state;
443 node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node);
445 struct extent_state *found;
446 found = rb_entry(node, struct extent_state, rb_node);
447 printk("found node %Lu %Lu on insert of %Lu %Lu\n", found->start, found->end, prealloc->start, prealloc->end);
448 free_extent_state(prealloc);
455 * utility function to clear some bits in an extent state struct.
456 * it will optionally wake up any one waiting on this state (wake == 1), or
457 * forcibly remove the state from the tree (delete == 1).
459 * If no bits are set on the state struct after clearing things, the
460 * struct is freed and removed from the tree
462 static int clear_state_bit(struct extent_map_tree *tree,
463 struct extent_state *state, int bits, int wake,
466 int ret = state->state & bits;
467 state->state &= ~bits;
470 if (delete || state->state == 0) {
471 if (state->in_tree) {
472 rb_erase(&state->rb_node, &tree->state);
474 free_extent_state(state);
479 merge_state(tree, state);
485 * clear some bits on a range in the tree. This may require splitting
486 * or inserting elements in the tree, so the gfp mask is used to
487 * indicate which allocations or sleeping are allowed.
489 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
490 * the given range from the tree regardless of state (ie for truncate).
492 * the range [start, end] is inclusive.
494 * This takes the tree lock, and returns < 0 on error, > 0 if any of the
495 * bits were already set, or zero if none of the bits were already set.
497 int clear_extent_bit(struct extent_map_tree *tree, u64 start, u64 end,
498 int bits, int wake, int delete, gfp_t mask)
500 struct extent_state *state;
501 struct extent_state *prealloc = NULL;
502 struct rb_node *node;
508 if (!prealloc && (mask & __GFP_WAIT)) {
509 prealloc = alloc_extent_state(mask);
514 write_lock_irqsave(&tree->lock, flags);
516 * this search will find the extents that end after
519 node = tree_search(&tree->state, start);
522 state = rb_entry(node, struct extent_state, rb_node);
523 if (state->start > end)
525 WARN_ON(state->end < start);
528 * | ---- desired range ---- |
530 * | ------------- state -------------- |
532 * We need to split the extent we found, and may flip
533 * bits on second half.
535 * If the extent we found extends past our range, we
536 * just split and search again. It'll get split again
537 * the next time though.
539 * If the extent we found is inside our range, we clear
540 * the desired bit on it.
543 if (state->start < start) {
544 err = split_state(tree, state, prealloc, start);
545 BUG_ON(err == -EEXIST);
549 if (state->end <= end) {
550 start = state->end + 1;
551 set |= clear_state_bit(tree, state, bits,
554 start = state->start;
559 * | ---- desired range ---- |
561 * We need to split the extent, and clear the bit
564 if (state->start <= end && state->end > end) {
565 err = split_state(tree, state, prealloc, end + 1);
566 BUG_ON(err == -EEXIST);
570 set |= clear_state_bit(tree, prealloc, bits,
576 start = state->end + 1;
577 set |= clear_state_bit(tree, state, bits, wake, delete);
581 write_unlock_irqrestore(&tree->lock, flags);
583 free_extent_state(prealloc);
590 write_unlock_irqrestore(&tree->lock, flags);
591 if (mask & __GFP_WAIT)
595 EXPORT_SYMBOL(clear_extent_bit);
597 static int wait_on_state(struct extent_map_tree *tree,
598 struct extent_state *state)
601 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
602 read_unlock_irq(&tree->lock);
604 read_lock_irq(&tree->lock);
605 finish_wait(&state->wq, &wait);
610 * waits for one or more bits to clear on a range in the state tree.
611 * The range [start, end] is inclusive.
612 * The tree lock is taken by this function
614 int wait_extent_bit(struct extent_map_tree *tree, u64 start, u64 end, int bits)
616 struct extent_state *state;
617 struct rb_node *node;
619 read_lock_irq(&tree->lock);
623 * this search will find all the extents that end after
626 node = tree_search(&tree->state, start);
630 state = rb_entry(node, struct extent_state, rb_node);
632 if (state->start > end)
635 if (state->state & bits) {
636 start = state->start;
637 atomic_inc(&state->refs);
638 wait_on_state(tree, state);
639 free_extent_state(state);
642 start = state->end + 1;
647 if (need_resched()) {
648 read_unlock_irq(&tree->lock);
650 read_lock_irq(&tree->lock);
654 read_unlock_irq(&tree->lock);
657 EXPORT_SYMBOL(wait_extent_bit);
660 * set some bits on a range in the tree. This may require allocations
661 * or sleeping, so the gfp mask is used to indicate what is allowed.
663 * If 'exclusive' == 1, this will fail with -EEXIST if some part of the
664 * range already has the desired bits set. The start of the existing
665 * range is returned in failed_start in this case.
667 * [start, end] is inclusive
668 * This takes the tree lock.
670 int set_extent_bit(struct extent_map_tree *tree, u64 start, u64 end, int bits,
671 int exclusive, u64 *failed_start, gfp_t mask)
673 struct extent_state *state;
674 struct extent_state *prealloc = NULL;
675 struct rb_node *node;
682 if (!prealloc && (mask & __GFP_WAIT)) {
683 prealloc = alloc_extent_state(mask);
688 write_lock_irqsave(&tree->lock, flags);
690 * this search will find all the extents that end after
693 node = tree_search(&tree->state, start);
695 err = insert_state(tree, prealloc, start, end, bits);
697 BUG_ON(err == -EEXIST);
701 state = rb_entry(node, struct extent_state, rb_node);
702 last_start = state->start;
703 last_end = state->end;
706 * | ---- desired range ---- |
709 * Just lock what we found and keep going
711 if (state->start == start && state->end <= end) {
712 set = state->state & bits;
713 if (set && exclusive) {
714 *failed_start = state->start;
718 state->state |= bits;
719 start = state->end + 1;
720 merge_state(tree, state);
725 * | ---- desired range ---- |
728 * | ------------- state -------------- |
730 * We need to split the extent we found, and may flip bits on
733 * If the extent we found extends past our
734 * range, we just split and search again. It'll get split
735 * again the next time though.
737 * If the extent we found is inside our range, we set the
740 if (state->start < start) {
741 set = state->state & bits;
742 if (exclusive && set) {
743 *failed_start = start;
747 err = split_state(tree, state, prealloc, start);
748 BUG_ON(err == -EEXIST);
752 if (state->end <= end) {
753 state->state |= bits;
754 start = state->end + 1;
755 merge_state(tree, state);
757 start = state->start;
762 * | ---- desired range ---- |
763 * | state | or | state |
765 * There's a hole, we need to insert something in it and
766 * ignore the extent we found.
768 if (state->start > start) {
770 if (end < last_start)
773 this_end = last_start -1;
774 err = insert_state(tree, prealloc, start, this_end,
777 BUG_ON(err == -EEXIST);
780 start = this_end + 1;
784 * | ---- desired range ---- |
786 * We need to split the extent, and set the bit
789 if (state->start <= end && state->end > end) {
790 set = state->state & bits;
791 if (exclusive && set) {
792 *failed_start = start;
796 err = split_state(tree, state, prealloc, end + 1);
797 BUG_ON(err == -EEXIST);
799 prealloc->state |= bits;
800 merge_state(tree, prealloc);
808 write_unlock_irqrestore(&tree->lock, flags);
810 free_extent_state(prealloc);
817 write_unlock_irqrestore(&tree->lock, flags);
818 if (mask & __GFP_WAIT)
822 EXPORT_SYMBOL(set_extent_bit);
824 /* wrappers around set/clear extent bit */
825 int set_extent_dirty(struct extent_map_tree *tree, u64 start, u64 end,
828 return set_extent_bit(tree, start, end, EXTENT_DIRTY, 0, NULL,
831 EXPORT_SYMBOL(set_extent_dirty);
833 int set_extent_bits(struct extent_map_tree *tree, u64 start, u64 end,
834 int bits, gfp_t mask)
836 return set_extent_bit(tree, start, end, bits, 0, NULL,
839 EXPORT_SYMBOL(set_extent_bits);
841 int clear_extent_bits(struct extent_map_tree *tree, u64 start, u64 end,
842 int bits, gfp_t mask)
844 return clear_extent_bit(tree, start, end, bits, 0, 0, mask);
846 EXPORT_SYMBOL(clear_extent_bits);
848 int set_extent_delalloc(struct extent_map_tree *tree, u64 start, u64 end,
851 return set_extent_bit(tree, start, end,
852 EXTENT_DELALLOC | EXTENT_DIRTY, 0, NULL,
855 EXPORT_SYMBOL(set_extent_delalloc);
857 int clear_extent_dirty(struct extent_map_tree *tree, u64 start, u64 end,
860 return clear_extent_bit(tree, start, end,
861 EXTENT_DIRTY | EXTENT_DELALLOC, 0, 0, mask);
863 EXPORT_SYMBOL(clear_extent_dirty);
865 int set_extent_new(struct extent_map_tree *tree, u64 start, u64 end,
868 return set_extent_bit(tree, start, end, EXTENT_NEW, 0, NULL,
871 EXPORT_SYMBOL(set_extent_new);
873 int clear_extent_new(struct extent_map_tree *tree, u64 start, u64 end,
876 return clear_extent_bit(tree, start, end, EXTENT_NEW, 0, 0, mask);
878 EXPORT_SYMBOL(clear_extent_new);
880 int set_extent_uptodate(struct extent_map_tree *tree, u64 start, u64 end,
883 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, NULL,
886 EXPORT_SYMBOL(set_extent_uptodate);
888 int clear_extent_uptodate(struct extent_map_tree *tree, u64 start, u64 end,
891 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0, mask);
893 EXPORT_SYMBOL(clear_extent_uptodate);
895 int set_extent_writeback(struct extent_map_tree *tree, u64 start, u64 end,
898 return set_extent_bit(tree, start, end, EXTENT_WRITEBACK,
901 EXPORT_SYMBOL(set_extent_writeback);
903 int clear_extent_writeback(struct extent_map_tree *tree, u64 start, u64 end,
906 return clear_extent_bit(tree, start, end, EXTENT_WRITEBACK, 1, 0, mask);
908 EXPORT_SYMBOL(clear_extent_writeback);
910 int wait_on_extent_writeback(struct extent_map_tree *tree, u64 start, u64 end)
912 return wait_extent_bit(tree, start, end, EXTENT_WRITEBACK);
914 EXPORT_SYMBOL(wait_on_extent_writeback);
917 * locks a range in ascending order, waiting for any locked regions
918 * it hits on the way. [start,end] are inclusive, and this will sleep.
920 int lock_extent(struct extent_map_tree *tree, u64 start, u64 end, gfp_t mask)
925 err = set_extent_bit(tree, start, end, EXTENT_LOCKED, 1,
926 &failed_start, mask);
927 if (err == -EEXIST && (mask & __GFP_WAIT)) {
928 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
929 start = failed_start;
933 WARN_ON(start > end);
937 EXPORT_SYMBOL(lock_extent);
939 int unlock_extent(struct extent_map_tree *tree, u64 start, u64 end,
942 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, mask);
944 EXPORT_SYMBOL(unlock_extent);
947 * helper function to set pages and extents in the tree dirty
949 int set_range_dirty(struct extent_map_tree *tree, u64 start, u64 end)
951 unsigned long index = start >> PAGE_CACHE_SHIFT;
952 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
955 while (index <= end_index) {
956 page = find_get_page(tree->mapping, index);
958 __set_page_dirty_nobuffers(page);
959 page_cache_release(page);
962 set_extent_dirty(tree, start, end, GFP_NOFS);
965 EXPORT_SYMBOL(set_range_dirty);
968 * helper function to set both pages and extents in the tree writeback
970 int set_range_writeback(struct extent_map_tree *tree, u64 start, u64 end)
972 unsigned long index = start >> PAGE_CACHE_SHIFT;
973 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
976 while (index <= end_index) {
977 page = find_get_page(tree->mapping, index);
979 set_page_writeback(page);
980 page_cache_release(page);
983 set_extent_writeback(tree, start, end, GFP_NOFS);
986 EXPORT_SYMBOL(set_range_writeback);
988 int find_first_extent_bit(struct extent_map_tree *tree, u64 start,
989 u64 *start_ret, u64 *end_ret, int bits)
991 struct rb_node *node;
992 struct extent_state *state;
995 read_lock_irq(&tree->lock);
997 * this search will find all the extents that end after
1000 node = tree_search(&tree->state, start);
1001 if (!node || IS_ERR(node)) {
1006 state = rb_entry(node, struct extent_state, rb_node);
1007 if (state->end >= start && (state->state & bits)) {
1008 *start_ret = state->start;
1009 *end_ret = state->end;
1013 node = rb_next(node);
1018 read_unlock_irq(&tree->lock);
1021 EXPORT_SYMBOL(find_first_extent_bit);
1023 u64 find_lock_delalloc_range(struct extent_map_tree *tree,
1024 u64 start, u64 lock_start, u64 *end, u64 max_bytes)
1026 struct rb_node *node;
1027 struct extent_state *state;
1028 u64 cur_start = start;
1030 u64 total_bytes = 0;
1032 write_lock_irq(&tree->lock);
1034 * this search will find all the extents that end after
1038 node = tree_search(&tree->state, cur_start);
1039 if (!node || IS_ERR(node)) {
1044 state = rb_entry(node, struct extent_state, rb_node);
1045 if (state->start != cur_start) {
1048 if (!(state->state & EXTENT_DELALLOC)) {
1051 if (state->start >= lock_start) {
1052 if (state->state & EXTENT_LOCKED) {
1054 atomic_inc(&state->refs);
1055 prepare_to_wait(&state->wq, &wait,
1056 TASK_UNINTERRUPTIBLE);
1057 write_unlock_irq(&tree->lock);
1059 write_lock_irq(&tree->lock);
1060 finish_wait(&state->wq, &wait);
1061 free_extent_state(state);
1064 state->state |= EXTENT_LOCKED;
1068 cur_start = state->end + 1;
1069 node = rb_next(node);
1072 total_bytes += state->end - state->start + 1;
1073 if (total_bytes >= max_bytes)
1077 write_unlock_irq(&tree->lock);
1082 * helper function to lock both pages and extents in the tree.
1083 * pages must be locked first.
1085 int lock_range(struct extent_map_tree *tree, u64 start, u64 end)
1087 unsigned long index = start >> PAGE_CACHE_SHIFT;
1088 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1092 while (index <= end_index) {
1093 page = grab_cache_page(tree->mapping, index);
1099 err = PTR_ERR(page);
1104 lock_extent(tree, start, end, GFP_NOFS);
1109 * we failed above in getting the page at 'index', so we undo here
1110 * up to but not including the page at 'index'
1113 index = start >> PAGE_CACHE_SHIFT;
1114 while (index < end_index) {
1115 page = find_get_page(tree->mapping, index);
1117 page_cache_release(page);
1122 EXPORT_SYMBOL(lock_range);
1125 * helper function to unlock both pages and extents in the tree.
1127 int unlock_range(struct extent_map_tree *tree, u64 start, u64 end)
1129 unsigned long index = start >> PAGE_CACHE_SHIFT;
1130 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1133 while (index <= end_index) {
1134 page = find_get_page(tree->mapping, index);
1136 page_cache_release(page);
1139 unlock_extent(tree, start, end, GFP_NOFS);
1142 EXPORT_SYMBOL(unlock_range);
1144 int set_state_private(struct extent_map_tree *tree, u64 start, u64 private)
1146 struct rb_node *node;
1147 struct extent_state *state;
1150 write_lock_irq(&tree->lock);
1152 * this search will find all the extents that end after
1155 node = tree_search(&tree->state, start);
1156 if (!node || IS_ERR(node)) {
1160 state = rb_entry(node, struct extent_state, rb_node);
1161 if (state->start != start) {
1165 state->private = private;
1167 write_unlock_irq(&tree->lock);
1171 int get_state_private(struct extent_map_tree *tree, u64 start, u64 *private)
1173 struct rb_node *node;
1174 struct extent_state *state;
1177 read_lock_irq(&tree->lock);
1179 * this search will find all the extents that end after
1182 node = tree_search(&tree->state, start);
1183 if (!node || IS_ERR(node)) {
1187 state = rb_entry(node, struct extent_state, rb_node);
1188 if (state->start != start) {
1192 *private = state->private;
1194 read_unlock_irq(&tree->lock);
1199 * searches a range in the state tree for a given mask.
1200 * If 'filled' == 1, this returns 1 only if ever extent in the tree
1201 * has the bits set. Otherwise, 1 is returned if any bit in the
1202 * range is found set.
1204 int test_range_bit(struct extent_map_tree *tree, u64 start, u64 end,
1205 int bits, int filled)
1207 struct extent_state *state = NULL;
1208 struct rb_node *node;
1211 read_lock_irq(&tree->lock);
1212 node = tree_search(&tree->state, start);
1213 while (node && start <= end) {
1214 state = rb_entry(node, struct extent_state, rb_node);
1216 if (filled && state->start > start) {
1221 if (state->start > end)
1224 if (state->state & bits) {
1228 } else if (filled) {
1232 start = state->end + 1;
1235 node = rb_next(node);
1237 read_unlock_irq(&tree->lock);
1240 EXPORT_SYMBOL(test_range_bit);
1243 * helper function to set a given page up to date if all the
1244 * extents in the tree for that page are up to date
1246 static int check_page_uptodate(struct extent_map_tree *tree,
1249 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1250 u64 end = start + PAGE_CACHE_SIZE - 1;
1251 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1))
1252 SetPageUptodate(page);
1257 * helper function to unlock a page if all the extents in the tree
1258 * for that page are unlocked
1260 static int check_page_locked(struct extent_map_tree *tree,
1263 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1264 u64 end = start + PAGE_CACHE_SIZE - 1;
1265 if (!test_range_bit(tree, start, end, EXTENT_LOCKED, 0))
1271 * helper function to end page writeback if all the extents
1272 * in the tree for that page are done with writeback
1274 static int check_page_writeback(struct extent_map_tree *tree,
1277 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1278 u64 end = start + PAGE_CACHE_SIZE - 1;
1279 if (!test_range_bit(tree, start, end, EXTENT_WRITEBACK, 0))
1280 end_page_writeback(page);
1284 /* lots and lots of room for performance fixes in the end_bio funcs */
1287 * after a writepage IO is done, we need to:
1288 * clear the uptodate bits on error
1289 * clear the writeback bits in the extent tree for this IO
1290 * end_page_writeback if the page has no more pending IO
1292 * Scheduling is not allowed, so the extent state tree is expected
1293 * to have one and only one object corresponding to this IO.
1295 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
1296 static void end_bio_extent_writepage(struct bio *bio, int err)
1298 static int end_bio_extent_writepage(struct bio *bio,
1299 unsigned int bytes_done, int err)
1302 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1303 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1304 struct extent_map_tree *tree = bio->bi_private;
1309 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
1315 struct page *page = bvec->bv_page;
1316 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
1318 end = start + bvec->bv_len - 1;
1320 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
1325 if (--bvec >= bio->bi_io_vec)
1326 prefetchw(&bvec->bv_page->flags);
1329 clear_extent_uptodate(tree, start, end, GFP_ATOMIC);
1330 ClearPageUptodate(page);
1333 clear_extent_writeback(tree, start, end, GFP_ATOMIC);
1336 end_page_writeback(page);
1338 check_page_writeback(tree, page);
1339 if (tree->ops && tree->ops->writepage_end_io_hook)
1340 tree->ops->writepage_end_io_hook(page, start, end);
1341 } while (bvec >= bio->bi_io_vec);
1344 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
1350 * after a readpage IO is done, we need to:
1351 * clear the uptodate bits on error
1352 * set the uptodate bits if things worked
1353 * set the page up to date if all extents in the tree are uptodate
1354 * clear the lock bit in the extent tree
1355 * unlock the page if there are no other extents locked for it
1357 * Scheduling is not allowed, so the extent state tree is expected
1358 * to have one and only one object corresponding to this IO.
1360 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
1361 static void end_bio_extent_readpage(struct bio *bio, int err)
1363 static int end_bio_extent_readpage(struct bio *bio,
1364 unsigned int bytes_done, int err)
1367 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1368 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1369 struct extent_map_tree *tree = bio->bi_private;
1375 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
1381 struct page *page = bvec->bv_page;
1382 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
1384 end = start + bvec->bv_len - 1;
1386 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
1391 if (--bvec >= bio->bi_io_vec)
1392 prefetchw(&bvec->bv_page->flags);
1394 if (uptodate && tree->ops && tree->ops->readpage_end_io_hook) {
1395 ret = tree->ops->readpage_end_io_hook(page, start, end);
1400 set_extent_uptodate(tree, start, end, GFP_ATOMIC);
1402 SetPageUptodate(page);
1404 check_page_uptodate(tree, page);
1406 ClearPageUptodate(page);
1410 unlock_extent(tree, start, end, GFP_ATOMIC);
1415 check_page_locked(tree, page);
1416 } while (bvec >= bio->bi_io_vec);
1419 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
1425 * IO done from prepare_write is pretty simple, we just unlock
1426 * the structs in the extent tree when done, and set the uptodate bits
1429 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,23)
1430 static void end_bio_extent_preparewrite(struct bio *bio, int err)
1432 static int end_bio_extent_preparewrite(struct bio *bio,
1433 unsigned int bytes_done, int err)
1436 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1437 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1438 struct extent_map_tree *tree = bio->bi_private;
1442 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
1448 struct page *page = bvec->bv_page;
1449 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
1451 end = start + bvec->bv_len - 1;
1453 if (--bvec >= bio->bi_io_vec)
1454 prefetchw(&bvec->bv_page->flags);
1457 set_extent_uptodate(tree, start, end, GFP_ATOMIC);
1459 ClearPageUptodate(page);
1463 unlock_extent(tree, start, end, GFP_ATOMIC);
1465 } while (bvec >= bio->bi_io_vec);
1468 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,23)
1474 extent_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
1479 bio = bio_alloc(gfp_flags, nr_vecs);
1481 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
1482 while (!bio && (nr_vecs /= 2))
1483 bio = bio_alloc(gfp_flags, nr_vecs);
1487 bio->bi_bdev = bdev;
1488 bio->bi_sector = first_sector;
1493 static int submit_one_bio(int rw, struct bio *bio)
1497 submit_bio(rw, bio);
1498 if (bio_flagged(bio, BIO_EOPNOTSUPP))
1504 static int submit_extent_page(int rw, struct extent_map_tree *tree,
1505 struct page *page, sector_t sector,
1506 size_t size, unsigned long offset,
1507 struct block_device *bdev,
1508 struct bio **bio_ret,
1509 unsigned long max_pages,
1510 bio_end_io_t end_io_func)
1516 if (bio_ret && *bio_ret) {
1518 if (bio->bi_sector + (bio->bi_size >> 9) != sector ||
1519 bio_add_page(bio, page, size, offset) < size) {
1520 ret = submit_one_bio(rw, bio);
1526 nr = min_t(int, max_pages, bio_get_nr_vecs(bdev));
1527 bio = extent_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
1529 printk("failed to allocate bio nr %d\n", nr);
1531 bio_add_page(bio, page, size, offset);
1532 bio->bi_end_io = end_io_func;
1533 bio->bi_private = tree;
1537 ret = submit_one_bio(rw, bio);
1543 void set_page_extent_mapped(struct page *page)
1545 if (!PagePrivate(page)) {
1546 SetPagePrivate(page);
1547 WARN_ON(!page->mapping->a_ops->invalidatepage);
1548 set_page_private(page, EXTENT_PAGE_PRIVATE);
1549 page_cache_get(page);
1554 * basic readpage implementation. Locked extent state structs are inserted
1555 * into the tree that are removed when the IO is done (by the end_io
1558 static int __extent_read_full_page(struct extent_map_tree *tree,
1560 get_extent_t *get_extent,
1563 struct inode *inode = page->mapping->host;
1564 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1565 u64 page_end = start + PAGE_CACHE_SIZE - 1;
1569 u64 last_byte = i_size_read(inode);
1573 struct extent_map *em;
1574 struct block_device *bdev;
1577 size_t page_offset = 0;
1579 size_t blocksize = inode->i_sb->s_blocksize;
1581 set_page_extent_mapped(page);
1584 lock_extent(tree, start, end, GFP_NOFS);
1586 while (cur <= end) {
1587 if (cur >= last_byte) {
1588 iosize = PAGE_CACHE_SIZE - page_offset;
1589 zero_user_page(page, page_offset, iosize, KM_USER0);
1590 set_extent_uptodate(tree, cur, cur + iosize - 1,
1592 unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
1595 em = get_extent(inode, page, page_offset, cur, end, 0);
1596 if (IS_ERR(em) || !em) {
1598 unlock_extent(tree, cur, end, GFP_NOFS);
1602 extent_offset = cur - em->start;
1603 BUG_ON(em->end < cur);
1606 iosize = min(em->end - cur, end - cur) + 1;
1607 cur_end = min(em->end, end);
1608 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
1609 sector = (em->block_start + extent_offset) >> 9;
1611 block_start = em->block_start;
1612 free_extent_map(em);
1615 /* we've found a hole, just zero and go on */
1616 if (block_start == EXTENT_MAP_HOLE) {
1617 zero_user_page(page, page_offset, iosize, KM_USER0);
1618 set_extent_uptodate(tree, cur, cur + iosize - 1,
1620 unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
1622 page_offset += iosize;
1625 /* the get_extent function already copied into the page */
1626 if (test_range_bit(tree, cur, cur_end, EXTENT_UPTODATE, 1)) {
1627 unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
1629 page_offset += iosize;
1634 if (tree->ops && tree->ops->readpage_io_hook) {
1635 ret = tree->ops->readpage_io_hook(page, cur,
1639 unsigned long nr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
1641 ret = submit_extent_page(READ, tree, page,
1642 sector, iosize, page_offset,
1644 end_bio_extent_readpage);
1649 page_offset += iosize;
1653 if (!PageError(page))
1654 SetPageUptodate(page);
1660 int extent_read_full_page(struct extent_map_tree *tree, struct page *page,
1661 get_extent_t *get_extent)
1663 struct bio *bio = NULL;
1666 ret = __extent_read_full_page(tree, page, get_extent, &bio);
1668 submit_one_bio(READ, bio);
1671 EXPORT_SYMBOL(extent_read_full_page);
1674 * the writepage semantics are similar to regular writepage. extent
1675 * records are inserted to lock ranges in the tree, and as dirty areas
1676 * are found, they are marked writeback. Then the lock bits are removed
1677 * and the end_io handler clears the writeback ranges
1679 static int __extent_writepage(struct page *page, struct writeback_control *wbc,
1682 struct inode *inode = page->mapping->host;
1683 struct extent_page_data *epd = data;
1684 struct extent_map_tree *tree = epd->tree;
1685 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1686 u64 page_end = start + PAGE_CACHE_SIZE - 1;
1690 u64 last_byte = i_size_read(inode);
1694 struct extent_map *em;
1695 struct block_device *bdev;
1698 size_t page_offset = 0;
1700 loff_t i_size = i_size_read(inode);
1701 unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
1705 WARN_ON(!PageLocked(page));
1706 if (page->index > end_index) {
1707 clear_extent_dirty(tree, start, page_end, GFP_NOFS);
1712 if (page->index == end_index) {
1713 size_t offset = i_size & (PAGE_CACHE_SIZE - 1);
1714 zero_user_page(page, offset,
1715 PAGE_CACHE_SIZE - offset, KM_USER0);
1718 set_page_extent_mapped(page);
1720 lock_extent(tree, start, page_end, GFP_NOFS);
1721 nr_delalloc = find_lock_delalloc_range(tree, start, page_end + 1,
1725 tree->ops->fill_delalloc(inode, start, delalloc_end);
1726 if (delalloc_end >= page_end + 1) {
1727 clear_extent_bit(tree, page_end + 1, delalloc_end,
1728 EXTENT_LOCKED | EXTENT_DELALLOC,
1731 clear_extent_bit(tree, start, page_end, EXTENT_DELALLOC,
1733 if (test_range_bit(tree, start, page_end, EXTENT_DELALLOC, 0)) {
1734 printk("found delalloc bits after clear extent_bit\n");
1736 } else if (test_range_bit(tree, start, page_end, EXTENT_DELALLOC, 0)) {
1737 printk("found delalloc bits after find_delalloc_range returns 0\n");
1741 if (test_range_bit(tree, start, page_end, EXTENT_DELALLOC, 0)) {
1742 printk("found delalloc bits after lock_extent\n");
1745 if (last_byte <= start) {
1746 clear_extent_dirty(tree, start, page_end, GFP_NOFS);
1750 set_extent_uptodate(tree, start, page_end, GFP_NOFS);
1751 blocksize = inode->i_sb->s_blocksize;
1753 while (cur <= end) {
1754 if (cur >= last_byte) {
1755 clear_extent_dirty(tree, cur, page_end, GFP_NOFS);
1758 em = epd->get_extent(inode, page, page_offset, cur, end, 1);
1759 if (IS_ERR(em) || !em) {
1764 extent_offset = cur - em->start;
1765 BUG_ON(em->end < cur);
1767 iosize = min(em->end - cur, end - cur) + 1;
1768 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
1769 sector = (em->block_start + extent_offset) >> 9;
1771 block_start = em->block_start;
1772 free_extent_map(em);
1775 if (block_start == EXTENT_MAP_HOLE ||
1776 block_start == EXTENT_MAP_INLINE) {
1777 clear_extent_dirty(tree, cur,
1778 cur + iosize - 1, GFP_NOFS);
1780 page_offset += iosize;
1784 /* leave this out until we have a page_mkwrite call */
1785 if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
1788 page_offset += iosize;
1791 clear_extent_dirty(tree, cur, cur + iosize - 1, GFP_NOFS);
1792 if (tree->ops && tree->ops->writepage_io_hook) {
1793 ret = tree->ops->writepage_io_hook(page, cur,
1801 unsigned long nr = end_index + 1;
1802 set_range_writeback(tree, cur, cur + iosize - 1);
1804 ret = submit_extent_page(WRITE, tree, page, sector,
1805 iosize, page_offset, bdev,
1807 end_bio_extent_writepage);
1812 page_offset += iosize;
1816 unlock_extent(tree, start, page_end, GFP_NOFS);
1821 int extent_write_full_page(struct extent_map_tree *tree, struct page *page,
1822 get_extent_t *get_extent,
1823 struct writeback_control *wbc)
1826 struct extent_page_data epd = {
1829 .get_extent = get_extent,
1832 ret = __extent_writepage(page, wbc, &epd);
1834 submit_one_bio(WRITE, epd.bio);
1837 EXPORT_SYMBOL(extent_write_full_page);
1839 int extent_writepages(struct extent_map_tree *tree,
1840 struct address_space *mapping,
1841 get_extent_t *get_extent,
1842 struct writeback_control *wbc)
1845 struct extent_page_data epd = {
1848 .get_extent = get_extent,
1851 ret = write_cache_pages(mapping, wbc, __extent_writepage, &epd);
1853 submit_one_bio(WRITE, epd.bio);
1856 EXPORT_SYMBOL(extent_writepages);
1858 int extent_readpages(struct extent_map_tree *tree,
1859 struct address_space *mapping,
1860 struct list_head *pages, unsigned nr_pages,
1861 get_extent_t get_extent)
1863 struct bio *bio = NULL;
1865 struct pagevec pvec;
1867 pagevec_init(&pvec, 0);
1868 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
1869 struct page *page = list_entry(pages->prev, struct page, lru);
1871 prefetchw(&page->flags);
1872 list_del(&page->lru);
1874 * what we want to do here is call add_to_page_cache_lru,
1875 * but that isn't exported, so we reproduce it here
1877 if (!add_to_page_cache(page, mapping,
1878 page->index, GFP_KERNEL)) {
1880 /* open coding of lru_cache_add, also not exported */
1881 page_cache_get(page);
1882 if (!pagevec_add(&pvec, page))
1883 __pagevec_lru_add(&pvec);
1884 __extent_read_full_page(tree, page, get_extent, &bio);
1886 page_cache_release(page);
1888 if (pagevec_count(&pvec))
1889 __pagevec_lru_add(&pvec);
1890 BUG_ON(!list_empty(pages));
1892 submit_one_bio(READ, bio);
1895 EXPORT_SYMBOL(extent_readpages);
1898 * basic invalidatepage code, this waits on any locked or writeback
1899 * ranges corresponding to the page, and then deletes any extent state
1900 * records from the tree
1902 int extent_invalidatepage(struct extent_map_tree *tree,
1903 struct page *page, unsigned long offset)
1905 u64 start = ((u64)page->index << PAGE_CACHE_SHIFT);
1906 u64 end = start + PAGE_CACHE_SIZE - 1;
1907 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
1909 start += (offset + blocksize -1) & ~(blocksize - 1);
1913 lock_extent(tree, start, end, GFP_NOFS);
1914 wait_on_extent_writeback(tree, start, end);
1915 clear_extent_bit(tree, start, end,
1916 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC,
1920 EXPORT_SYMBOL(extent_invalidatepage);
1923 * simple commit_write call, set_range_dirty is used to mark both
1924 * the pages and the extent records as dirty
1926 int extent_commit_write(struct extent_map_tree *tree,
1927 struct inode *inode, struct page *page,
1928 unsigned from, unsigned to)
1930 loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
1932 set_page_extent_mapped(page);
1933 set_page_dirty(page);
1935 if (pos > inode->i_size) {
1936 i_size_write(inode, pos);
1937 mark_inode_dirty(inode);
1941 EXPORT_SYMBOL(extent_commit_write);
1943 int extent_prepare_write(struct extent_map_tree *tree,
1944 struct inode *inode, struct page *page,
1945 unsigned from, unsigned to, get_extent_t *get_extent)
1947 u64 page_start = (u64)page->index << PAGE_CACHE_SHIFT;
1948 u64 page_end = page_start + PAGE_CACHE_SIZE - 1;
1950 u64 orig_block_start;
1953 struct extent_map *em;
1954 unsigned blocksize = 1 << inode->i_blkbits;
1955 size_t page_offset = 0;
1956 size_t block_off_start;
1957 size_t block_off_end;
1963 set_page_extent_mapped(page);
1965 block_start = (page_start + from) & ~((u64)blocksize - 1);
1966 block_end = (page_start + to - 1) | (blocksize - 1);
1967 orig_block_start = block_start;
1969 lock_extent(tree, page_start, page_end, GFP_NOFS);
1970 while(block_start <= block_end) {
1971 em = get_extent(inode, page, page_offset, block_start,
1973 if (IS_ERR(em) || !em) {
1976 cur_end = min(block_end, em->end);
1977 block_off_start = block_start & (PAGE_CACHE_SIZE - 1);
1978 block_off_end = block_off_start + blocksize;
1979 isnew = clear_extent_new(tree, block_start, cur_end, GFP_NOFS);
1981 if (!PageUptodate(page) && isnew &&
1982 (block_off_end > to || block_off_start < from)) {
1985 kaddr = kmap_atomic(page, KM_USER0);
1986 if (block_off_end > to)
1987 memset(kaddr + to, 0, block_off_end - to);
1988 if (block_off_start < from)
1989 memset(kaddr + block_off_start, 0,
1990 from - block_off_start);
1991 flush_dcache_page(page);
1992 kunmap_atomic(kaddr, KM_USER0);
1994 if (!isnew && !PageUptodate(page) &&
1995 (block_off_end > to || block_off_start < from) &&
1996 !test_range_bit(tree, block_start, cur_end,
1997 EXTENT_UPTODATE, 1)) {
1999 u64 extent_offset = block_start - em->start;
2001 sector = (em->block_start + extent_offset) >> 9;
2002 iosize = (cur_end - block_start + blocksize - 1) &
2003 ~((u64)blocksize - 1);
2005 * we've already got the extent locked, but we
2006 * need to split the state such that our end_bio
2007 * handler can clear the lock.
2009 set_extent_bit(tree, block_start,
2010 block_start + iosize - 1,
2011 EXTENT_LOCKED, 0, NULL, GFP_NOFS);
2012 ret = submit_extent_page(READ, tree, page,
2013 sector, iosize, page_offset, em->bdev,
2015 end_bio_extent_preparewrite);
2017 block_start = block_start + iosize;
2019 set_extent_uptodate(tree, block_start, cur_end,
2021 unlock_extent(tree, block_start, cur_end, GFP_NOFS);
2022 block_start = cur_end + 1;
2024 page_offset = block_start & (PAGE_CACHE_SIZE - 1);
2025 free_extent_map(em);
2028 wait_extent_bit(tree, orig_block_start,
2029 block_end, EXTENT_LOCKED);
2031 check_page_uptodate(tree, page);
2033 /* FIXME, zero out newly allocated blocks on error */
2036 EXPORT_SYMBOL(extent_prepare_write);
2039 * a helper for releasepage. As long as there are no locked extents
2040 * in the range corresponding to the page, both state records and extent
2041 * map records are removed
2043 int try_release_extent_mapping(struct extent_map_tree *tree, struct page *page)
2045 struct extent_map *em;
2046 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2047 u64 end = start + PAGE_CACHE_SIZE - 1;
2048 u64 orig_start = start;
2051 while (start <= end) {
2052 em = lookup_extent_mapping(tree, start, end);
2053 if (!em || IS_ERR(em))
2055 if (!test_range_bit(tree, em->start, em->end,
2056 EXTENT_LOCKED, 0)) {
2057 remove_extent_mapping(tree, em);
2058 /* once for the rb tree */
2059 free_extent_map(em);
2061 start = em->end + 1;
2063 free_extent_map(em);
2065 if (test_range_bit(tree, orig_start, end, EXTENT_LOCKED, 0))
2068 clear_extent_bit(tree, orig_start, end, EXTENT_UPTODATE,
2072 EXPORT_SYMBOL(try_release_extent_mapping);
2074 sector_t extent_bmap(struct address_space *mapping, sector_t iblock,
2075 get_extent_t *get_extent)
2077 struct inode *inode = mapping->host;
2078 u64 start = iblock << inode->i_blkbits;
2079 u64 end = start + (1 << inode->i_blkbits) - 1;
2080 sector_t sector = 0;
2081 struct extent_map *em;
2083 em = get_extent(inode, NULL, 0, start, end, 0);
2084 if (!em || IS_ERR(em))
2087 if (em->block_start == EXTENT_MAP_INLINE ||
2088 em->block_start == EXTENT_MAP_HOLE)
2091 sector = (em->block_start + start - em->start) >> inode->i_blkbits;
2093 free_extent_map(em);
2097 static int add_lru(struct extent_map_tree *tree, struct extent_buffer *eb)
2099 if (list_empty(&eb->lru)) {
2100 extent_buffer_get(eb);
2101 list_add(&eb->lru, &tree->buffer_lru);
2103 if (tree->lru_size >= BUFFER_LRU_MAX) {
2104 struct extent_buffer *rm;
2105 rm = list_entry(tree->buffer_lru.prev,
2106 struct extent_buffer, lru);
2108 list_del_init(&rm->lru);
2109 free_extent_buffer(rm);
2112 list_move(&eb->lru, &tree->buffer_lru);
2115 static struct extent_buffer *find_lru(struct extent_map_tree *tree,
2116 u64 start, unsigned long len)
2118 struct list_head *lru = &tree->buffer_lru;
2119 struct list_head *cur = lru->next;
2120 struct extent_buffer *eb;
2122 if (list_empty(lru))
2126 eb = list_entry(cur, struct extent_buffer, lru);
2127 if (eb->start == start && eb->len == len) {
2128 extent_buffer_get(eb);
2132 } while (cur != lru);
2136 static inline unsigned long num_extent_pages(u64 start, u64 len)
2138 return ((start + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT) -
2139 (start >> PAGE_CACHE_SHIFT);
2142 static inline struct page *extent_buffer_page(struct extent_buffer *eb,
2146 struct address_space *mapping;
2149 return eb->first_page;
2150 i += eb->start >> PAGE_CACHE_SHIFT;
2151 mapping = eb->first_page->mapping;
2152 read_lock_irq(&mapping->tree_lock);
2153 p = radix_tree_lookup(&mapping->page_tree, i);
2154 read_unlock_irq(&mapping->tree_lock);
2158 static struct extent_buffer *__alloc_extent_buffer(struct extent_map_tree *tree,
2163 struct extent_buffer *eb = NULL;
2165 spin_lock(&tree->lru_lock);
2166 eb = find_lru(tree, start, len);
2167 spin_unlock(&tree->lru_lock);
2172 eb = kmem_cache_zalloc(extent_buffer_cache, mask);
2173 INIT_LIST_HEAD(&eb->lru);
2176 atomic_set(&eb->refs, 1);
2181 static void __free_extent_buffer(struct extent_buffer *eb)
2183 kmem_cache_free(extent_buffer_cache, eb);
2186 struct extent_buffer *alloc_extent_buffer(struct extent_map_tree *tree,
2187 u64 start, unsigned long len,
2191 unsigned long num_pages = num_extent_pages(start, len);
2193 unsigned long index = start >> PAGE_CACHE_SHIFT;
2194 struct extent_buffer *eb;
2196 struct address_space *mapping = tree->mapping;
2199 eb = __alloc_extent_buffer(tree, start, len, mask);
2200 if (!eb || IS_ERR(eb))
2203 if (eb->flags & EXTENT_BUFFER_FILLED)
2207 eb->first_page = page0;
2210 page_cache_get(page0);
2211 mark_page_accessed(page0);
2212 set_page_extent_mapped(page0);
2213 WARN_ON(!PageUptodate(page0));
2214 set_page_private(page0, EXTENT_PAGE_PRIVATE_FIRST_PAGE |
2219 for (; i < num_pages; i++, index++) {
2220 p = find_or_create_page(mapping, index, mask | __GFP_HIGHMEM);
2225 set_page_extent_mapped(p);
2226 mark_page_accessed(p);
2229 set_page_private(p, EXTENT_PAGE_PRIVATE_FIRST_PAGE |
2232 set_page_private(p, EXTENT_PAGE_PRIVATE);
2234 if (!PageUptodate(p))
2239 eb->flags |= EXTENT_UPTODATE;
2240 eb->flags |= EXTENT_BUFFER_FILLED;
2243 spin_lock(&tree->lru_lock);
2245 spin_unlock(&tree->lru_lock);
2249 spin_lock(&tree->lru_lock);
2250 list_del_init(&eb->lru);
2251 spin_unlock(&tree->lru_lock);
2252 if (!atomic_dec_and_test(&eb->refs))
2254 for (index = 1; index < i; index++) {
2255 page_cache_release(extent_buffer_page(eb, index));
2258 page_cache_release(extent_buffer_page(eb, 0));
2259 __free_extent_buffer(eb);
2262 EXPORT_SYMBOL(alloc_extent_buffer);
2264 struct extent_buffer *find_extent_buffer(struct extent_map_tree *tree,
2265 u64 start, unsigned long len,
2268 unsigned long num_pages = num_extent_pages(start, len);
2270 unsigned long index = start >> PAGE_CACHE_SHIFT;
2271 struct extent_buffer *eb;
2273 struct address_space *mapping = tree->mapping;
2276 eb = __alloc_extent_buffer(tree, start, len, mask);
2277 if (!eb || IS_ERR(eb))
2280 if (eb->flags & EXTENT_BUFFER_FILLED)
2283 for (i = 0; i < num_pages; i++, index++) {
2284 p = find_lock_page(mapping, index);
2288 set_page_extent_mapped(p);
2289 mark_page_accessed(p);
2293 set_page_private(p, EXTENT_PAGE_PRIVATE_FIRST_PAGE |
2296 set_page_private(p, EXTENT_PAGE_PRIVATE);
2299 if (!PageUptodate(p))
2304 eb->flags |= EXTENT_UPTODATE;
2305 eb->flags |= EXTENT_BUFFER_FILLED;
2308 spin_lock(&tree->lru_lock);
2310 spin_unlock(&tree->lru_lock);
2313 spin_lock(&tree->lru_lock);
2314 list_del_init(&eb->lru);
2315 spin_unlock(&tree->lru_lock);
2316 if (!atomic_dec_and_test(&eb->refs))
2318 for (index = 1; index < i; index++) {
2319 page_cache_release(extent_buffer_page(eb, index));
2322 page_cache_release(extent_buffer_page(eb, 0));
2323 __free_extent_buffer(eb);
2326 EXPORT_SYMBOL(find_extent_buffer);
2328 void free_extent_buffer(struct extent_buffer *eb)
2331 unsigned long num_pages;
2336 if (!atomic_dec_and_test(&eb->refs))
2339 WARN_ON(!list_empty(&eb->lru));
2340 num_pages = num_extent_pages(eb->start, eb->len);
2342 for (i = 1; i < num_pages; i++) {
2343 page_cache_release(extent_buffer_page(eb, i));
2345 page_cache_release(extent_buffer_page(eb, 0));
2346 __free_extent_buffer(eb);
2348 EXPORT_SYMBOL(free_extent_buffer);
2350 int clear_extent_buffer_dirty(struct extent_map_tree *tree,
2351 struct extent_buffer *eb)
2355 unsigned long num_pages;
2358 u64 start = eb->start;
2359 u64 end = start + eb->len - 1;
2361 set = clear_extent_dirty(tree, start, end, GFP_NOFS);
2362 num_pages = num_extent_pages(eb->start, eb->len);
2364 for (i = 0; i < num_pages; i++) {
2365 page = extent_buffer_page(eb, i);
2368 * if we're on the last page or the first page and the
2369 * block isn't aligned on a page boundary, do extra checks
2370 * to make sure we don't clean page that is partially dirty
2372 if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) ||
2373 ((i == num_pages - 1) &&
2374 ((eb->start + eb->len) & (PAGE_CACHE_SIZE - 1)))) {
2375 start = (u64)page->index << PAGE_CACHE_SHIFT;
2376 end = start + PAGE_CACHE_SIZE - 1;
2377 if (test_range_bit(tree, start, end,
2383 clear_page_dirty_for_io(page);
2388 EXPORT_SYMBOL(clear_extent_buffer_dirty);
2390 int wait_on_extent_buffer_writeback(struct extent_map_tree *tree,
2391 struct extent_buffer *eb)
2393 return wait_on_extent_writeback(tree, eb->start,
2394 eb->start + eb->len - 1);
2396 EXPORT_SYMBOL(wait_on_extent_buffer_writeback);
2398 int set_extent_buffer_dirty(struct extent_map_tree *tree,
2399 struct extent_buffer *eb)
2402 unsigned long num_pages;
2404 num_pages = num_extent_pages(eb->start, eb->len);
2405 for (i = 0; i < num_pages; i++) {
2406 struct page *page = extent_buffer_page(eb, i);
2407 /* writepage may need to do something special for the
2408 * first page, we have to make sure page->private is
2409 * properly set. releasepage may drop page->private
2410 * on us if the page isn't already dirty.
2414 set_page_private(page,
2415 EXTENT_PAGE_PRIVATE_FIRST_PAGE |
2418 __set_page_dirty_nobuffers(extent_buffer_page(eb, i));
2422 return set_extent_dirty(tree, eb->start,
2423 eb->start + eb->len - 1, GFP_NOFS);
2425 EXPORT_SYMBOL(set_extent_buffer_dirty);
2427 int set_extent_buffer_uptodate(struct extent_map_tree *tree,
2428 struct extent_buffer *eb)
2432 unsigned long num_pages;
2434 num_pages = num_extent_pages(eb->start, eb->len);
2436 set_extent_uptodate(tree, eb->start, eb->start + eb->len - 1,
2438 for (i = 0; i < num_pages; i++) {
2439 page = extent_buffer_page(eb, i);
2440 if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) ||
2441 ((i == num_pages - 1) &&
2442 ((eb->start + eb->len) & (PAGE_CACHE_SIZE - 1)))) {
2443 check_page_uptodate(tree, page);
2446 SetPageUptodate(page);
2450 EXPORT_SYMBOL(set_extent_buffer_uptodate);
2452 int extent_buffer_uptodate(struct extent_map_tree *tree,
2453 struct extent_buffer *eb)
2455 if (eb->flags & EXTENT_UPTODATE)
2457 return test_range_bit(tree, eb->start, eb->start + eb->len - 1,
2458 EXTENT_UPTODATE, 1);
2460 EXPORT_SYMBOL(extent_buffer_uptodate);
2462 int read_extent_buffer_pages(struct extent_map_tree *tree,
2463 struct extent_buffer *eb,
2468 unsigned long start_i;
2472 unsigned long num_pages;
2474 if (eb->flags & EXTENT_UPTODATE)
2477 if (0 && test_range_bit(tree, eb->start, eb->start + eb->len - 1,
2478 EXTENT_UPTODATE, 1)) {
2483 WARN_ON(start < eb->start);
2484 start_i = (start >> PAGE_CACHE_SHIFT) -
2485 (eb->start >> PAGE_CACHE_SHIFT);
2490 num_pages = num_extent_pages(eb->start, eb->len);
2491 for (i = start_i; i < num_pages; i++) {
2492 page = extent_buffer_page(eb, i);
2493 if (PageUptodate(page)) {
2497 if (TestSetPageLocked(page)) {
2503 if (!PageUptodate(page)) {
2504 err = page->mapping->a_ops->readpage(NULL, page);
2517 for (i = start_i; i < num_pages; i++) {
2518 page = extent_buffer_page(eb, i);
2519 wait_on_page_locked(page);
2520 if (!PageUptodate(page)) {
2525 eb->flags |= EXTENT_UPTODATE;
2528 EXPORT_SYMBOL(read_extent_buffer_pages);
2530 void read_extent_buffer(struct extent_buffer *eb, void *dstv,
2531 unsigned long start,
2538 char *dst = (char *)dstv;
2539 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
2540 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
2541 unsigned long num_pages = num_extent_pages(eb->start, eb->len);
2543 WARN_ON(start > eb->len);
2544 WARN_ON(start + len > eb->start + eb->len);
2546 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
2549 page = extent_buffer_page(eb, i);
2550 if (!PageUptodate(page)) {
2551 printk("page %lu not up to date i %lu, total %lu, len %lu\n", page->index, i, num_pages, eb->len);
2554 WARN_ON(!PageUptodate(page));
2556 cur = min(len, (PAGE_CACHE_SIZE - offset));
2557 kaddr = kmap_atomic(page, KM_USER1);
2558 memcpy(dst, kaddr + offset, cur);
2559 kunmap_atomic(kaddr, KM_USER1);
2567 EXPORT_SYMBOL(read_extent_buffer);
2569 int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
2570 unsigned long min_len, char **token, char **map,
2571 unsigned long *map_start,
2572 unsigned long *map_len, int km)
2574 size_t offset = start & (PAGE_CACHE_SIZE - 1);
2577 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
2578 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
2579 unsigned long end_i = (start_offset + start + min_len - 1) >>
2586 offset = start_offset;
2590 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
2592 if (start + min_len > eb->len) {
2593 printk("bad mapping eb start %Lu len %lu, wanted %lu %lu\n", eb->start, eb->len, start, min_len);
2597 p = extent_buffer_page(eb, i);
2598 WARN_ON(!PageUptodate(p));
2599 kaddr = kmap_atomic(p, km);
2601 *map = kaddr + offset;
2602 *map_len = PAGE_CACHE_SIZE - offset;
2605 EXPORT_SYMBOL(map_private_extent_buffer);
2607 int map_extent_buffer(struct extent_buffer *eb, unsigned long start,
2608 unsigned long min_len,
2609 char **token, char **map,
2610 unsigned long *map_start,
2611 unsigned long *map_len, int km)
2615 if (eb->map_token) {
2616 unmap_extent_buffer(eb, eb->map_token, km);
2617 eb->map_token = NULL;
2620 err = map_private_extent_buffer(eb, start, min_len, token, map,
2621 map_start, map_len, km);
2623 eb->map_token = *token;
2625 eb->map_start = *map_start;
2626 eb->map_len = *map_len;
2630 EXPORT_SYMBOL(map_extent_buffer);
2632 void unmap_extent_buffer(struct extent_buffer *eb, char *token, int km)
2634 kunmap_atomic(token, km);
2636 EXPORT_SYMBOL(unmap_extent_buffer);
2638 int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
2639 unsigned long start,
2646 char *ptr = (char *)ptrv;
2647 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
2648 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
2651 WARN_ON(start > eb->len);
2652 WARN_ON(start + len > eb->start + eb->len);
2654 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
2657 page = extent_buffer_page(eb, i);
2658 WARN_ON(!PageUptodate(page));
2660 cur = min(len, (PAGE_CACHE_SIZE - offset));
2662 kaddr = kmap_atomic(page, KM_USER0);
2663 ret = memcmp(ptr, kaddr + offset, cur);
2664 kunmap_atomic(kaddr, KM_USER0);
2675 EXPORT_SYMBOL(memcmp_extent_buffer);
2677 void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
2678 unsigned long start, unsigned long len)
2684 char *src = (char *)srcv;
2685 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
2686 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
2688 WARN_ON(start > eb->len);
2689 WARN_ON(start + len > eb->start + eb->len);
2691 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
2694 page = extent_buffer_page(eb, i);
2695 WARN_ON(!PageUptodate(page));
2697 cur = min(len, PAGE_CACHE_SIZE - offset);
2698 kaddr = kmap_atomic(page, KM_USER1);
2699 memcpy(kaddr + offset, src, cur);
2700 kunmap_atomic(kaddr, KM_USER1);
2708 EXPORT_SYMBOL(write_extent_buffer);
2710 void memset_extent_buffer(struct extent_buffer *eb, char c,
2711 unsigned long start, unsigned long len)
2717 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
2718 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
2720 WARN_ON(start > eb->len);
2721 WARN_ON(start + len > eb->start + eb->len);
2723 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
2726 page = extent_buffer_page(eb, i);
2727 WARN_ON(!PageUptodate(page));
2729 cur = min(len, PAGE_CACHE_SIZE - offset);
2730 kaddr = kmap_atomic(page, KM_USER0);
2731 memset(kaddr + offset, c, cur);
2732 kunmap_atomic(kaddr, KM_USER0);
2739 EXPORT_SYMBOL(memset_extent_buffer);
2741 void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
2742 unsigned long dst_offset, unsigned long src_offset,
2745 u64 dst_len = dst->len;
2750 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
2751 unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
2753 WARN_ON(src->len != dst_len);
2755 offset = (start_offset + dst_offset) &
2756 ((unsigned long)PAGE_CACHE_SIZE - 1);
2759 page = extent_buffer_page(dst, i);
2760 WARN_ON(!PageUptodate(page));
2762 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
2764 kaddr = kmap_atomic(page, KM_USER0);
2765 read_extent_buffer(src, kaddr + offset, src_offset, cur);
2766 kunmap_atomic(kaddr, KM_USER0);
2774 EXPORT_SYMBOL(copy_extent_buffer);
2776 static void move_pages(struct page *dst_page, struct page *src_page,
2777 unsigned long dst_off, unsigned long src_off,
2780 char *dst_kaddr = kmap_atomic(dst_page, KM_USER0);
2781 if (dst_page == src_page) {
2782 memmove(dst_kaddr + dst_off, dst_kaddr + src_off, len);
2784 char *src_kaddr = kmap_atomic(src_page, KM_USER1);
2785 char *p = dst_kaddr + dst_off + len;
2786 char *s = src_kaddr + src_off + len;
2791 kunmap_atomic(src_kaddr, KM_USER1);
2793 kunmap_atomic(dst_kaddr, KM_USER0);
2796 static void copy_pages(struct page *dst_page, struct page *src_page,
2797 unsigned long dst_off, unsigned long src_off,
2800 char *dst_kaddr = kmap_atomic(dst_page, KM_USER0);
2803 if (dst_page != src_page)
2804 src_kaddr = kmap_atomic(src_page, KM_USER1);
2806 src_kaddr = dst_kaddr;
2808 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
2809 kunmap_atomic(dst_kaddr, KM_USER0);
2810 if (dst_page != src_page)
2811 kunmap_atomic(src_kaddr, KM_USER1);
2814 void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
2815 unsigned long src_offset, unsigned long len)
2818 size_t dst_off_in_page;
2819 size_t src_off_in_page;
2820 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
2821 unsigned long dst_i;
2822 unsigned long src_i;
2824 if (src_offset + len > dst->len) {
2825 printk("memmove bogus src_offset %lu move len %lu len %lu\n",
2826 src_offset, len, dst->len);
2829 if (dst_offset + len > dst->len) {
2830 printk("memmove bogus dst_offset %lu move len %lu len %lu\n",
2831 dst_offset, len, dst->len);
2836 dst_off_in_page = (start_offset + dst_offset) &
2837 ((unsigned long)PAGE_CACHE_SIZE - 1);
2838 src_off_in_page = (start_offset + src_offset) &
2839 ((unsigned long)PAGE_CACHE_SIZE - 1);
2841 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
2842 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
2844 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
2846 cur = min_t(unsigned long, cur,
2847 (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
2849 copy_pages(extent_buffer_page(dst, dst_i),
2850 extent_buffer_page(dst, src_i),
2851 dst_off_in_page, src_off_in_page, cur);
2858 EXPORT_SYMBOL(memcpy_extent_buffer);
2860 void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
2861 unsigned long src_offset, unsigned long len)
2864 size_t dst_off_in_page;
2865 size_t src_off_in_page;
2866 unsigned long dst_end = dst_offset + len - 1;
2867 unsigned long src_end = src_offset + len - 1;
2868 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
2869 unsigned long dst_i;
2870 unsigned long src_i;
2872 if (src_offset + len > dst->len) {
2873 printk("memmove bogus src_offset %lu move len %lu len %lu\n",
2874 src_offset, len, dst->len);
2877 if (dst_offset + len > dst->len) {
2878 printk("memmove bogus dst_offset %lu move len %lu len %lu\n",
2879 dst_offset, len, dst->len);
2882 if (dst_offset < src_offset) {
2883 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
2887 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
2888 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
2890 dst_off_in_page = (start_offset + dst_end) &
2891 ((unsigned long)PAGE_CACHE_SIZE - 1);
2892 src_off_in_page = (start_offset + src_end) &
2893 ((unsigned long)PAGE_CACHE_SIZE - 1);
2895 cur = min_t(unsigned long, len, src_off_in_page + 1);
2896 cur = min(cur, dst_off_in_page + 1);
2897 move_pages(extent_buffer_page(dst, dst_i),
2898 extent_buffer_page(dst, src_i),
2899 dst_off_in_page - cur + 1,
2900 src_off_in_page - cur + 1, cur);
2907 EXPORT_SYMBOL(memmove_extent_buffer);