4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
21 #include <linux/kernel.h>
22 #include <linux/syscalls.h>
25 #include <linux/percpu.h>
26 #include <linux/slab.h>
27 #include <linux/capability.h>
28 #include <linux/blkdev.h>
29 #include <linux/file.h>
30 #include <linux/quotaops.h>
31 #include <linux/highmem.h>
32 #include <linux/module.h>
33 #include <linux/writeback.h>
34 #include <linux/hash.h>
35 #include <linux/suspend.h>
36 #include <linux/buffer_head.h>
37 #include <linux/task_io_accounting_ops.h>
38 #include <linux/bio.h>
39 #include <linux/notifier.h>
40 #include <linux/cpu.h>
41 #include <linux/bitops.h>
42 #include <linux/mpage.h>
43 #include <linux/bit_spinlock.h>
45 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
47 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
50 init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
52 bh->b_end_io = handler;
53 bh->b_private = private;
56 static int sync_buffer(void *word)
58 struct block_device *bd;
59 struct buffer_head *bh
60 = container_of(word, struct buffer_head, b_state);
65 blk_run_address_space(bd->bd_inode->i_mapping);
70 void __lock_buffer(struct buffer_head *bh)
72 wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer,
73 TASK_UNINTERRUPTIBLE);
75 EXPORT_SYMBOL(__lock_buffer);
77 void unlock_buffer(struct buffer_head *bh)
79 clear_bit_unlock(BH_Lock, &bh->b_state);
80 smp_mb__after_clear_bit();
81 wake_up_bit(&bh->b_state, BH_Lock);
85 * Block until a buffer comes unlocked. This doesn't stop it
86 * from becoming locked again - you have to lock it yourself
87 * if you want to preserve its state.
89 void __wait_on_buffer(struct buffer_head * bh)
91 wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE);
95 __clear_page_buffers(struct page *page)
97 ClearPagePrivate(page);
98 set_page_private(page, 0);
99 page_cache_release(page);
103 static int quiet_error(struct buffer_head *bh)
105 if (!test_bit(BH_Quiet, &bh->b_state) && printk_ratelimit())
111 static void buffer_io_error(struct buffer_head *bh)
113 char b[BDEVNAME_SIZE];
114 printk(KERN_ERR "Buffer I/O error on device %s, logical block %Lu\n",
115 bdevname(bh->b_bdev, b),
116 (unsigned long long)bh->b_blocknr);
120 * End-of-IO handler helper function which does not touch the bh after
122 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
123 * a race there is benign: unlock_buffer() only use the bh's address for
124 * hashing after unlocking the buffer, so it doesn't actually touch the bh
127 static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
130 set_buffer_uptodate(bh);
132 /* This happens, due to failed READA attempts. */
133 clear_buffer_uptodate(bh);
139 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
140 * unlock the buffer. This is what ll_rw_block uses too.
142 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
144 __end_buffer_read_notouch(bh, uptodate);
148 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
150 char b[BDEVNAME_SIZE];
153 set_buffer_uptodate(bh);
155 if (!buffer_eopnotsupp(bh) && !quiet_error(bh)) {
157 printk(KERN_WARNING "lost page write due to "
159 bdevname(bh->b_bdev, b));
161 set_buffer_write_io_error(bh);
162 clear_buffer_uptodate(bh);
169 * Various filesystems appear to want __find_get_block to be non-blocking.
170 * But it's the page lock which protects the buffers. To get around this,
171 * we get exclusion from try_to_free_buffers with the blockdev mapping's
174 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
175 * may be quite high. This code could TryLock the page, and if that
176 * succeeds, there is no need to take private_lock. (But if
177 * private_lock is contended then so is mapping->tree_lock).
179 static struct buffer_head *
180 __find_get_block_slow(struct block_device *bdev, sector_t block)
182 struct inode *bd_inode = bdev->bd_inode;
183 struct address_space *bd_mapping = bd_inode->i_mapping;
184 struct buffer_head *ret = NULL;
186 struct buffer_head *bh;
187 struct buffer_head *head;
191 index = block >> (PAGE_CACHE_SHIFT - bd_inode->i_blkbits);
192 page = find_get_page(bd_mapping, index);
196 spin_lock(&bd_mapping->private_lock);
197 if (!page_has_buffers(page))
199 head = page_buffers(page);
202 if (bh->b_blocknr == block) {
207 if (!buffer_mapped(bh))
209 bh = bh->b_this_page;
210 } while (bh != head);
212 /* we might be here because some of the buffers on this page are
213 * not mapped. This is due to various races between
214 * file io on the block device and getblk. It gets dealt with
215 * elsewhere, don't buffer_error if we had some unmapped buffers
218 printk("__find_get_block_slow() failed. "
219 "block=%llu, b_blocknr=%llu\n",
220 (unsigned long long)block,
221 (unsigned long long)bh->b_blocknr);
222 printk("b_state=0x%08lx, b_size=%zu\n",
223 bh->b_state, bh->b_size);
224 printk("device blocksize: %d\n", 1 << bd_inode->i_blkbits);
227 spin_unlock(&bd_mapping->private_lock);
228 page_cache_release(page);
233 /* If invalidate_buffers() will trash dirty buffers, it means some kind
234 of fs corruption is going on. Trashing dirty data always imply losing
235 information that was supposed to be just stored on the physical layer
238 Thus invalidate_buffers in general usage is not allwowed to trash
239 dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
240 be preserved. These buffers are simply skipped.
242 We also skip buffers which are still in use. For example this can
243 happen if a userspace program is reading the block device.
245 NOTE: In the case where the user removed a removable-media-disk even if
246 there's still dirty data not synced on disk (due a bug in the device driver
247 or due an error of the user), by not destroying the dirty buffers we could
248 generate corruption also on the next media inserted, thus a parameter is
249 necessary to handle this case in the most safe way possible (trying
250 to not corrupt also the new disk inserted with the data belonging to
251 the old now corrupted disk). Also for the ramdisk the natural thing
252 to do in order to release the ramdisk memory is to destroy dirty buffers.
254 These are two special cases. Normal usage imply the device driver
255 to issue a sync on the device (without waiting I/O completion) and
256 then an invalidate_buffers call that doesn't trash dirty buffers.
258 For handling cache coherency with the blkdev pagecache the 'update' case
259 is been introduced. It is needed to re-read from disk any pinned
260 buffer. NOTE: re-reading from disk is destructive so we can do it only
261 when we assume nobody is changing the buffercache under our I/O and when
262 we think the disk contains more recent information than the buffercache.
263 The update == 1 pass marks the buffers we need to update, the update == 2
264 pass does the actual I/O. */
265 void invalidate_bdev(struct block_device *bdev)
267 struct address_space *mapping = bdev->bd_inode->i_mapping;
269 if (mapping->nrpages == 0)
272 invalidate_bh_lrus();
273 invalidate_mapping_pages(mapping, 0, -1);
277 * Kick pdflush then try to free up some ZONE_NORMAL memory.
279 static void free_more_memory(void)
284 wakeup_pdflush(1024);
287 for_each_online_node(nid) {
288 (void)first_zones_zonelist(node_zonelist(nid, GFP_NOFS),
289 gfp_zone(GFP_NOFS), NULL,
292 try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,
298 * I/O completion handler for block_read_full_page() - pages
299 * which come unlocked at the end of I/O.
301 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
304 struct buffer_head *first;
305 struct buffer_head *tmp;
307 int page_uptodate = 1;
309 BUG_ON(!buffer_async_read(bh));
313 set_buffer_uptodate(bh);
315 clear_buffer_uptodate(bh);
316 if (!quiet_error(bh))
322 * Be _very_ careful from here on. Bad things can happen if
323 * two buffer heads end IO at almost the same time and both
324 * decide that the page is now completely done.
326 first = page_buffers(page);
327 local_irq_save(flags);
328 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
329 clear_buffer_async_read(bh);
333 if (!buffer_uptodate(tmp))
335 if (buffer_async_read(tmp)) {
336 BUG_ON(!buffer_locked(tmp));
339 tmp = tmp->b_this_page;
341 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
342 local_irq_restore(flags);
345 * If none of the buffers had errors and they are all
346 * uptodate then we can set the page uptodate.
348 if (page_uptodate && !PageError(page))
349 SetPageUptodate(page);
354 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
355 local_irq_restore(flags);
360 * Completion handler for block_write_full_page() - pages which are unlocked
361 * during I/O, and which have PageWriteback cleared upon I/O completion.
363 static void end_buffer_async_write(struct buffer_head *bh, int uptodate)
365 char b[BDEVNAME_SIZE];
367 struct buffer_head *first;
368 struct buffer_head *tmp;
371 BUG_ON(!buffer_async_write(bh));
375 set_buffer_uptodate(bh);
377 if (!quiet_error(bh)) {
379 printk(KERN_WARNING "lost page write due to "
381 bdevname(bh->b_bdev, b));
383 set_bit(AS_EIO, &page->mapping->flags);
384 set_buffer_write_io_error(bh);
385 clear_buffer_uptodate(bh);
389 first = page_buffers(page);
390 local_irq_save(flags);
391 bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
393 clear_buffer_async_write(bh);
395 tmp = bh->b_this_page;
397 if (buffer_async_write(tmp)) {
398 BUG_ON(!buffer_locked(tmp));
401 tmp = tmp->b_this_page;
403 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
404 local_irq_restore(flags);
405 end_page_writeback(page);
409 bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
410 local_irq_restore(flags);
415 * If a page's buffers are under async readin (end_buffer_async_read
416 * completion) then there is a possibility that another thread of
417 * control could lock one of the buffers after it has completed
418 * but while some of the other buffers have not completed. This
419 * locked buffer would confuse end_buffer_async_read() into not unlocking
420 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
421 * that this buffer is not under async I/O.
423 * The page comes unlocked when it has no locked buffer_async buffers
426 * PageLocked prevents anyone starting new async I/O reads any of
429 * PageWriteback is used to prevent simultaneous writeout of the same
432 * PageLocked prevents anyone from starting writeback of a page which is
433 * under read I/O (PageWriteback is only ever set against a locked page).
435 static void mark_buffer_async_read(struct buffer_head *bh)
437 bh->b_end_io = end_buffer_async_read;
438 set_buffer_async_read(bh);
441 void mark_buffer_async_write(struct buffer_head *bh)
443 bh->b_end_io = end_buffer_async_write;
444 set_buffer_async_write(bh);
446 EXPORT_SYMBOL(mark_buffer_async_write);
450 * fs/buffer.c contains helper functions for buffer-backed address space's
451 * fsync functions. A common requirement for buffer-based filesystems is
452 * that certain data from the backing blockdev needs to be written out for
453 * a successful fsync(). For example, ext2 indirect blocks need to be
454 * written back and waited upon before fsync() returns.
456 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
457 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
458 * management of a list of dependent buffers at ->i_mapping->private_list.
460 * Locking is a little subtle: try_to_free_buffers() will remove buffers
461 * from their controlling inode's queue when they are being freed. But
462 * try_to_free_buffers() will be operating against the *blockdev* mapping
463 * at the time, not against the S_ISREG file which depends on those buffers.
464 * So the locking for private_list is via the private_lock in the address_space
465 * which backs the buffers. Which is different from the address_space
466 * against which the buffers are listed. So for a particular address_space,
467 * mapping->private_lock does *not* protect mapping->private_list! In fact,
468 * mapping->private_list will always be protected by the backing blockdev's
471 * Which introduces a requirement: all buffers on an address_space's
472 * ->private_list must be from the same address_space: the blockdev's.
474 * address_spaces which do not place buffers at ->private_list via these
475 * utility functions are free to use private_lock and private_list for
476 * whatever they want. The only requirement is that list_empty(private_list)
477 * be true at clear_inode() time.
479 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
480 * filesystems should do that. invalidate_inode_buffers() should just go
481 * BUG_ON(!list_empty).
483 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
484 * take an address_space, not an inode. And it should be called
485 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
488 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
489 * list if it is already on a list. Because if the buffer is on a list,
490 * it *must* already be on the right one. If not, the filesystem is being
491 * silly. This will save a ton of locking. But first we have to ensure
492 * that buffers are taken *off* the old inode's list when they are freed
493 * (presumably in truncate). That requires careful auditing of all
494 * filesystems (do it inside bforget()). It could also be done by bringing
499 * The buffer's backing address_space's private_lock must be held
501 static void __remove_assoc_queue(struct buffer_head *bh)
503 list_del_init(&bh->b_assoc_buffers);
504 WARN_ON(!bh->b_assoc_map);
505 if (buffer_write_io_error(bh))
506 set_bit(AS_EIO, &bh->b_assoc_map->flags);
507 bh->b_assoc_map = NULL;
510 int inode_has_buffers(struct inode *inode)
512 return !list_empty(&inode->i_data.private_list);
516 * osync is designed to support O_SYNC io. It waits synchronously for
517 * all already-submitted IO to complete, but does not queue any new
518 * writes to the disk.
520 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
521 * you dirty the buffers, and then use osync_inode_buffers to wait for
522 * completion. Any other dirty buffers which are not yet queued for
523 * write will not be flushed to disk by the osync.
525 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
527 struct buffer_head *bh;
533 list_for_each_prev(p, list) {
535 if (buffer_locked(bh)) {
539 if (!buffer_uptodate(bh))
551 * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
552 * @mapping: the mapping which wants those buffers written
554 * Starts I/O against the buffers at mapping->private_list, and waits upon
557 * Basically, this is a convenience function for fsync().
558 * @mapping is a file or directory which needs those buffers to be written for
559 * a successful fsync().
561 int sync_mapping_buffers(struct address_space *mapping)
563 struct address_space *buffer_mapping = mapping->assoc_mapping;
565 if (buffer_mapping == NULL || list_empty(&mapping->private_list))
568 return fsync_buffers_list(&buffer_mapping->private_lock,
569 &mapping->private_list);
571 EXPORT_SYMBOL(sync_mapping_buffers);
574 * Called when we've recently written block `bblock', and it is known that
575 * `bblock' was for a buffer_boundary() buffer. This means that the block at
576 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
577 * dirty, schedule it for IO. So that indirects merge nicely with their data.
579 void write_boundary_block(struct block_device *bdev,
580 sector_t bblock, unsigned blocksize)
582 struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
584 if (buffer_dirty(bh))
585 ll_rw_block(WRITE, 1, &bh);
590 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
592 struct address_space *mapping = inode->i_mapping;
593 struct address_space *buffer_mapping = bh->b_page->mapping;
595 mark_buffer_dirty(bh);
596 if (!mapping->assoc_mapping) {
597 mapping->assoc_mapping = buffer_mapping;
599 BUG_ON(mapping->assoc_mapping != buffer_mapping);
601 if (!bh->b_assoc_map) {
602 spin_lock(&buffer_mapping->private_lock);
603 list_move_tail(&bh->b_assoc_buffers,
604 &mapping->private_list);
605 bh->b_assoc_map = mapping;
606 spin_unlock(&buffer_mapping->private_lock);
609 EXPORT_SYMBOL(mark_buffer_dirty_inode);
612 * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
615 * If warn is true, then emit a warning if the page is not uptodate and has
616 * not been truncated.
618 static void __set_page_dirty(struct page *page,
619 struct address_space *mapping, int warn)
621 spin_lock_irq(&mapping->tree_lock);
622 if (page->mapping) { /* Race with truncate? */
623 WARN_ON_ONCE(warn && !PageUptodate(page));
624 account_page_dirtied(page, mapping);
625 radix_tree_tag_set(&mapping->page_tree,
626 page_index(page), PAGECACHE_TAG_DIRTY);
628 spin_unlock_irq(&mapping->tree_lock);
629 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
633 * Add a page to the dirty page list.
635 * It is a sad fact of life that this function is called from several places
636 * deeply under spinlocking. It may not sleep.
638 * If the page has buffers, the uptodate buffers are set dirty, to preserve
639 * dirty-state coherency between the page and the buffers. It the page does
640 * not have buffers then when they are later attached they will all be set
643 * The buffers are dirtied before the page is dirtied. There's a small race
644 * window in which a writepage caller may see the page cleanness but not the
645 * buffer dirtiness. That's fine. If this code were to set the page dirty
646 * before the buffers, a concurrent writepage caller could clear the page dirty
647 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
648 * page on the dirty page list.
650 * We use private_lock to lock against try_to_free_buffers while using the
651 * page's buffer list. Also use this to protect against clean buffers being
652 * added to the page after it was set dirty.
654 * FIXME: may need to call ->reservepage here as well. That's rather up to the
655 * address_space though.
657 int __set_page_dirty_buffers(struct page *page)
660 struct address_space *mapping = page_mapping(page);
662 if (unlikely(!mapping))
663 return !TestSetPageDirty(page);
665 spin_lock(&mapping->private_lock);
666 if (page_has_buffers(page)) {
667 struct buffer_head *head = page_buffers(page);
668 struct buffer_head *bh = head;
671 set_buffer_dirty(bh);
672 bh = bh->b_this_page;
673 } while (bh != head);
675 newly_dirty = !TestSetPageDirty(page);
676 spin_unlock(&mapping->private_lock);
679 __set_page_dirty(page, mapping, 1);
682 EXPORT_SYMBOL(__set_page_dirty_buffers);
685 * Write out and wait upon a list of buffers.
687 * We have conflicting pressures: we want to make sure that all
688 * initially dirty buffers get waited on, but that any subsequently
689 * dirtied buffers don't. After all, we don't want fsync to last
690 * forever if somebody is actively writing to the file.
692 * Do this in two main stages: first we copy dirty buffers to a
693 * temporary inode list, queueing the writes as we go. Then we clean
694 * up, waiting for those writes to complete.
696 * During this second stage, any subsequent updates to the file may end
697 * up refiling the buffer on the original inode's dirty list again, so
698 * there is a chance we will end up with a buffer queued for write but
699 * not yet completed on that list. So, as a final cleanup we go through
700 * the osync code to catch these locked, dirty buffers without requeuing
701 * any newly dirty buffers for write.
703 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
705 struct buffer_head *bh;
706 struct list_head tmp;
707 struct address_space *mapping;
710 INIT_LIST_HEAD(&tmp);
713 while (!list_empty(list)) {
714 bh = BH_ENTRY(list->next);
715 mapping = bh->b_assoc_map;
716 __remove_assoc_queue(bh);
717 /* Avoid race with mark_buffer_dirty_inode() which does
718 * a lockless check and we rely on seeing the dirty bit */
720 if (buffer_dirty(bh) || buffer_locked(bh)) {
721 list_add(&bh->b_assoc_buffers, &tmp);
722 bh->b_assoc_map = mapping;
723 if (buffer_dirty(bh)) {
727 * Ensure any pending I/O completes so that
728 * ll_rw_block() actually writes the current
729 * contents - it is a noop if I/O is still in
730 * flight on potentially older contents.
732 ll_rw_block(SWRITE_SYNC, 1, &bh);
739 while (!list_empty(&tmp)) {
740 bh = BH_ENTRY(tmp.prev);
742 mapping = bh->b_assoc_map;
743 __remove_assoc_queue(bh);
744 /* Avoid race with mark_buffer_dirty_inode() which does
745 * a lockless check and we rely on seeing the dirty bit */
747 if (buffer_dirty(bh)) {
748 list_add(&bh->b_assoc_buffers,
749 &mapping->private_list);
750 bh->b_assoc_map = mapping;
754 if (!buffer_uptodate(bh))
761 err2 = osync_buffers_list(lock, list);
769 * Invalidate any and all dirty buffers on a given inode. We are
770 * probably unmounting the fs, but that doesn't mean we have already
771 * done a sync(). Just drop the buffers from the inode list.
773 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
774 * assumes that all the buffers are against the blockdev. Not true
777 void invalidate_inode_buffers(struct inode *inode)
779 if (inode_has_buffers(inode)) {
780 struct address_space *mapping = &inode->i_data;
781 struct list_head *list = &mapping->private_list;
782 struct address_space *buffer_mapping = mapping->assoc_mapping;
784 spin_lock(&buffer_mapping->private_lock);
785 while (!list_empty(list))
786 __remove_assoc_queue(BH_ENTRY(list->next));
787 spin_unlock(&buffer_mapping->private_lock);
790 EXPORT_SYMBOL(invalidate_inode_buffers);
793 * Remove any clean buffers from the inode's buffer list. This is called
794 * when we're trying to free the inode itself. Those buffers can pin it.
796 * Returns true if all buffers were removed.
798 int remove_inode_buffers(struct inode *inode)
802 if (inode_has_buffers(inode)) {
803 struct address_space *mapping = &inode->i_data;
804 struct list_head *list = &mapping->private_list;
805 struct address_space *buffer_mapping = mapping->assoc_mapping;
807 spin_lock(&buffer_mapping->private_lock);
808 while (!list_empty(list)) {
809 struct buffer_head *bh = BH_ENTRY(list->next);
810 if (buffer_dirty(bh)) {
814 __remove_assoc_queue(bh);
816 spin_unlock(&buffer_mapping->private_lock);
822 * Create the appropriate buffers when given a page for data area and
823 * the size of each buffer.. Use the bh->b_this_page linked list to
824 * follow the buffers created. Return NULL if unable to create more
827 * The retry flag is used to differentiate async IO (paging, swapping)
828 * which may not fail from ordinary buffer allocations.
830 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
833 struct buffer_head *bh, *head;
839 while ((offset -= size) >= 0) {
840 bh = alloc_buffer_head(GFP_NOFS);
845 bh->b_this_page = head;
850 atomic_set(&bh->b_count, 0);
851 bh->b_private = NULL;
854 /* Link the buffer to its page */
855 set_bh_page(bh, page, offset);
857 init_buffer(bh, NULL, NULL);
861 * In case anything failed, we just free everything we got.
867 head = head->b_this_page;
868 free_buffer_head(bh);
873 * Return failure for non-async IO requests. Async IO requests
874 * are not allowed to fail, so we have to wait until buffer heads
875 * become available. But we don't want tasks sleeping with
876 * partially complete buffers, so all were released above.
881 /* We're _really_ low on memory. Now we just
882 * wait for old buffer heads to become free due to
883 * finishing IO. Since this is an async request and
884 * the reserve list is empty, we're sure there are
885 * async buffer heads in use.
890 EXPORT_SYMBOL_GPL(alloc_page_buffers);
893 link_dev_buffers(struct page *page, struct buffer_head *head)
895 struct buffer_head *bh, *tail;
900 bh = bh->b_this_page;
902 tail->b_this_page = head;
903 attach_page_buffers(page, head);
907 * Initialise the state of a blockdev page's buffers.
910 init_page_buffers(struct page *page, struct block_device *bdev,
911 sector_t block, int size)
913 struct buffer_head *head = page_buffers(page);
914 struct buffer_head *bh = head;
915 int uptodate = PageUptodate(page);
918 if (!buffer_mapped(bh)) {
919 init_buffer(bh, NULL, NULL);
921 bh->b_blocknr = block;
923 set_buffer_uptodate(bh);
924 set_buffer_mapped(bh);
927 bh = bh->b_this_page;
928 } while (bh != head);
932 * Create the page-cache page that contains the requested block.
934 * This is user purely for blockdev mappings.
937 grow_dev_page(struct block_device *bdev, sector_t block,
938 pgoff_t index, int size)
940 struct inode *inode = bdev->bd_inode;
942 struct buffer_head *bh;
944 page = find_or_create_page(inode->i_mapping, index,
945 (mapping_gfp_mask(inode->i_mapping) & ~__GFP_FS)|__GFP_MOVABLE);
949 BUG_ON(!PageLocked(page));
951 if (page_has_buffers(page)) {
952 bh = page_buffers(page);
953 if (bh->b_size == size) {
954 init_page_buffers(page, bdev, block, size);
957 if (!try_to_free_buffers(page))
962 * Allocate some buffers for this page
964 bh = alloc_page_buffers(page, size, 0);
969 * Link the page to the buffers and initialise them. Take the
970 * lock to be atomic wrt __find_get_block(), which does not
971 * run under the page lock.
973 spin_lock(&inode->i_mapping->private_lock);
974 link_dev_buffers(page, bh);
975 init_page_buffers(page, bdev, block, size);
976 spin_unlock(&inode->i_mapping->private_lock);
982 page_cache_release(page);
987 * Create buffers for the specified block device block's page. If
988 * that page was dirty, the buffers are set dirty also.
991 grow_buffers(struct block_device *bdev, sector_t block, int size)
1000 } while ((size << sizebits) < PAGE_SIZE);
1002 index = block >> sizebits;
1005 * Check for a block which wants to lie outside our maximum possible
1006 * pagecache index. (this comparison is done using sector_t types).
1008 if (unlikely(index != block >> sizebits)) {
1009 char b[BDEVNAME_SIZE];
1011 printk(KERN_ERR "%s: requested out-of-range block %llu for "
1013 __func__, (unsigned long long)block,
1017 block = index << sizebits;
1018 /* Create a page with the proper size buffers.. */
1019 page = grow_dev_page(bdev, block, index, size);
1023 page_cache_release(page);
1027 static struct buffer_head *
1028 __getblk_slow(struct block_device *bdev, sector_t block, int size)
1030 /* Size must be multiple of hard sectorsize */
1031 if (unlikely(size & (bdev_hardsect_size(bdev)-1) ||
1032 (size < 512 || size > PAGE_SIZE))) {
1033 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1035 printk(KERN_ERR "hardsect size: %d\n",
1036 bdev_hardsect_size(bdev));
1043 struct buffer_head * bh;
1046 bh = __find_get_block(bdev, block, size);
1050 ret = grow_buffers(bdev, block, size);
1059 * The relationship between dirty buffers and dirty pages:
1061 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1062 * the page is tagged dirty in its radix tree.
1064 * At all times, the dirtiness of the buffers represents the dirtiness of
1065 * subsections of the page. If the page has buffers, the page dirty bit is
1066 * merely a hint about the true dirty state.
1068 * When a page is set dirty in its entirety, all its buffers are marked dirty
1069 * (if the page has buffers).
1071 * When a buffer is marked dirty, its page is dirtied, but the page's other
1074 * Also. When blockdev buffers are explicitly read with bread(), they
1075 * individually become uptodate. But their backing page remains not
1076 * uptodate - even if all of its buffers are uptodate. A subsequent
1077 * block_read_full_page() against that page will discover all the uptodate
1078 * buffers, will set the page uptodate and will perform no I/O.
1082 * mark_buffer_dirty - mark a buffer_head as needing writeout
1083 * @bh: the buffer_head to mark dirty
1085 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1086 * backing page dirty, then tag the page as dirty in its address_space's radix
1087 * tree and then attach the address_space's inode to its superblock's dirty
1090 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1091 * mapping->tree_lock and the global inode_lock.
1093 void mark_buffer_dirty(struct buffer_head *bh)
1095 WARN_ON_ONCE(!buffer_uptodate(bh));
1098 * Very *carefully* optimize the it-is-already-dirty case.
1100 * Don't let the final "is it dirty" escape to before we
1101 * perhaps modified the buffer.
1103 if (buffer_dirty(bh)) {
1105 if (buffer_dirty(bh))
1109 if (!test_set_buffer_dirty(bh)) {
1110 struct page *page = bh->b_page;
1111 if (!TestSetPageDirty(page))
1112 __set_page_dirty(page, page_mapping(page), 0);
1117 * Decrement a buffer_head's reference count. If all buffers against a page
1118 * have zero reference count, are clean and unlocked, and if the page is clean
1119 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1120 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1121 * a page but it ends up not being freed, and buffers may later be reattached).
1123 void __brelse(struct buffer_head * buf)
1125 if (atomic_read(&buf->b_count)) {
1129 WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1133 * bforget() is like brelse(), except it discards any
1134 * potentially dirty data.
1136 void __bforget(struct buffer_head *bh)
1138 clear_buffer_dirty(bh);
1139 if (bh->b_assoc_map) {
1140 struct address_space *buffer_mapping = bh->b_page->mapping;
1142 spin_lock(&buffer_mapping->private_lock);
1143 list_del_init(&bh->b_assoc_buffers);
1144 bh->b_assoc_map = NULL;
1145 spin_unlock(&buffer_mapping->private_lock);
1150 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1153 if (buffer_uptodate(bh)) {
1158 bh->b_end_io = end_buffer_read_sync;
1159 submit_bh(READ, bh);
1161 if (buffer_uptodate(bh))
1169 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1170 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1171 * refcount elevated by one when they're in an LRU. A buffer can only appear
1172 * once in a particular CPU's LRU. A single buffer can be present in multiple
1173 * CPU's LRUs at the same time.
1175 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1176 * sb_find_get_block().
1178 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1179 * a local interrupt disable for that.
1182 #define BH_LRU_SIZE 8
1185 struct buffer_head *bhs[BH_LRU_SIZE];
1188 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1191 #define bh_lru_lock() local_irq_disable()
1192 #define bh_lru_unlock() local_irq_enable()
1194 #define bh_lru_lock() preempt_disable()
1195 #define bh_lru_unlock() preempt_enable()
1198 static inline void check_irqs_on(void)
1200 #ifdef irqs_disabled
1201 BUG_ON(irqs_disabled());
1206 * The LRU management algorithm is dopey-but-simple. Sorry.
1208 static void bh_lru_install(struct buffer_head *bh)
1210 struct buffer_head *evictee = NULL;
1215 lru = &__get_cpu_var(bh_lrus);
1216 if (lru->bhs[0] != bh) {
1217 struct buffer_head *bhs[BH_LRU_SIZE];
1223 for (in = 0; in < BH_LRU_SIZE; in++) {
1224 struct buffer_head *bh2 = lru->bhs[in];
1229 if (out >= BH_LRU_SIZE) {
1230 BUG_ON(evictee != NULL);
1237 while (out < BH_LRU_SIZE)
1239 memcpy(lru->bhs, bhs, sizeof(bhs));
1248 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1250 static struct buffer_head *
1251 lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1253 struct buffer_head *ret = NULL;
1259 lru = &__get_cpu_var(bh_lrus);
1260 for (i = 0; i < BH_LRU_SIZE; i++) {
1261 struct buffer_head *bh = lru->bhs[i];
1263 if (bh && bh->b_bdev == bdev &&
1264 bh->b_blocknr == block && bh->b_size == size) {
1267 lru->bhs[i] = lru->bhs[i - 1];
1282 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1283 * it in the LRU and mark it as accessed. If it is not present then return
1286 struct buffer_head *
1287 __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1289 struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1292 bh = __find_get_block_slow(bdev, block);
1300 EXPORT_SYMBOL(__find_get_block);
1303 * __getblk will locate (and, if necessary, create) the buffer_head
1304 * which corresponds to the passed block_device, block and size. The
1305 * returned buffer has its reference count incremented.
1307 * __getblk() cannot fail - it just keeps trying. If you pass it an
1308 * illegal block number, __getblk() will happily return a buffer_head
1309 * which represents the non-existent block. Very weird.
1311 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1312 * attempt is failing. FIXME, perhaps?
1314 struct buffer_head *
1315 __getblk(struct block_device *bdev, sector_t block, unsigned size)
1317 struct buffer_head *bh = __find_get_block(bdev, block, size);
1321 bh = __getblk_slow(bdev, block, size);
1324 EXPORT_SYMBOL(__getblk);
1327 * Do async read-ahead on a buffer..
1329 void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1331 struct buffer_head *bh = __getblk(bdev, block, size);
1333 ll_rw_block(READA, 1, &bh);
1337 EXPORT_SYMBOL(__breadahead);
1340 * __bread() - reads a specified block and returns the bh
1341 * @bdev: the block_device to read from
1342 * @block: number of block
1343 * @size: size (in bytes) to read
1345 * Reads a specified block, and returns buffer head that contains it.
1346 * It returns NULL if the block was unreadable.
1348 struct buffer_head *
1349 __bread(struct block_device *bdev, sector_t block, unsigned size)
1351 struct buffer_head *bh = __getblk(bdev, block, size);
1353 if (likely(bh) && !buffer_uptodate(bh))
1354 bh = __bread_slow(bh);
1357 EXPORT_SYMBOL(__bread);
1360 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1361 * This doesn't race because it runs in each cpu either in irq
1362 * or with preempt disabled.
1364 static void invalidate_bh_lru(void *arg)
1366 struct bh_lru *b = &get_cpu_var(bh_lrus);
1369 for (i = 0; i < BH_LRU_SIZE; i++) {
1373 put_cpu_var(bh_lrus);
1376 void invalidate_bh_lrus(void)
1378 on_each_cpu(invalidate_bh_lru, NULL, 1);
1380 EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1382 void set_bh_page(struct buffer_head *bh,
1383 struct page *page, unsigned long offset)
1386 BUG_ON(offset >= PAGE_SIZE);
1387 if (PageHighMem(page))
1389 * This catches illegal uses and preserves the offset:
1391 bh->b_data = (char *)(0 + offset);
1393 bh->b_data = page_address(page) + offset;
1395 EXPORT_SYMBOL(set_bh_page);
1398 * Called when truncating a buffer on a page completely.
1400 static void discard_buffer(struct buffer_head * bh)
1403 clear_buffer_dirty(bh);
1405 clear_buffer_mapped(bh);
1406 clear_buffer_req(bh);
1407 clear_buffer_new(bh);
1408 clear_buffer_delay(bh);
1409 clear_buffer_unwritten(bh);
1414 * block_invalidatepage - invalidate part of all of a buffer-backed page
1416 * @page: the page which is affected
1417 * @offset: the index of the truncation point
1419 * block_invalidatepage() is called when all or part of the page has become
1420 * invalidatedby a truncate operation.
1422 * block_invalidatepage() does not have to release all buffers, but it must
1423 * ensure that no dirty buffer is left outside @offset and that no I/O
1424 * is underway against any of the blocks which are outside the truncation
1425 * point. Because the caller is about to free (and possibly reuse) those
1428 void block_invalidatepage(struct page *page, unsigned long offset)
1430 struct buffer_head *head, *bh, *next;
1431 unsigned int curr_off = 0;
1433 BUG_ON(!PageLocked(page));
1434 if (!page_has_buffers(page))
1437 head = page_buffers(page);
1440 unsigned int next_off = curr_off + bh->b_size;
1441 next = bh->b_this_page;
1444 * is this block fully invalidated?
1446 if (offset <= curr_off)
1448 curr_off = next_off;
1450 } while (bh != head);
1453 * We release buffers only if the entire page is being invalidated.
1454 * The get_block cached value has been unconditionally invalidated,
1455 * so real IO is not possible anymore.
1458 try_to_release_page(page, 0);
1462 EXPORT_SYMBOL(block_invalidatepage);
1465 * We attach and possibly dirty the buffers atomically wrt
1466 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1467 * is already excluded via the page lock.
1469 void create_empty_buffers(struct page *page,
1470 unsigned long blocksize, unsigned long b_state)
1472 struct buffer_head *bh, *head, *tail;
1474 head = alloc_page_buffers(page, blocksize, 1);
1477 bh->b_state |= b_state;
1479 bh = bh->b_this_page;
1481 tail->b_this_page = head;
1483 spin_lock(&page->mapping->private_lock);
1484 if (PageUptodate(page) || PageDirty(page)) {
1487 if (PageDirty(page))
1488 set_buffer_dirty(bh);
1489 if (PageUptodate(page))
1490 set_buffer_uptodate(bh);
1491 bh = bh->b_this_page;
1492 } while (bh != head);
1494 attach_page_buffers(page, head);
1495 spin_unlock(&page->mapping->private_lock);
1497 EXPORT_SYMBOL(create_empty_buffers);
1500 * We are taking a block for data and we don't want any output from any
1501 * buffer-cache aliases starting from return from that function and
1502 * until the moment when something will explicitly mark the buffer
1503 * dirty (hopefully that will not happen until we will free that block ;-)
1504 * We don't even need to mark it not-uptodate - nobody can expect
1505 * anything from a newly allocated buffer anyway. We used to used
1506 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1507 * don't want to mark the alias unmapped, for example - it would confuse
1508 * anyone who might pick it with bread() afterwards...
1510 * Also.. Note that bforget() doesn't lock the buffer. So there can
1511 * be writeout I/O going on against recently-freed buffers. We don't
1512 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1513 * only if we really need to. That happens here.
1515 void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1517 struct buffer_head *old_bh;
1521 old_bh = __find_get_block_slow(bdev, block);
1523 clear_buffer_dirty(old_bh);
1524 wait_on_buffer(old_bh);
1525 clear_buffer_req(old_bh);
1529 EXPORT_SYMBOL(unmap_underlying_metadata);
1532 * NOTE! All mapped/uptodate combinations are valid:
1534 * Mapped Uptodate Meaning
1536 * No No "unknown" - must do get_block()
1537 * No Yes "hole" - zero-filled
1538 * Yes No "allocated" - allocated on disk, not read in
1539 * Yes Yes "valid" - allocated and up-to-date in memory.
1541 * "Dirty" is valid only with the last case (mapped+uptodate).
1545 * While block_write_full_page is writing back the dirty buffers under
1546 * the page lock, whoever dirtied the buffers may decide to clean them
1547 * again at any time. We handle that by only looking at the buffer
1548 * state inside lock_buffer().
1550 * If block_write_full_page() is called for regular writeback
1551 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1552 * locked buffer. This only can happen if someone has written the buffer
1553 * directly, with submit_bh(). At the address_space level PageWriteback
1554 * prevents this contention from occurring.
1556 static int __block_write_full_page(struct inode *inode, struct page *page,
1557 get_block_t *get_block, struct writeback_control *wbc)
1561 sector_t last_block;
1562 struct buffer_head *bh, *head;
1563 const unsigned blocksize = 1 << inode->i_blkbits;
1564 int nr_underway = 0;
1566 BUG_ON(!PageLocked(page));
1568 last_block = (i_size_read(inode) - 1) >> inode->i_blkbits;
1570 if (!page_has_buffers(page)) {
1571 create_empty_buffers(page, blocksize,
1572 (1 << BH_Dirty)|(1 << BH_Uptodate));
1576 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1577 * here, and the (potentially unmapped) buffers may become dirty at
1578 * any time. If a buffer becomes dirty here after we've inspected it
1579 * then we just miss that fact, and the page stays dirty.
1581 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1582 * handle that here by just cleaning them.
1585 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
1586 head = page_buffers(page);
1590 * Get all the dirty buffers mapped to disk addresses and
1591 * handle any aliases from the underlying blockdev's mapping.
1594 if (block > last_block) {
1596 * mapped buffers outside i_size will occur, because
1597 * this page can be outside i_size when there is a
1598 * truncate in progress.
1601 * The buffer was zeroed by block_write_full_page()
1603 clear_buffer_dirty(bh);
1604 set_buffer_uptodate(bh);
1605 } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
1607 WARN_ON(bh->b_size != blocksize);
1608 err = get_block(inode, block, bh, 1);
1611 clear_buffer_delay(bh);
1612 if (buffer_new(bh)) {
1613 /* blockdev mappings never come here */
1614 clear_buffer_new(bh);
1615 unmap_underlying_metadata(bh->b_bdev,
1619 bh = bh->b_this_page;
1621 } while (bh != head);
1624 if (!buffer_mapped(bh))
1627 * If it's a fully non-blocking write attempt and we cannot
1628 * lock the buffer then redirty the page. Note that this can
1629 * potentially cause a busy-wait loop from pdflush and kswapd
1630 * activity, but those code paths have their own higher-level
1633 if (wbc->sync_mode != WB_SYNC_NONE || !wbc->nonblocking) {
1635 } else if (!trylock_buffer(bh)) {
1636 redirty_page_for_writepage(wbc, page);
1639 if (test_clear_buffer_dirty(bh)) {
1640 mark_buffer_async_write(bh);
1644 } while ((bh = bh->b_this_page) != head);
1647 * The page and its buffers are protected by PageWriteback(), so we can
1648 * drop the bh refcounts early.
1650 BUG_ON(PageWriteback(page));
1651 set_page_writeback(page);
1654 struct buffer_head *next = bh->b_this_page;
1655 if (buffer_async_write(bh)) {
1656 submit_bh(WRITE, bh);
1660 } while (bh != head);
1665 if (nr_underway == 0) {
1667 * The page was marked dirty, but the buffers were
1668 * clean. Someone wrote them back by hand with
1669 * ll_rw_block/submit_bh. A rare case.
1671 end_page_writeback(page);
1674 * The page and buffer_heads can be released at any time from
1682 * ENOSPC, or some other error. We may already have added some
1683 * blocks to the file, so we need to write these out to avoid
1684 * exposing stale data.
1685 * The page is currently locked and not marked for writeback
1688 /* Recovery: lock and submit the mapped buffers */
1690 if (buffer_mapped(bh) && buffer_dirty(bh) &&
1691 !buffer_delay(bh)) {
1693 mark_buffer_async_write(bh);
1696 * The buffer may have been set dirty during
1697 * attachment to a dirty page.
1699 clear_buffer_dirty(bh);
1701 } while ((bh = bh->b_this_page) != head);
1703 BUG_ON(PageWriteback(page));
1704 mapping_set_error(page->mapping, err);
1705 set_page_writeback(page);
1707 struct buffer_head *next = bh->b_this_page;
1708 if (buffer_async_write(bh)) {
1709 clear_buffer_dirty(bh);
1710 submit_bh(WRITE, bh);
1714 } while (bh != head);
1720 * If a page has any new buffers, zero them out here, and mark them uptodate
1721 * and dirty so they'll be written out (in order to prevent uninitialised
1722 * block data from leaking). And clear the new bit.
1724 void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
1726 unsigned int block_start, block_end;
1727 struct buffer_head *head, *bh;
1729 BUG_ON(!PageLocked(page));
1730 if (!page_has_buffers(page))
1733 bh = head = page_buffers(page);
1736 block_end = block_start + bh->b_size;
1738 if (buffer_new(bh)) {
1739 if (block_end > from && block_start < to) {
1740 if (!PageUptodate(page)) {
1741 unsigned start, size;
1743 start = max(from, block_start);
1744 size = min(to, block_end) - start;
1746 zero_user(page, start, size);
1747 set_buffer_uptodate(bh);
1750 clear_buffer_new(bh);
1751 mark_buffer_dirty(bh);
1755 block_start = block_end;
1756 bh = bh->b_this_page;
1757 } while (bh != head);
1759 EXPORT_SYMBOL(page_zero_new_buffers);
1761 static int __block_prepare_write(struct inode *inode, struct page *page,
1762 unsigned from, unsigned to, get_block_t *get_block)
1764 unsigned block_start, block_end;
1767 unsigned blocksize, bbits;
1768 struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1770 BUG_ON(!PageLocked(page));
1771 BUG_ON(from > PAGE_CACHE_SIZE);
1772 BUG_ON(to > PAGE_CACHE_SIZE);
1775 blocksize = 1 << inode->i_blkbits;
1776 if (!page_has_buffers(page))
1777 create_empty_buffers(page, blocksize, 0);
1778 head = page_buffers(page);
1780 bbits = inode->i_blkbits;
1781 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - bbits);
1783 for(bh = head, block_start = 0; bh != head || !block_start;
1784 block++, block_start=block_end, bh = bh->b_this_page) {
1785 block_end = block_start + blocksize;
1786 if (block_end <= from || block_start >= to) {
1787 if (PageUptodate(page)) {
1788 if (!buffer_uptodate(bh))
1789 set_buffer_uptodate(bh);
1794 clear_buffer_new(bh);
1795 if (!buffer_mapped(bh)) {
1796 WARN_ON(bh->b_size != blocksize);
1797 err = get_block(inode, block, bh, 1);
1800 if (buffer_new(bh)) {
1801 unmap_underlying_metadata(bh->b_bdev,
1803 if (PageUptodate(page)) {
1804 clear_buffer_new(bh);
1805 set_buffer_uptodate(bh);
1806 mark_buffer_dirty(bh);
1809 if (block_end > to || block_start < from)
1810 zero_user_segments(page,
1816 if (PageUptodate(page)) {
1817 if (!buffer_uptodate(bh))
1818 set_buffer_uptodate(bh);
1821 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1822 !buffer_unwritten(bh) &&
1823 (block_start < from || block_end > to)) {
1824 ll_rw_block(READ, 1, &bh);
1829 * If we issued read requests - let them complete.
1831 while(wait_bh > wait) {
1832 wait_on_buffer(*--wait_bh);
1833 if (!buffer_uptodate(*wait_bh))
1837 page_zero_new_buffers(page, from, to);
1841 static int __block_commit_write(struct inode *inode, struct page *page,
1842 unsigned from, unsigned to)
1844 unsigned block_start, block_end;
1847 struct buffer_head *bh, *head;
1849 blocksize = 1 << inode->i_blkbits;
1851 for(bh = head = page_buffers(page), block_start = 0;
1852 bh != head || !block_start;
1853 block_start=block_end, bh = bh->b_this_page) {
1854 block_end = block_start + blocksize;
1855 if (block_end <= from || block_start >= to) {
1856 if (!buffer_uptodate(bh))
1859 set_buffer_uptodate(bh);
1860 mark_buffer_dirty(bh);
1862 clear_buffer_new(bh);
1866 * If this is a partial write which happened to make all buffers
1867 * uptodate then we can optimize away a bogus readpage() for
1868 * the next read(). Here we 'discover' whether the page went
1869 * uptodate as a result of this (potentially partial) write.
1872 SetPageUptodate(page);
1877 * block_write_begin takes care of the basic task of block allocation and
1878 * bringing partial write blocks uptodate first.
1880 * If *pagep is not NULL, then block_write_begin uses the locked page
1881 * at *pagep rather than allocating its own. In this case, the page will
1882 * not be unlocked or deallocated on failure.
1884 int block_write_begin(struct file *file, struct address_space *mapping,
1885 loff_t pos, unsigned len, unsigned flags,
1886 struct page **pagep, void **fsdata,
1887 get_block_t *get_block)
1889 struct inode *inode = mapping->host;
1893 unsigned start, end;
1896 index = pos >> PAGE_CACHE_SHIFT;
1897 start = pos & (PAGE_CACHE_SIZE - 1);
1903 page = grab_cache_page_write_begin(mapping, index, flags);
1910 BUG_ON(!PageLocked(page));
1912 status = __block_prepare_write(inode, page, start, end, get_block);
1913 if (unlikely(status)) {
1914 ClearPageUptodate(page);
1918 page_cache_release(page);
1922 * prepare_write() may have instantiated a few blocks
1923 * outside i_size. Trim these off again. Don't need
1924 * i_size_read because we hold i_mutex.
1926 if (pos + len > inode->i_size)
1927 vmtruncate(inode, inode->i_size);
1934 EXPORT_SYMBOL(block_write_begin);
1936 int block_write_end(struct file *file, struct address_space *mapping,
1937 loff_t pos, unsigned len, unsigned copied,
1938 struct page *page, void *fsdata)
1940 struct inode *inode = mapping->host;
1943 start = pos & (PAGE_CACHE_SIZE - 1);
1945 if (unlikely(copied < len)) {
1947 * The buffers that were written will now be uptodate, so we
1948 * don't have to worry about a readpage reading them and
1949 * overwriting a partial write. However if we have encountered
1950 * a short write and only partially written into a buffer, it
1951 * will not be marked uptodate, so a readpage might come in and
1952 * destroy our partial write.
1954 * Do the simplest thing, and just treat any short write to a
1955 * non uptodate page as a zero-length write, and force the
1956 * caller to redo the whole thing.
1958 if (!PageUptodate(page))
1961 page_zero_new_buffers(page, start+copied, start+len);
1963 flush_dcache_page(page);
1965 /* This could be a short (even 0-length) commit */
1966 __block_commit_write(inode, page, start, start+copied);
1970 EXPORT_SYMBOL(block_write_end);
1972 int generic_write_end(struct file *file, struct address_space *mapping,
1973 loff_t pos, unsigned len, unsigned copied,
1974 struct page *page, void *fsdata)
1976 struct inode *inode = mapping->host;
1977 int i_size_changed = 0;
1979 copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
1982 * No need to use i_size_read() here, the i_size
1983 * cannot change under us because we hold i_mutex.
1985 * But it's important to update i_size while still holding page lock:
1986 * page writeout could otherwise come in and zero beyond i_size.
1988 if (pos+copied > inode->i_size) {
1989 i_size_write(inode, pos+copied);
1994 page_cache_release(page);
1997 * Don't mark the inode dirty under page lock. First, it unnecessarily
1998 * makes the holding time of page lock longer. Second, it forces lock
1999 * ordering of page lock and transaction start for journaling
2003 mark_inode_dirty(inode);
2007 EXPORT_SYMBOL(generic_write_end);
2010 * block_is_partially_uptodate checks whether buffers within a page are
2013 * Returns true if all buffers which correspond to a file portion
2014 * we want to read are uptodate.
2016 int block_is_partially_uptodate(struct page *page, read_descriptor_t *desc,
2019 struct inode *inode = page->mapping->host;
2020 unsigned block_start, block_end, blocksize;
2022 struct buffer_head *bh, *head;
2025 if (!page_has_buffers(page))
2028 blocksize = 1 << inode->i_blkbits;
2029 to = min_t(unsigned, PAGE_CACHE_SIZE - from, desc->count);
2031 if (from < blocksize && to > PAGE_CACHE_SIZE - blocksize)
2034 head = page_buffers(page);
2038 block_end = block_start + blocksize;
2039 if (block_end > from && block_start < to) {
2040 if (!buffer_uptodate(bh)) {
2044 if (block_end >= to)
2047 block_start = block_end;
2048 bh = bh->b_this_page;
2049 } while (bh != head);
2053 EXPORT_SYMBOL(block_is_partially_uptodate);
2056 * Generic "read page" function for block devices that have the normal
2057 * get_block functionality. This is most of the block device filesystems.
2058 * Reads the page asynchronously --- the unlock_buffer() and
2059 * set/clear_buffer_uptodate() functions propagate buffer state into the
2060 * page struct once IO has completed.
2062 int block_read_full_page(struct page *page, get_block_t *get_block)
2064 struct inode *inode = page->mapping->host;
2065 sector_t iblock, lblock;
2066 struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2067 unsigned int blocksize;
2069 int fully_mapped = 1;
2071 BUG_ON(!PageLocked(page));
2072 blocksize = 1 << inode->i_blkbits;
2073 if (!page_has_buffers(page))
2074 create_empty_buffers(page, blocksize, 0);
2075 head = page_buffers(page);
2077 iblock = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2078 lblock = (i_size_read(inode)+blocksize-1) >> inode->i_blkbits;
2084 if (buffer_uptodate(bh))
2087 if (!buffer_mapped(bh)) {
2091 if (iblock < lblock) {
2092 WARN_ON(bh->b_size != blocksize);
2093 err = get_block(inode, iblock, bh, 0);
2097 if (!buffer_mapped(bh)) {
2098 zero_user(page, i * blocksize, blocksize);
2100 set_buffer_uptodate(bh);
2104 * get_block() might have updated the buffer
2107 if (buffer_uptodate(bh))
2111 } while (i++, iblock++, (bh = bh->b_this_page) != head);
2114 SetPageMappedToDisk(page);
2118 * All buffers are uptodate - we can set the page uptodate
2119 * as well. But not if get_block() returned an error.
2121 if (!PageError(page))
2122 SetPageUptodate(page);
2127 /* Stage two: lock the buffers */
2128 for (i = 0; i < nr; i++) {
2131 mark_buffer_async_read(bh);
2135 * Stage 3: start the IO. Check for uptodateness
2136 * inside the buffer lock in case another process reading
2137 * the underlying blockdev brought it uptodate (the sct fix).
2139 for (i = 0; i < nr; i++) {
2141 if (buffer_uptodate(bh))
2142 end_buffer_async_read(bh, 1);
2144 submit_bh(READ, bh);
2149 /* utility function for filesystems that need to do work on expanding
2150 * truncates. Uses filesystem pagecache writes to allow the filesystem to
2151 * deal with the hole.
2153 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2155 struct address_space *mapping = inode->i_mapping;
2158 unsigned long limit;
2162 limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
2163 if (limit != RLIM_INFINITY && size > (loff_t)limit) {
2164 send_sig(SIGXFSZ, current, 0);
2167 if (size > inode->i_sb->s_maxbytes)
2170 err = pagecache_write_begin(NULL, mapping, size, 0,
2171 AOP_FLAG_UNINTERRUPTIBLE|AOP_FLAG_CONT_EXPAND,
2176 err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
2183 static int cont_expand_zero(struct file *file, struct address_space *mapping,
2184 loff_t pos, loff_t *bytes)
2186 struct inode *inode = mapping->host;
2187 unsigned blocksize = 1 << inode->i_blkbits;
2190 pgoff_t index, curidx;
2192 unsigned zerofrom, offset, len;
2195 index = pos >> PAGE_CACHE_SHIFT;
2196 offset = pos & ~PAGE_CACHE_MASK;
2198 while (index > (curidx = (curpos = *bytes)>>PAGE_CACHE_SHIFT)) {
2199 zerofrom = curpos & ~PAGE_CACHE_MASK;
2200 if (zerofrom & (blocksize-1)) {
2201 *bytes |= (blocksize-1);
2204 len = PAGE_CACHE_SIZE - zerofrom;
2206 err = pagecache_write_begin(file, mapping, curpos, len,
2207 AOP_FLAG_UNINTERRUPTIBLE,
2211 zero_user(page, zerofrom, len);
2212 err = pagecache_write_end(file, mapping, curpos, len, len,
2219 balance_dirty_pages_ratelimited(mapping);
2222 /* page covers the boundary, find the boundary offset */
2223 if (index == curidx) {
2224 zerofrom = curpos & ~PAGE_CACHE_MASK;
2225 /* if we will expand the thing last block will be filled */
2226 if (offset <= zerofrom) {
2229 if (zerofrom & (blocksize-1)) {
2230 *bytes |= (blocksize-1);
2233 len = offset - zerofrom;
2235 err = pagecache_write_begin(file, mapping, curpos, len,
2236 AOP_FLAG_UNINTERRUPTIBLE,
2240 zero_user(page, zerofrom, len);
2241 err = pagecache_write_end(file, mapping, curpos, len, len,
2253 * For moronic filesystems that do not allow holes in file.
2254 * We may have to extend the file.
2256 int cont_write_begin(struct file *file, struct address_space *mapping,
2257 loff_t pos, unsigned len, unsigned flags,
2258 struct page **pagep, void **fsdata,
2259 get_block_t *get_block, loff_t *bytes)
2261 struct inode *inode = mapping->host;
2262 unsigned blocksize = 1 << inode->i_blkbits;
2266 err = cont_expand_zero(file, mapping, pos, bytes);
2270 zerofrom = *bytes & ~PAGE_CACHE_MASK;
2271 if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2272 *bytes |= (blocksize-1);
2277 err = block_write_begin(file, mapping, pos, len,
2278 flags, pagep, fsdata, get_block);
2283 int block_prepare_write(struct page *page, unsigned from, unsigned to,
2284 get_block_t *get_block)
2286 struct inode *inode = page->mapping->host;
2287 int err = __block_prepare_write(inode, page, from, to, get_block);
2289 ClearPageUptodate(page);
2293 int block_commit_write(struct page *page, unsigned from, unsigned to)
2295 struct inode *inode = page->mapping->host;
2296 __block_commit_write(inode,page,from,to);
2301 * block_page_mkwrite() is not allowed to change the file size as it gets
2302 * called from a page fault handler when a page is first dirtied. Hence we must
2303 * be careful to check for EOF conditions here. We set the page up correctly
2304 * for a written page which means we get ENOSPC checking when writing into
2305 * holes and correct delalloc and unwritten extent mapping on filesystems that
2306 * support these features.
2308 * We are not allowed to take the i_mutex here so we have to play games to
2309 * protect against truncate races as the page could now be beyond EOF. Because
2310 * vmtruncate() writes the inode size before removing pages, once we have the
2311 * page lock we can determine safely if the page is beyond EOF. If it is not
2312 * beyond EOF, then the page is guaranteed safe against truncation until we
2316 block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
2317 get_block_t get_block)
2319 struct page *page = vmf->page;
2320 struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
2326 size = i_size_read(inode);
2327 if ((page->mapping != inode->i_mapping) ||
2328 (page_offset(page) > size)) {
2329 /* page got truncated out from underneath us */
2333 /* page is wholly or partially inside EOF */
2334 if (((page->index + 1) << PAGE_CACHE_SHIFT) > size)
2335 end = size & ~PAGE_CACHE_MASK;
2337 end = PAGE_CACHE_SIZE;
2339 ret = block_prepare_write(page, 0, end, get_block);
2341 ret = block_commit_write(page, 0, end);
2345 ret = VM_FAULT_SIGBUS;
2352 * nobh_write_begin()'s prereads are special: the buffer_heads are freed
2353 * immediately, while under the page lock. So it needs a special end_io
2354 * handler which does not touch the bh after unlocking it.
2356 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2358 __end_buffer_read_notouch(bh, uptodate);
2362 * Attach the singly-linked list of buffers created by nobh_write_begin, to
2363 * the page (converting it to circular linked list and taking care of page
2366 static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
2368 struct buffer_head *bh;
2370 BUG_ON(!PageLocked(page));
2372 spin_lock(&page->mapping->private_lock);
2375 if (PageDirty(page))
2376 set_buffer_dirty(bh);
2377 if (!bh->b_this_page)
2378 bh->b_this_page = head;
2379 bh = bh->b_this_page;
2380 } while (bh != head);
2381 attach_page_buffers(page, head);
2382 spin_unlock(&page->mapping->private_lock);
2386 * On entry, the page is fully not uptodate.
2387 * On exit the page is fully uptodate in the areas outside (from,to)
2389 int nobh_write_begin(struct file *file, struct address_space *mapping,
2390 loff_t pos, unsigned len, unsigned flags,
2391 struct page **pagep, void **fsdata,
2392 get_block_t *get_block)
2394 struct inode *inode = mapping->host;
2395 const unsigned blkbits = inode->i_blkbits;
2396 const unsigned blocksize = 1 << blkbits;
2397 struct buffer_head *head, *bh;
2401 unsigned block_in_page;
2402 unsigned block_start, block_end;
2403 sector_t block_in_file;
2406 int is_mapped_to_disk = 1;
2408 index = pos >> PAGE_CACHE_SHIFT;
2409 from = pos & (PAGE_CACHE_SIZE - 1);
2412 page = grab_cache_page_write_begin(mapping, index, flags);
2418 if (page_has_buffers(page)) {
2420 page_cache_release(page);
2422 return block_write_begin(file, mapping, pos, len, flags, pagep,
2426 if (PageMappedToDisk(page))
2430 * Allocate buffers so that we can keep track of state, and potentially
2431 * attach them to the page if an error occurs. In the common case of
2432 * no error, they will just be freed again without ever being attached
2433 * to the page (which is all OK, because we're under the page lock).
2435 * Be careful: the buffer linked list is a NULL terminated one, rather
2436 * than the circular one we're used to.
2438 head = alloc_page_buffers(page, blocksize, 0);
2444 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
2447 * We loop across all blocks in the page, whether or not they are
2448 * part of the affected region. This is so we can discover if the
2449 * page is fully mapped-to-disk.
2451 for (block_start = 0, block_in_page = 0, bh = head;
2452 block_start < PAGE_CACHE_SIZE;
2453 block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
2456 block_end = block_start + blocksize;
2459 if (block_start >= to)
2461 ret = get_block(inode, block_in_file + block_in_page,
2465 if (!buffer_mapped(bh))
2466 is_mapped_to_disk = 0;
2468 unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
2469 if (PageUptodate(page)) {
2470 set_buffer_uptodate(bh);
2473 if (buffer_new(bh) || !buffer_mapped(bh)) {
2474 zero_user_segments(page, block_start, from,
2478 if (buffer_uptodate(bh))
2479 continue; /* reiserfs does this */
2480 if (block_start < from || block_end > to) {
2482 bh->b_end_io = end_buffer_read_nobh;
2483 submit_bh(READ, bh);
2490 * The page is locked, so these buffers are protected from
2491 * any VM or truncate activity. Hence we don't need to care
2492 * for the buffer_head refcounts.
2494 for (bh = head; bh; bh = bh->b_this_page) {
2496 if (!buffer_uptodate(bh))
2503 if (is_mapped_to_disk)
2504 SetPageMappedToDisk(page);
2506 *fsdata = head; /* to be released by nobh_write_end */
2513 * Error recovery is a bit difficult. We need to zero out blocks that
2514 * were newly allocated, and dirty them to ensure they get written out.
2515 * Buffers need to be attached to the page at this point, otherwise
2516 * the handling of potential IO errors during writeout would be hard
2517 * (could try doing synchronous writeout, but what if that fails too?)
2519 attach_nobh_buffers(page, head);
2520 page_zero_new_buffers(page, from, to);
2524 page_cache_release(page);
2527 if (pos + len > inode->i_size)
2528 vmtruncate(inode, inode->i_size);
2532 EXPORT_SYMBOL(nobh_write_begin);
2534 int nobh_write_end(struct file *file, struct address_space *mapping,
2535 loff_t pos, unsigned len, unsigned copied,
2536 struct page *page, void *fsdata)
2538 struct inode *inode = page->mapping->host;
2539 struct buffer_head *head = fsdata;
2540 struct buffer_head *bh;
2541 BUG_ON(fsdata != NULL && page_has_buffers(page));
2543 if (unlikely(copied < len) && head)
2544 attach_nobh_buffers(page, head);
2545 if (page_has_buffers(page))
2546 return generic_write_end(file, mapping, pos, len,
2547 copied, page, fsdata);
2549 SetPageUptodate(page);
2550 set_page_dirty(page);
2551 if (pos+copied > inode->i_size) {
2552 i_size_write(inode, pos+copied);
2553 mark_inode_dirty(inode);
2557 page_cache_release(page);
2561 head = head->b_this_page;
2562 free_buffer_head(bh);
2567 EXPORT_SYMBOL(nobh_write_end);
2570 * nobh_writepage() - based on block_full_write_page() except
2571 * that it tries to operate without attaching bufferheads to
2574 int nobh_writepage(struct page *page, get_block_t *get_block,
2575 struct writeback_control *wbc)
2577 struct inode * const inode = page->mapping->host;
2578 loff_t i_size = i_size_read(inode);
2579 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2583 /* Is the page fully inside i_size? */
2584 if (page->index < end_index)
2587 /* Is the page fully outside i_size? (truncate in progress) */
2588 offset = i_size & (PAGE_CACHE_SIZE-1);
2589 if (page->index >= end_index+1 || !offset) {
2591 * The page may have dirty, unmapped buffers. For example,
2592 * they may have been added in ext3_writepage(). Make them
2593 * freeable here, so the page does not leak.
2596 /* Not really sure about this - do we need this ? */
2597 if (page->mapping->a_ops->invalidatepage)
2598 page->mapping->a_ops->invalidatepage(page, offset);
2601 return 0; /* don't care */
2605 * The page straddles i_size. It must be zeroed out on each and every
2606 * writepage invocation because it may be mmapped. "A file is mapped
2607 * in multiples of the page size. For a file that is not a multiple of
2608 * the page size, the remaining memory is zeroed when mapped, and
2609 * writes to that region are not written out to the file."
2611 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
2613 ret = mpage_writepage(page, get_block, wbc);
2615 ret = __block_write_full_page(inode, page, get_block, wbc);
2618 EXPORT_SYMBOL(nobh_writepage);
2620 int nobh_truncate_page(struct address_space *mapping,
2621 loff_t from, get_block_t *get_block)
2623 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2624 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2627 unsigned length, pos;
2628 struct inode *inode = mapping->host;
2630 struct buffer_head map_bh;
2633 blocksize = 1 << inode->i_blkbits;
2634 length = offset & (blocksize - 1);
2636 /* Block boundary? Nothing to do */
2640 length = blocksize - length;
2641 iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2643 page = grab_cache_page(mapping, index);
2648 if (page_has_buffers(page)) {
2651 page_cache_release(page);
2652 return block_truncate_page(mapping, from, get_block);
2655 /* Find the buffer that contains "offset" */
2657 while (offset >= pos) {
2662 err = get_block(inode, iblock, &map_bh, 0);
2665 /* unmapped? It's a hole - nothing to do */
2666 if (!buffer_mapped(&map_bh))
2669 /* Ok, it's mapped. Make sure it's up-to-date */
2670 if (!PageUptodate(page)) {
2671 err = mapping->a_ops->readpage(NULL, page);
2673 page_cache_release(page);
2677 if (!PageUptodate(page)) {
2681 if (page_has_buffers(page))
2684 zero_user(page, offset, length);
2685 set_page_dirty(page);
2690 page_cache_release(page);
2694 EXPORT_SYMBOL(nobh_truncate_page);
2696 int block_truncate_page(struct address_space *mapping,
2697 loff_t from, get_block_t *get_block)
2699 pgoff_t index = from >> PAGE_CACHE_SHIFT;
2700 unsigned offset = from & (PAGE_CACHE_SIZE-1);
2703 unsigned length, pos;
2704 struct inode *inode = mapping->host;
2706 struct buffer_head *bh;
2709 blocksize = 1 << inode->i_blkbits;
2710 length = offset & (blocksize - 1);
2712 /* Block boundary? Nothing to do */
2716 length = blocksize - length;
2717 iblock = (sector_t)index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
2719 page = grab_cache_page(mapping, index);
2724 if (!page_has_buffers(page))
2725 create_empty_buffers(page, blocksize, 0);
2727 /* Find the buffer that contains "offset" */
2728 bh = page_buffers(page);
2730 while (offset >= pos) {
2731 bh = bh->b_this_page;
2737 if (!buffer_mapped(bh)) {
2738 WARN_ON(bh->b_size != blocksize);
2739 err = get_block(inode, iblock, bh, 0);
2742 /* unmapped? It's a hole - nothing to do */
2743 if (!buffer_mapped(bh))
2747 /* Ok, it's mapped. Make sure it's up-to-date */
2748 if (PageUptodate(page))
2749 set_buffer_uptodate(bh);
2751 if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2753 ll_rw_block(READ, 1, &bh);
2755 /* Uhhuh. Read error. Complain and punt. */
2756 if (!buffer_uptodate(bh))
2760 zero_user(page, offset, length);
2761 mark_buffer_dirty(bh);
2766 page_cache_release(page);
2772 * The generic ->writepage function for buffer-backed address_spaces
2774 int block_write_full_page(struct page *page, get_block_t *get_block,
2775 struct writeback_control *wbc)
2777 struct inode * const inode = page->mapping->host;
2778 loff_t i_size = i_size_read(inode);
2779 const pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2782 /* Is the page fully inside i_size? */
2783 if (page->index < end_index)
2784 return __block_write_full_page(inode, page, get_block, wbc);
2786 /* Is the page fully outside i_size? (truncate in progress) */
2787 offset = i_size & (PAGE_CACHE_SIZE-1);
2788 if (page->index >= end_index+1 || !offset) {
2790 * The page may have dirty, unmapped buffers. For example,
2791 * they may have been added in ext3_writepage(). Make them
2792 * freeable here, so the page does not leak.
2794 do_invalidatepage(page, 0);
2796 return 0; /* don't care */
2800 * The page straddles i_size. It must be zeroed out on each and every
2801 * writepage invokation because it may be mmapped. "A file is mapped
2802 * in multiples of the page size. For a file that is not a multiple of
2803 * the page size, the remaining memory is zeroed when mapped, and
2804 * writes to that region are not written out to the file."
2806 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
2807 return __block_write_full_page(inode, page, get_block, wbc);
2810 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2811 get_block_t *get_block)
2813 struct buffer_head tmp;
2814 struct inode *inode = mapping->host;
2817 tmp.b_size = 1 << inode->i_blkbits;
2818 get_block(inode, block, &tmp, 0);
2819 return tmp.b_blocknr;
2822 static void end_bio_bh_io_sync(struct bio *bio, int err)
2824 struct buffer_head *bh = bio->bi_private;
2826 if (err == -EOPNOTSUPP) {
2827 set_bit(BIO_EOPNOTSUPP, &bio->bi_flags);
2828 set_bit(BH_Eopnotsupp, &bh->b_state);
2831 if (unlikely (test_bit(BIO_QUIET,&bio->bi_flags)))
2832 set_bit(BH_Quiet, &bh->b_state);
2834 bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags));
2838 int submit_bh(int rw, struct buffer_head * bh)
2843 BUG_ON(!buffer_locked(bh));
2844 BUG_ON(!buffer_mapped(bh));
2845 BUG_ON(!bh->b_end_io);
2848 * Mask in barrier bit for a write (could be either a WRITE or a
2851 if (buffer_ordered(bh) && (rw & WRITE))
2852 rw |= WRITE_BARRIER;
2855 * Only clear out a write error when rewriting
2857 if (test_set_buffer_req(bh) && (rw & WRITE))
2858 clear_buffer_write_io_error(bh);
2861 * from here on down, it's all bio -- do the initial mapping,
2862 * submit_bio -> generic_make_request may further map this bio around
2864 bio = bio_alloc(GFP_NOIO, 1);
2866 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2867 bio->bi_bdev = bh->b_bdev;
2868 bio->bi_io_vec[0].bv_page = bh->b_page;
2869 bio->bi_io_vec[0].bv_len = bh->b_size;
2870 bio->bi_io_vec[0].bv_offset = bh_offset(bh);
2874 bio->bi_size = bh->b_size;
2876 bio->bi_end_io = end_bio_bh_io_sync;
2877 bio->bi_private = bh;
2880 submit_bio(rw, bio);
2882 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2890 * ll_rw_block: low-level access to block devices (DEPRECATED)
2891 * @rw: whether to %READ or %WRITE or %SWRITE or maybe %READA (readahead)
2892 * @nr: number of &struct buffer_heads in the array
2893 * @bhs: array of pointers to &struct buffer_head
2895 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
2896 * requests an I/O operation on them, either a %READ or a %WRITE. The third
2897 * %SWRITE is like %WRITE only we make sure that the *current* data in buffers
2898 * are sent to disk. The fourth %READA option is described in the documentation
2899 * for generic_make_request() which ll_rw_block() calls.
2901 * This function drops any buffer that it cannot get a lock on (with the
2902 * BH_Lock state bit) unless SWRITE is required, any buffer that appears to be
2903 * clean when doing a write request, and any buffer that appears to be
2904 * up-to-date when doing read request. Further it marks as clean buffers that
2905 * are processed for writing (the buffer cache won't assume that they are
2906 * actually clean until the buffer gets unlocked).
2908 * ll_rw_block sets b_end_io to simple completion handler that marks
2909 * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2912 * All of the buffers must be for the same device, and must also be a
2913 * multiple of the current approved size for the device.
2915 void ll_rw_block(int rw, int nr, struct buffer_head *bhs[])
2919 for (i = 0; i < nr; i++) {
2920 struct buffer_head *bh = bhs[i];
2922 if (rw == SWRITE || rw == SWRITE_SYNC)
2924 else if (!trylock_buffer(bh))
2927 if (rw == WRITE || rw == SWRITE || rw == SWRITE_SYNC) {
2928 if (test_clear_buffer_dirty(bh)) {
2929 bh->b_end_io = end_buffer_write_sync;
2931 if (rw == SWRITE_SYNC)
2932 submit_bh(WRITE_SYNC, bh);
2934 submit_bh(WRITE, bh);
2938 if (!buffer_uptodate(bh)) {
2939 bh->b_end_io = end_buffer_read_sync;
2950 * For a data-integrity writeout, we need to wait upon any in-progress I/O
2951 * and then start new I/O and then wait upon it. The caller must have a ref on
2954 int sync_dirty_buffer(struct buffer_head *bh)
2958 WARN_ON(atomic_read(&bh->b_count) < 1);
2960 if (test_clear_buffer_dirty(bh)) {
2962 bh->b_end_io = end_buffer_write_sync;
2963 ret = submit_bh(WRITE, bh);
2965 if (buffer_eopnotsupp(bh)) {
2966 clear_buffer_eopnotsupp(bh);
2969 if (!ret && !buffer_uptodate(bh))
2978 * try_to_free_buffers() checks if all the buffers on this particular page
2979 * are unused, and releases them if so.
2981 * Exclusion against try_to_free_buffers may be obtained by either
2982 * locking the page or by holding its mapping's private_lock.
2984 * If the page is dirty but all the buffers are clean then we need to
2985 * be sure to mark the page clean as well. This is because the page
2986 * may be against a block device, and a later reattachment of buffers
2987 * to a dirty page will set *all* buffers dirty. Which would corrupt
2988 * filesystem data on the same device.
2990 * The same applies to regular filesystem pages: if all the buffers are
2991 * clean then we set the page clean and proceed. To do that, we require
2992 * total exclusion from __set_page_dirty_buffers(). That is obtained with
2995 * try_to_free_buffers() is non-blocking.
2997 static inline int buffer_busy(struct buffer_head *bh)
2999 return atomic_read(&bh->b_count) |
3000 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
3004 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
3006 struct buffer_head *head = page_buffers(page);
3007 struct buffer_head *bh;
3011 if (buffer_write_io_error(bh) && page->mapping)
3012 set_bit(AS_EIO, &page->mapping->flags);
3013 if (buffer_busy(bh))
3015 bh = bh->b_this_page;
3016 } while (bh != head);
3019 struct buffer_head *next = bh->b_this_page;
3021 if (bh->b_assoc_map)
3022 __remove_assoc_queue(bh);
3024 } while (bh != head);
3025 *buffers_to_free = head;
3026 __clear_page_buffers(page);
3032 int try_to_free_buffers(struct page *page)
3034 struct address_space * const mapping = page->mapping;
3035 struct buffer_head *buffers_to_free = NULL;
3038 BUG_ON(!PageLocked(page));
3039 if (PageWriteback(page))
3042 if (mapping == NULL) { /* can this still happen? */
3043 ret = drop_buffers(page, &buffers_to_free);
3047 spin_lock(&mapping->private_lock);
3048 ret = drop_buffers(page, &buffers_to_free);
3051 * If the filesystem writes its buffers by hand (eg ext3)
3052 * then we can have clean buffers against a dirty page. We
3053 * clean the page here; otherwise the VM will never notice
3054 * that the filesystem did any IO at all.
3056 * Also, during truncate, discard_buffer will have marked all
3057 * the page's buffers clean. We discover that here and clean
3060 * private_lock must be held over this entire operation in order
3061 * to synchronise against __set_page_dirty_buffers and prevent the
3062 * dirty bit from being lost.
3065 cancel_dirty_page(page, PAGE_CACHE_SIZE);
3066 spin_unlock(&mapping->private_lock);
3068 if (buffers_to_free) {
3069 struct buffer_head *bh = buffers_to_free;
3072 struct buffer_head *next = bh->b_this_page;
3073 free_buffer_head(bh);
3075 } while (bh != buffers_to_free);
3079 EXPORT_SYMBOL(try_to_free_buffers);
3081 void block_sync_page(struct page *page)
3083 struct address_space *mapping;
3086 mapping = page_mapping(page);
3088 blk_run_backing_dev(mapping->backing_dev_info, page);
3092 * There are no bdflush tunables left. But distributions are
3093 * still running obsolete flush daemons, so we terminate them here.
3095 * Use of bdflush() is deprecated and will be removed in a future kernel.
3096 * The `pdflush' kernel threads fully replace bdflush daemons and this call.
3098 SYSCALL_DEFINE2(bdflush, int, func, long, data)
3100 static int msg_count;
3102 if (!capable(CAP_SYS_ADMIN))
3105 if (msg_count < 5) {
3108 "warning: process `%s' used the obsolete bdflush"
3109 " system call\n", current->comm);
3110 printk(KERN_INFO "Fix your initscripts?\n");
3119 * Buffer-head allocation
3121 static struct kmem_cache *bh_cachep;
3124 * Once the number of bh's in the machine exceeds this level, we start
3125 * stripping them in writeback.
3127 static int max_buffer_heads;
3129 int buffer_heads_over_limit;
3131 struct bh_accounting {
3132 int nr; /* Number of live bh's */
3133 int ratelimit; /* Limit cacheline bouncing */
3136 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3138 static void recalc_bh_state(void)
3143 if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
3145 __get_cpu_var(bh_accounting).ratelimit = 0;
3146 for_each_online_cpu(i)
3147 tot += per_cpu(bh_accounting, i).nr;
3148 buffer_heads_over_limit = (tot > max_buffer_heads);
3151 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3153 struct buffer_head *ret = kmem_cache_alloc(bh_cachep, gfp_flags);
3155 INIT_LIST_HEAD(&ret->b_assoc_buffers);
3156 get_cpu_var(bh_accounting).nr++;
3158 put_cpu_var(bh_accounting);
3162 EXPORT_SYMBOL(alloc_buffer_head);
3164 void free_buffer_head(struct buffer_head *bh)
3166 BUG_ON(!list_empty(&bh->b_assoc_buffers));
3167 kmem_cache_free(bh_cachep, bh);
3168 get_cpu_var(bh_accounting).nr--;
3170 put_cpu_var(bh_accounting);
3172 EXPORT_SYMBOL(free_buffer_head);
3174 static void buffer_exit_cpu(int cpu)
3177 struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3179 for (i = 0; i < BH_LRU_SIZE; i++) {
3183 get_cpu_var(bh_accounting).nr += per_cpu(bh_accounting, cpu).nr;
3184 per_cpu(bh_accounting, cpu).nr = 0;
3185 put_cpu_var(bh_accounting);
3188 static int buffer_cpu_notify(struct notifier_block *self,
3189 unsigned long action, void *hcpu)
3191 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
3192 buffer_exit_cpu((unsigned long)hcpu);
3197 * bh_uptodate_or_lock - Test whether the buffer is uptodate
3198 * @bh: struct buffer_head
3200 * Return true if the buffer is up-to-date and false,
3201 * with the buffer locked, if not.
3203 int bh_uptodate_or_lock(struct buffer_head *bh)
3205 if (!buffer_uptodate(bh)) {
3207 if (!buffer_uptodate(bh))
3213 EXPORT_SYMBOL(bh_uptodate_or_lock);
3216 * bh_submit_read - Submit a locked buffer for reading
3217 * @bh: struct buffer_head
3219 * Returns zero on success and -EIO on error.
3221 int bh_submit_read(struct buffer_head *bh)
3223 BUG_ON(!buffer_locked(bh));
3225 if (buffer_uptodate(bh)) {
3231 bh->b_end_io = end_buffer_read_sync;
3232 submit_bh(READ, bh);
3234 if (buffer_uptodate(bh))
3238 EXPORT_SYMBOL(bh_submit_read);
3241 init_buffer_head(void *data)
3243 struct buffer_head *bh = data;
3245 memset(bh, 0, sizeof(*bh));
3246 INIT_LIST_HEAD(&bh->b_assoc_buffers);
3249 void __init buffer_init(void)
3253 bh_cachep = kmem_cache_create("buffer_head",
3254 sizeof(struct buffer_head), 0,
3255 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3260 * Limit the bh occupancy to 10% of ZONE_NORMAL
3262 nrpages = (nr_free_buffer_pages() * 10) / 100;
3263 max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3264 hotcpu_notifier(buffer_cpu_notify, 0);
3267 EXPORT_SYMBOL(__bforget);
3268 EXPORT_SYMBOL(__brelse);
3269 EXPORT_SYMBOL(__wait_on_buffer);
3270 EXPORT_SYMBOL(block_commit_write);
3271 EXPORT_SYMBOL(block_prepare_write);
3272 EXPORT_SYMBOL(block_page_mkwrite);
3273 EXPORT_SYMBOL(block_read_full_page);
3274 EXPORT_SYMBOL(block_sync_page);
3275 EXPORT_SYMBOL(block_truncate_page);
3276 EXPORT_SYMBOL(block_write_full_page);
3277 EXPORT_SYMBOL(cont_write_begin);
3278 EXPORT_SYMBOL(end_buffer_read_sync);
3279 EXPORT_SYMBOL(end_buffer_write_sync);
3280 EXPORT_SYMBOL(file_fsync);
3281 EXPORT_SYMBOL(fsync_bdev);
3282 EXPORT_SYMBOL(generic_block_bmap);
3283 EXPORT_SYMBOL(generic_cont_expand_simple);
3284 EXPORT_SYMBOL(init_buffer);
3285 EXPORT_SYMBOL(invalidate_bdev);
3286 EXPORT_SYMBOL(ll_rw_block);
3287 EXPORT_SYMBOL(mark_buffer_dirty);
3288 EXPORT_SYMBOL(submit_bh);
3289 EXPORT_SYMBOL(sync_dirty_buffer);
3290 EXPORT_SYMBOL(unlock_buffer);