2 * Copyright (c) 2000-2006 Silicon Graphics, Inc.
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
7 * published by the Free Software Foundation.
9 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
19 #include <linux/stddef.h>
20 #include <linux/errno.h>
21 #include <linux/slab.h>
22 #include <linux/pagemap.h>
23 #include <linux/init.h>
24 #include <linux/vmalloc.h>
25 #include <linux/bio.h>
26 #include <linux/sysctl.h>
27 #include <linux/proc_fs.h>
28 #include <linux/workqueue.h>
29 #include <linux/percpu.h>
30 #include <linux/blkdev.h>
31 #include <linux/hash.h>
32 #include <linux/kthread.h>
33 #include <linux/migrate.h>
34 #include <linux/backing-dev.h>
35 #include <linux/freezer.h>
37 static kmem_zone_t *xfs_buf_zone;
38 STATIC int xfsbufd(void *);
39 STATIC int xfsbufd_wakeup(int, gfp_t);
40 STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
41 static struct shrinker xfs_buf_shake = {
42 .shrink = xfsbufd_wakeup,
43 .seeks = DEFAULT_SEEKS,
46 static struct workqueue_struct *xfslogd_workqueue;
47 struct workqueue_struct *xfsdatad_workqueue;
57 ktrace_enter(xfs_buf_trace_buf,
59 (void *)(unsigned long)bp->b_flags,
60 (void *)(unsigned long)bp->b_hold.counter,
61 (void *)(unsigned long)bp->b_sema.count,
64 (void *)(unsigned long)((bp->b_file_offset>>32) & 0xffffffff),
65 (void *)(unsigned long)(bp->b_file_offset & 0xffffffff),
66 (void *)(unsigned long)bp->b_buffer_length,
67 NULL, NULL, NULL, NULL, NULL);
69 ktrace_t *xfs_buf_trace_buf;
70 #define XFS_BUF_TRACE_SIZE 4096
71 #define XB_TRACE(bp, id, data) \
72 xfs_buf_trace(bp, id, (void *)data, (void *)__builtin_return_address(0))
74 #define XB_TRACE(bp, id, data) do { } while (0)
77 #ifdef XFS_BUF_LOCK_TRACKING
78 # define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid)
79 # define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1)
80 # define XB_GET_OWNER(bp) ((bp)->b_last_holder)
82 # define XB_SET_OWNER(bp) do { } while (0)
83 # define XB_CLEAR_OWNER(bp) do { } while (0)
84 # define XB_GET_OWNER(bp) do { } while (0)
87 #define xb_to_gfp(flags) \
88 ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
89 ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
91 #define xb_to_km(flags) \
92 (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
94 #define xfs_buf_allocate(flags) \
95 kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
96 #define xfs_buf_deallocate(bp) \
97 kmem_zone_free(xfs_buf_zone, (bp));
100 * Page Region interfaces.
102 * For pages in filesystems where the blocksize is smaller than the
103 * pagesize, we use the page->private field (long) to hold a bitmap
104 * of uptodate regions within the page.
106 * Each such region is "bytes per page / bits per long" bytes long.
108 * NBPPR == number-of-bytes-per-page-region
109 * BTOPR == bytes-to-page-region (rounded up)
110 * BTOPRT == bytes-to-page-region-truncated (rounded down)
112 #if (BITS_PER_LONG == 32)
113 #define PRSHIFT (PAGE_CACHE_SHIFT - 5) /* (32 == 1<<5) */
114 #elif (BITS_PER_LONG == 64)
115 #define PRSHIFT (PAGE_CACHE_SHIFT - 6) /* (64 == 1<<6) */
117 #error BITS_PER_LONG must be 32 or 64
119 #define NBPPR (PAGE_CACHE_SIZE/BITS_PER_LONG)
120 #define BTOPR(b) (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
121 #define BTOPRT(b) (((unsigned int)(b) >> PRSHIFT))
131 first = BTOPR(offset);
132 final = BTOPRT(offset + length - 1);
133 first = min(first, final);
136 mask <<= BITS_PER_LONG - (final - first);
137 mask >>= BITS_PER_LONG - (final);
139 ASSERT(offset + length <= PAGE_CACHE_SIZE);
140 ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
151 set_page_private(page,
152 page_private(page) | page_region_mask(offset, length));
153 if (page_private(page) == ~0UL)
154 SetPageUptodate(page);
163 unsigned long mask = page_region_mask(offset, length);
165 return (mask && (page_private(page) & mask) == mask);
169 * Mapping of multi-page buffers into contiguous virtual space
172 typedef struct a_list {
177 static a_list_t *as_free_head;
178 static int as_list_len;
179 static DEFINE_SPINLOCK(as_lock);
182 * Try to batch vunmaps because they are costly.
192 * Xen needs to be able to make sure it can get an exclusive
193 * RO mapping of pages it wants to turn into a pagetable. If
194 * a newly allocated page is also still being vmap()ed by xfs,
195 * it will cause pagetable construction to fail. This is a
196 * quick workaround to always eagerly unmap pages so that Xen
203 aentry = kmalloc(sizeof(a_list_t), GFP_NOWAIT);
204 if (likely(aentry)) {
206 aentry->next = as_free_head;
207 aentry->vm_addr = addr;
208 as_free_head = aentry;
210 spin_unlock(&as_lock);
217 purge_addresses(void)
219 a_list_t *aentry, *old;
221 if (as_free_head == NULL)
225 aentry = as_free_head;
228 spin_unlock(&as_lock);
230 while ((old = aentry) != NULL) {
231 vunmap(aentry->vm_addr);
232 aentry = aentry->next;
238 * Internal xfs_buf_t object manipulation
244 xfs_buftarg_t *target,
245 xfs_off_t range_base,
247 xfs_buf_flags_t flags)
250 * We don't want certain flags to appear in b_flags.
252 flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
254 memset(bp, 0, sizeof(xfs_buf_t));
255 atomic_set(&bp->b_hold, 1);
256 init_completion(&bp->b_iowait);
257 INIT_LIST_HEAD(&bp->b_list);
258 INIT_LIST_HEAD(&bp->b_hash_list);
259 init_MUTEX_LOCKED(&bp->b_sema); /* held, no waiters */
261 bp->b_target = target;
262 bp->b_file_offset = range_base;
264 * Set buffer_length and count_desired to the same value initially.
265 * I/O routines should use count_desired, which will be the same in
266 * most cases but may be reset (e.g. XFS recovery).
268 bp->b_buffer_length = bp->b_count_desired = range_length;
270 bp->b_bn = XFS_BUF_DADDR_NULL;
271 atomic_set(&bp->b_pin_count, 0);
272 init_waitqueue_head(&bp->b_waiters);
274 XFS_STATS_INC(xb_create);
275 XB_TRACE(bp, "initialize", target);
279 * Allocate a page array capable of holding a specified number
280 * of pages, and point the page buf at it.
286 xfs_buf_flags_t flags)
288 /* Make sure that we have a page list */
289 if (bp->b_pages == NULL) {
290 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
291 bp->b_page_count = page_count;
292 if (page_count <= XB_PAGES) {
293 bp->b_pages = bp->b_page_array;
295 bp->b_pages = kmem_alloc(sizeof(struct page *) *
296 page_count, xb_to_km(flags));
297 if (bp->b_pages == NULL)
300 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
306 * Frees b_pages if it was allocated.
312 if (bp->b_pages != bp->b_page_array) {
313 kmem_free(bp->b_pages);
318 * Releases the specified buffer.
320 * The modification state of any associated pages is left unchanged.
321 * The buffer most not be on any hash - use xfs_buf_rele instead for
322 * hashed and refcounted buffers
328 XB_TRACE(bp, "free", 0);
330 ASSERT(list_empty(&bp->b_hash_list));
332 if (bp->b_flags & (_XBF_PAGE_CACHE|_XBF_PAGES)) {
335 if ((bp->b_flags & XBF_MAPPED) && (bp->b_page_count > 1))
336 free_address(bp->b_addr - bp->b_offset);
338 for (i = 0; i < bp->b_page_count; i++) {
339 struct page *page = bp->b_pages[i];
341 if (bp->b_flags & _XBF_PAGE_CACHE)
342 ASSERT(!PagePrivate(page));
343 page_cache_release(page);
345 _xfs_buf_free_pages(bp);
348 xfs_buf_deallocate(bp);
352 * Finds all pages for buffer in question and builds it's page list.
355 _xfs_buf_lookup_pages(
359 struct address_space *mapping = bp->b_target->bt_mapping;
360 size_t blocksize = bp->b_target->bt_bsize;
361 size_t size = bp->b_count_desired;
362 size_t nbytes, offset;
363 gfp_t gfp_mask = xb_to_gfp(flags);
364 unsigned short page_count, i;
369 end = bp->b_file_offset + bp->b_buffer_length;
370 page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
372 error = _xfs_buf_get_pages(bp, page_count, flags);
375 bp->b_flags |= _XBF_PAGE_CACHE;
377 offset = bp->b_offset;
378 first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
380 for (i = 0; i < bp->b_page_count; i++) {
385 page = find_or_create_page(mapping, first + i, gfp_mask);
386 if (unlikely(page == NULL)) {
387 if (flags & XBF_READ_AHEAD) {
388 bp->b_page_count = i;
389 for (i = 0; i < bp->b_page_count; i++)
390 unlock_page(bp->b_pages[i]);
395 * This could deadlock.
397 * But until all the XFS lowlevel code is revamped to
398 * handle buffer allocation failures we can't do much.
400 if (!(++retries % 100))
402 "XFS: possible memory allocation "
403 "deadlock in %s (mode:0x%x)\n",
406 XFS_STATS_INC(xb_page_retries);
407 xfsbufd_wakeup(0, gfp_mask);
408 congestion_wait(WRITE, HZ/50);
412 XFS_STATS_INC(xb_page_found);
414 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
417 ASSERT(!PagePrivate(page));
418 if (!PageUptodate(page)) {
420 if (blocksize >= PAGE_CACHE_SIZE) {
421 if (flags & XBF_READ)
422 bp->b_flags |= _XBF_PAGE_LOCKED;
423 } else if (!PagePrivate(page)) {
424 if (test_page_region(page, offset, nbytes))
429 bp->b_pages[i] = page;
433 if (!(bp->b_flags & _XBF_PAGE_LOCKED)) {
434 for (i = 0; i < bp->b_page_count; i++)
435 unlock_page(bp->b_pages[i]);
438 if (page_count == bp->b_page_count)
439 bp->b_flags |= XBF_DONE;
441 XB_TRACE(bp, "lookup_pages", (long)page_count);
446 * Map buffer into kernel address-space if nessecary.
453 /* A single page buffer is always mappable */
454 if (bp->b_page_count == 1) {
455 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
456 bp->b_flags |= XBF_MAPPED;
457 } else if (flags & XBF_MAPPED) {
458 if (as_list_len > 64)
460 bp->b_addr = vmap(bp->b_pages, bp->b_page_count,
461 VM_MAP, PAGE_KERNEL);
462 if (unlikely(bp->b_addr == NULL))
464 bp->b_addr += bp->b_offset;
465 bp->b_flags |= XBF_MAPPED;
472 * Finding and Reading Buffers
476 * Look up, and creates if absent, a lockable buffer for
477 * a given range of an inode. The buffer is returned
478 * locked. If other overlapping buffers exist, they are
479 * released before the new buffer is created and locked,
480 * which may imply that this call will block until those buffers
481 * are unlocked. No I/O is implied by this call.
485 xfs_buftarg_t *btp, /* block device target */
486 xfs_off_t ioff, /* starting offset of range */
487 size_t isize, /* length of range */
488 xfs_buf_flags_t flags,
491 xfs_off_t range_base;
496 range_base = (ioff << BBSHIFT);
497 range_length = (isize << BBSHIFT);
499 /* Check for IOs smaller than the sector size / not sector aligned */
500 ASSERT(!(range_length < (1 << btp->bt_sshift)));
501 ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
503 hash = &btp->bt_hash[hash_long((unsigned long)ioff, btp->bt_hashshift)];
505 spin_lock(&hash->bh_lock);
507 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
508 ASSERT(btp == bp->b_target);
509 if (bp->b_file_offset == range_base &&
510 bp->b_buffer_length == range_length) {
512 * If we look at something, bring it to the
513 * front of the list for next time.
515 atomic_inc(&bp->b_hold);
516 list_move(&bp->b_hash_list, &hash->bh_list);
523 _xfs_buf_initialize(new_bp, btp, range_base,
524 range_length, flags);
525 new_bp->b_hash = hash;
526 list_add(&new_bp->b_hash_list, &hash->bh_list);
528 XFS_STATS_INC(xb_miss_locked);
531 spin_unlock(&hash->bh_lock);
535 spin_unlock(&hash->bh_lock);
537 /* Attempt to get the semaphore without sleeping,
538 * if this does not work then we need to drop the
539 * spinlock and do a hard attempt on the semaphore.
541 if (down_trylock(&bp->b_sema)) {
542 if (!(flags & XBF_TRYLOCK)) {
543 /* wait for buffer ownership */
544 XB_TRACE(bp, "get_lock", 0);
546 XFS_STATS_INC(xb_get_locked_waited);
548 /* We asked for a trylock and failed, no need
549 * to look at file offset and length here, we
550 * know that this buffer at least overlaps our
551 * buffer and is locked, therefore our buffer
552 * either does not exist, or is this buffer.
555 XFS_STATS_INC(xb_busy_locked);
563 if (bp->b_flags & XBF_STALE) {
564 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
565 bp->b_flags &= XBF_MAPPED;
567 XB_TRACE(bp, "got_lock", 0);
568 XFS_STATS_INC(xb_get_locked);
573 * Assembles a buffer covering the specified range.
574 * Storage in memory for all portions of the buffer will be allocated,
575 * although backing storage may not be.
579 xfs_buftarg_t *target,/* target for buffer */
580 xfs_off_t ioff, /* starting offset of range */
581 size_t isize, /* length of range */
582 xfs_buf_flags_t flags)
584 xfs_buf_t *bp, *new_bp;
587 new_bp = xfs_buf_allocate(flags);
588 if (unlikely(!new_bp))
591 bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
593 error = _xfs_buf_lookup_pages(bp, flags);
597 xfs_buf_deallocate(new_bp);
598 if (unlikely(bp == NULL))
602 for (i = 0; i < bp->b_page_count; i++)
603 mark_page_accessed(bp->b_pages[i]);
605 if (!(bp->b_flags & XBF_MAPPED)) {
606 error = _xfs_buf_map_pages(bp, flags);
607 if (unlikely(error)) {
608 printk(KERN_WARNING "%s: failed to map pages\n",
614 XFS_STATS_INC(xb_get);
617 * Always fill in the block number now, the mapped cases can do
618 * their own overlay of this later.
621 bp->b_count_desired = bp->b_buffer_length;
623 XB_TRACE(bp, "get", (unsigned long)flags);
627 if (flags & (XBF_LOCK | XBF_TRYLOCK))
636 xfs_buf_flags_t flags)
640 XB_TRACE(bp, "_xfs_buf_read", (unsigned long)flags);
642 ASSERT(!(flags & (XBF_DELWRI|XBF_WRITE)));
643 ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
645 bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
646 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
647 bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | \
648 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
650 status = xfs_buf_iorequest(bp);
651 if (!status && !(flags & XBF_ASYNC))
652 status = xfs_buf_iowait(bp);
658 xfs_buftarg_t *target,
661 xfs_buf_flags_t flags)
667 bp = xfs_buf_get_flags(target, ioff, isize, flags);
669 if (!XFS_BUF_ISDONE(bp)) {
670 XB_TRACE(bp, "read", (unsigned long)flags);
671 XFS_STATS_INC(xb_get_read);
672 _xfs_buf_read(bp, flags);
673 } else if (flags & XBF_ASYNC) {
674 XB_TRACE(bp, "read_async", (unsigned long)flags);
676 * Read ahead call which is already satisfied,
681 XB_TRACE(bp, "read_done", (unsigned long)flags);
682 /* We do not want read in the flags */
683 bp->b_flags &= ~XBF_READ;
690 if (flags & (XBF_LOCK | XBF_TRYLOCK))
697 * If we are not low on memory then do the readahead in a deadlock
702 xfs_buftarg_t *target,
705 xfs_buf_flags_t flags)
707 struct backing_dev_info *bdi;
709 bdi = target->bt_mapping->backing_dev_info;
710 if (bdi_read_congested(bdi))
713 flags |= (XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD);
714 xfs_buf_read_flags(target, ioff, isize, flags);
720 xfs_buftarg_t *target)
724 bp = xfs_buf_allocate(0);
726 _xfs_buf_initialize(bp, target, 0, len, 0);
730 static inline struct page *
734 if ((!is_vmalloc_addr(addr))) {
735 return virt_to_page(addr);
737 return vmalloc_to_page(addr);
742 xfs_buf_associate_memory(
749 unsigned long pageaddr;
750 unsigned long offset;
754 pageaddr = (unsigned long)mem & PAGE_CACHE_MASK;
755 offset = (unsigned long)mem - pageaddr;
756 buflen = PAGE_CACHE_ALIGN(len + offset);
757 page_count = buflen >> PAGE_CACHE_SHIFT;
759 /* Free any previous set of page pointers */
761 _xfs_buf_free_pages(bp);
766 rval = _xfs_buf_get_pages(bp, page_count, 0);
770 bp->b_offset = offset;
772 for (i = 0; i < bp->b_page_count; i++) {
773 bp->b_pages[i] = mem_to_page((void *)pageaddr);
774 pageaddr += PAGE_CACHE_SIZE;
777 bp->b_count_desired = len;
778 bp->b_buffer_length = buflen;
779 bp->b_flags |= XBF_MAPPED;
780 bp->b_flags &= ~_XBF_PAGE_LOCKED;
788 xfs_buftarg_t *target)
790 unsigned long page_count = PAGE_ALIGN(len) >> PAGE_SHIFT;
794 bp = xfs_buf_allocate(0);
795 if (unlikely(bp == NULL))
797 _xfs_buf_initialize(bp, target, 0, len, 0);
799 error = _xfs_buf_get_pages(bp, page_count, 0);
803 for (i = 0; i < page_count; i++) {
804 bp->b_pages[i] = alloc_page(GFP_KERNEL);
808 bp->b_flags |= _XBF_PAGES;
810 error = _xfs_buf_map_pages(bp, XBF_MAPPED);
811 if (unlikely(error)) {
812 printk(KERN_WARNING "%s: failed to map pages\n",
819 XB_TRACE(bp, "no_daddr", len);
824 __free_page(bp->b_pages[i]);
825 _xfs_buf_free_pages(bp);
827 xfs_buf_deallocate(bp);
833 * Increment reference count on buffer, to hold the buffer concurrently
834 * with another thread which may release (free) the buffer asynchronously.
835 * Must hold the buffer already to call this function.
841 atomic_inc(&bp->b_hold);
842 XB_TRACE(bp, "hold", 0);
846 * Releases a hold on the specified buffer. If the
847 * the hold count is 1, calls xfs_buf_free.
853 xfs_bufhash_t *hash = bp->b_hash;
855 XB_TRACE(bp, "rele", bp->b_relse);
857 if (unlikely(!hash)) {
858 ASSERT(!bp->b_relse);
859 if (atomic_dec_and_test(&bp->b_hold))
864 ASSERT(atomic_read(&bp->b_hold) > 0);
865 if (atomic_dec_and_lock(&bp->b_hold, &hash->bh_lock)) {
867 atomic_inc(&bp->b_hold);
868 spin_unlock(&hash->bh_lock);
869 (*(bp->b_relse)) (bp);
870 } else if (bp->b_flags & XBF_FS_MANAGED) {
871 spin_unlock(&hash->bh_lock);
873 ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
874 list_del_init(&bp->b_hash_list);
875 spin_unlock(&hash->bh_lock);
883 * Mutual exclusion on buffers. Locking model:
885 * Buffers associated with inodes for which buffer locking
886 * is not enabled are not protected by semaphores, and are
887 * assumed to be exclusively owned by the caller. There is a
888 * spinlock in the buffer, used by the caller when concurrent
889 * access is possible.
893 * Locks a buffer object, if it is not already locked.
894 * Note that this in no way locks the underlying pages, so it is only
895 * useful for synchronizing concurrent use of buffer objects, not for
896 * synchronizing independent access to the underlying pages.
904 locked = down_trylock(&bp->b_sema) == 0;
908 XB_TRACE(bp, "cond_lock", (long)locked);
909 return locked ? 0 : -EBUSY;
912 #if defined(DEBUG) || defined(XFS_BLI_TRACE)
917 return bp->b_sema.count;
922 * Locks a buffer object.
923 * Note that this in no way locks the underlying pages, so it is only
924 * useful for synchronizing concurrent use of buffer objects, not for
925 * synchronizing independent access to the underlying pages.
931 XB_TRACE(bp, "lock", 0);
932 if (atomic_read(&bp->b_io_remaining))
933 blk_run_address_space(bp->b_target->bt_mapping);
936 XB_TRACE(bp, "locked", 0);
940 * Releases the lock on the buffer object.
941 * If the buffer is marked delwri but is not queued, do so before we
942 * unlock the buffer as we need to set flags correctly. We also need to
943 * take a reference for the delwri queue because the unlocker is going to
944 * drop their's and they don't know we just queued it.
950 if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
951 atomic_inc(&bp->b_hold);
952 bp->b_flags |= XBF_ASYNC;
953 xfs_buf_delwri_queue(bp, 0);
958 XB_TRACE(bp, "unlock", 0);
963 * Pinning Buffer Storage in Memory
964 * Ensure that no attempt to force a buffer to disk will succeed.
970 atomic_inc(&bp->b_pin_count);
971 XB_TRACE(bp, "pin", (long)bp->b_pin_count.counter);
978 if (atomic_dec_and_test(&bp->b_pin_count))
979 wake_up_all(&bp->b_waiters);
980 XB_TRACE(bp, "unpin", (long)bp->b_pin_count.counter);
987 return atomic_read(&bp->b_pin_count);
994 DECLARE_WAITQUEUE (wait, current);
996 if (atomic_read(&bp->b_pin_count) == 0)
999 add_wait_queue(&bp->b_waiters, &wait);
1001 set_current_state(TASK_UNINTERRUPTIBLE);
1002 if (atomic_read(&bp->b_pin_count) == 0)
1004 if (atomic_read(&bp->b_io_remaining))
1005 blk_run_address_space(bp->b_target->bt_mapping);
1008 remove_wait_queue(&bp->b_waiters, &wait);
1009 set_current_state(TASK_RUNNING);
1013 * Buffer Utility Routines
1017 xfs_buf_iodone_work(
1018 struct work_struct *work)
1021 container_of(work, xfs_buf_t, b_iodone_work);
1024 * We can get an EOPNOTSUPP to ordered writes. Here we clear the
1025 * ordered flag and reissue them. Because we can't tell the higher
1026 * layers directly that they should not issue ordered I/O anymore, they
1027 * need to check if the _XFS_BARRIER_FAILED flag was set during I/O completion.
1029 if ((bp->b_error == EOPNOTSUPP) &&
1030 (bp->b_flags & (XBF_ORDERED|XBF_ASYNC)) == (XBF_ORDERED|XBF_ASYNC)) {
1031 XB_TRACE(bp, "ordered_retry", bp->b_iodone);
1032 bp->b_flags &= ~XBF_ORDERED;
1033 bp->b_flags |= _XFS_BARRIER_FAILED;
1034 xfs_buf_iorequest(bp);
1035 } else if (bp->b_iodone)
1036 (*(bp->b_iodone))(bp);
1037 else if (bp->b_flags & XBF_ASYNC)
1046 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
1047 if (bp->b_error == 0)
1048 bp->b_flags |= XBF_DONE;
1050 XB_TRACE(bp, "iodone", bp->b_iodone);
1052 if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
1054 INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
1055 queue_work(xfslogd_workqueue, &bp->b_iodone_work);
1057 xfs_buf_iodone_work(&bp->b_iodone_work);
1060 complete(&bp->b_iowait);
1069 ASSERT(error >= 0 && error <= 0xffff);
1070 bp->b_error = (unsigned short)error;
1071 XB_TRACE(bp, "ioerror", (unsigned long)error);
1079 XB_TRACE(bp, "bawrite", 0);
1081 ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
1083 xfs_buf_delwri_dequeue(bp);
1085 bp->b_flags &= ~(XBF_READ | XBF_DELWRI | XBF_READ_AHEAD);
1086 bp->b_flags |= (XBF_WRITE | XBF_ASYNC | _XBF_RUN_QUEUES);
1089 bp->b_strat = xfs_bdstrat_cb;
1090 return xfs_bdstrat_cb(bp);
1098 XB_TRACE(bp, "bdwrite", 0);
1100 bp->b_strat = xfs_bdstrat_cb;
1103 bp->b_flags &= ~XBF_READ;
1104 bp->b_flags |= (XBF_DELWRI | XBF_ASYNC);
1106 xfs_buf_delwri_queue(bp, 1);
1114 if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1115 bp->b_flags &= ~_XBF_PAGE_LOCKED;
1116 xfs_buf_ioend(bp, schedule);
1125 xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
1126 unsigned int blocksize = bp->b_target->bt_bsize;
1127 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1129 if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1133 struct page *page = bvec->bv_page;
1135 ASSERT(!PagePrivate(page));
1136 if (unlikely(bp->b_error)) {
1137 if (bp->b_flags & XBF_READ)
1138 ClearPageUptodate(page);
1139 } else if (blocksize >= PAGE_CACHE_SIZE) {
1140 SetPageUptodate(page);
1141 } else if (!PagePrivate(page) &&
1142 (bp->b_flags & _XBF_PAGE_CACHE)) {
1143 set_page_region(page, bvec->bv_offset, bvec->bv_len);
1146 if (--bvec >= bio->bi_io_vec)
1147 prefetchw(&bvec->bv_page->flags);
1149 if (bp->b_flags & _XBF_PAGE_LOCKED)
1151 } while (bvec >= bio->bi_io_vec);
1153 _xfs_buf_ioend(bp, 1);
1161 int rw, map_i, total_nr_pages, nr_pages;
1163 int offset = bp->b_offset;
1164 int size = bp->b_count_desired;
1165 sector_t sector = bp->b_bn;
1166 unsigned int blocksize = bp->b_target->bt_bsize;
1168 total_nr_pages = bp->b_page_count;
1171 if (bp->b_flags & XBF_ORDERED) {
1172 ASSERT(!(bp->b_flags & XBF_READ));
1174 } else if (bp->b_flags & _XBF_RUN_QUEUES) {
1175 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1176 bp->b_flags &= ~_XBF_RUN_QUEUES;
1177 rw = (bp->b_flags & XBF_WRITE) ? WRITE_SYNC : READ_SYNC;
1179 rw = (bp->b_flags & XBF_WRITE) ? WRITE :
1180 (bp->b_flags & XBF_READ_AHEAD) ? READA : READ;
1183 /* Special code path for reading a sub page size buffer in --
1184 * we populate up the whole page, and hence the other metadata
1185 * in the same page. This optimization is only valid when the
1186 * filesystem block size is not smaller than the page size.
1188 if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
1189 ((bp->b_flags & (XBF_READ|_XBF_PAGE_LOCKED)) ==
1190 (XBF_READ|_XBF_PAGE_LOCKED)) &&
1191 (blocksize >= PAGE_CACHE_SIZE)) {
1192 bio = bio_alloc(GFP_NOIO, 1);
1194 bio->bi_bdev = bp->b_target->bt_bdev;
1195 bio->bi_sector = sector - (offset >> BBSHIFT);
1196 bio->bi_end_io = xfs_buf_bio_end_io;
1197 bio->bi_private = bp;
1199 bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
1202 atomic_inc(&bp->b_io_remaining);
1208 atomic_inc(&bp->b_io_remaining);
1209 nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1210 if (nr_pages > total_nr_pages)
1211 nr_pages = total_nr_pages;
1213 bio = bio_alloc(GFP_NOIO, nr_pages);
1214 bio->bi_bdev = bp->b_target->bt_bdev;
1215 bio->bi_sector = sector;
1216 bio->bi_end_io = xfs_buf_bio_end_io;
1217 bio->bi_private = bp;
1219 for (; size && nr_pages; nr_pages--, map_i++) {
1220 int rbytes, nbytes = PAGE_CACHE_SIZE - offset;
1225 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1226 if (rbytes < nbytes)
1230 sector += nbytes >> BBSHIFT;
1236 if (likely(bio->bi_size)) {
1237 submit_bio(rw, bio);
1242 xfs_buf_ioerror(bp, EIO);
1250 XB_TRACE(bp, "iorequest", 0);
1252 if (bp->b_flags & XBF_DELWRI) {
1253 xfs_buf_delwri_queue(bp, 1);
1257 if (bp->b_flags & XBF_WRITE) {
1258 xfs_buf_wait_unpin(bp);
1263 /* Set the count to 1 initially, this will stop an I/O
1264 * completion callout which happens before we have started
1265 * all the I/O from calling xfs_buf_ioend too early.
1267 atomic_set(&bp->b_io_remaining, 1);
1268 _xfs_buf_ioapply(bp);
1269 _xfs_buf_ioend(bp, 0);
1276 * Waits for I/O to complete on the buffer supplied.
1277 * It returns immediately if no I/O is pending.
1278 * It returns the I/O error code, if any, or 0 if there was no error.
1284 XB_TRACE(bp, "iowait", 0);
1285 if (atomic_read(&bp->b_io_remaining))
1286 blk_run_address_space(bp->b_target->bt_mapping);
1287 wait_for_completion(&bp->b_iowait);
1288 XB_TRACE(bp, "iowaited", (long)bp->b_error);
1299 if (bp->b_flags & XBF_MAPPED)
1300 return XFS_BUF_PTR(bp) + offset;
1302 offset += bp->b_offset;
1303 page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1304 return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
1308 * Move data into or out of a buffer.
1312 xfs_buf_t *bp, /* buffer to process */
1313 size_t boff, /* starting buffer offset */
1314 size_t bsize, /* length to copy */
1315 caddr_t data, /* data address */
1316 xfs_buf_rw_t mode) /* read/write/zero flag */
1318 size_t bend, cpoff, csize;
1321 bend = boff + bsize;
1322 while (boff < bend) {
1323 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1324 cpoff = xfs_buf_poff(boff + bp->b_offset);
1325 csize = min_t(size_t,
1326 PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
1328 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1332 memset(page_address(page) + cpoff, 0, csize);
1335 memcpy(data, page_address(page) + cpoff, csize);
1338 memcpy(page_address(page) + cpoff, data, csize);
1347 * Handling of buffer targets (buftargs).
1351 * Wait for any bufs with callbacks that have been submitted but
1352 * have not yet returned... walk the hash list for the target.
1359 xfs_bufhash_t *hash;
1362 for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1363 hash = &btp->bt_hash[i];
1365 spin_lock(&hash->bh_lock);
1366 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
1367 ASSERT(btp == bp->b_target);
1368 if (!(bp->b_flags & XBF_FS_MANAGED)) {
1369 spin_unlock(&hash->bh_lock);
1371 * Catch superblock reference count leaks
1374 BUG_ON(bp->b_bn == 0);
1379 spin_unlock(&hash->bh_lock);
1384 * Allocate buffer hash table for a given target.
1385 * For devices containing metadata (i.e. not the log/realtime devices)
1386 * we need to allocate a much larger hash table.
1395 btp->bt_hashshift = external ? 3 : 8; /* 8 or 256 buckets */
1396 btp->bt_hashmask = (1 << btp->bt_hashshift) - 1;
1397 btp->bt_hash = kmem_zalloc((1 << btp->bt_hashshift) *
1398 sizeof(xfs_bufhash_t), KM_SLEEP | KM_LARGE);
1399 for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1400 spin_lock_init(&btp->bt_hash[i].bh_lock);
1401 INIT_LIST_HEAD(&btp->bt_hash[i].bh_list);
1409 kmem_free(btp->bt_hash);
1410 btp->bt_hash = NULL;
1414 * buftarg list for delwrite queue processing
1416 static LIST_HEAD(xfs_buftarg_list);
1417 static DEFINE_SPINLOCK(xfs_buftarg_lock);
1420 xfs_register_buftarg(
1423 spin_lock(&xfs_buftarg_lock);
1424 list_add(&btp->bt_list, &xfs_buftarg_list);
1425 spin_unlock(&xfs_buftarg_lock);
1429 xfs_unregister_buftarg(
1432 spin_lock(&xfs_buftarg_lock);
1433 list_del(&btp->bt_list);
1434 spin_unlock(&xfs_buftarg_lock);
1441 xfs_flush_buftarg(btp, 1);
1442 xfs_blkdev_issue_flush(btp);
1443 xfs_free_bufhash(btp);
1444 iput(btp->bt_mapping->host);
1446 /* Unregister the buftarg first so that we don't get a
1447 * wakeup finding a non-existent task
1449 xfs_unregister_buftarg(btp);
1450 kthread_stop(btp->bt_task);
1456 xfs_setsize_buftarg_flags(
1458 unsigned int blocksize,
1459 unsigned int sectorsize,
1462 btp->bt_bsize = blocksize;
1463 btp->bt_sshift = ffs(sectorsize) - 1;
1464 btp->bt_smask = sectorsize - 1;
1466 if (set_blocksize(btp->bt_bdev, sectorsize)) {
1468 "XFS: Cannot set_blocksize to %u on device %s\n",
1469 sectorsize, XFS_BUFTARG_NAME(btp));
1474 (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1476 "XFS: %u byte sectors in use on device %s. "
1477 "This is suboptimal; %u or greater is ideal.\n",
1478 sectorsize, XFS_BUFTARG_NAME(btp),
1479 (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1486 * When allocating the initial buffer target we have not yet
1487 * read in the superblock, so don't know what sized sectors
1488 * are being used is at this early stage. Play safe.
1491 xfs_setsize_buftarg_early(
1493 struct block_device *bdev)
1495 return xfs_setsize_buftarg_flags(btp,
1496 PAGE_CACHE_SIZE, bdev_hardsect_size(bdev), 0);
1500 xfs_setsize_buftarg(
1502 unsigned int blocksize,
1503 unsigned int sectorsize)
1505 return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1509 xfs_mapping_buftarg(
1511 struct block_device *bdev)
1513 struct backing_dev_info *bdi;
1514 struct inode *inode;
1515 struct address_space *mapping;
1516 static const struct address_space_operations mapping_aops = {
1517 .sync_page = block_sync_page,
1518 .migratepage = fail_migrate_page,
1521 inode = new_inode(bdev->bd_inode->i_sb);
1524 "XFS: Cannot allocate mapping inode for device %s\n",
1525 XFS_BUFTARG_NAME(btp));
1528 inode->i_mode = S_IFBLK;
1529 inode->i_bdev = bdev;
1530 inode->i_rdev = bdev->bd_dev;
1531 bdi = blk_get_backing_dev_info(bdev);
1533 bdi = &default_backing_dev_info;
1534 mapping = &inode->i_data;
1535 mapping->a_ops = &mapping_aops;
1536 mapping->backing_dev_info = bdi;
1537 mapping_set_gfp_mask(mapping, GFP_NOFS);
1538 btp->bt_mapping = mapping;
1543 xfs_alloc_delwrite_queue(
1548 INIT_LIST_HEAD(&btp->bt_list);
1549 INIT_LIST_HEAD(&btp->bt_delwrite_queue);
1550 spin_lock_init(&btp->bt_delwrite_lock);
1552 btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd");
1553 if (IS_ERR(btp->bt_task)) {
1554 error = PTR_ERR(btp->bt_task);
1557 xfs_register_buftarg(btp);
1564 struct block_device *bdev,
1569 btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1571 btp->bt_dev = bdev->bd_dev;
1572 btp->bt_bdev = bdev;
1573 if (xfs_setsize_buftarg_early(btp, bdev))
1575 if (xfs_mapping_buftarg(btp, bdev))
1577 if (xfs_alloc_delwrite_queue(btp))
1579 xfs_alloc_bufhash(btp, external);
1589 * Delayed write buffer handling
1592 xfs_buf_delwri_queue(
1596 struct list_head *dwq = &bp->b_target->bt_delwrite_queue;
1597 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
1599 XB_TRACE(bp, "delwri_q", (long)unlock);
1600 ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
1603 /* If already in the queue, dequeue and place at tail */
1604 if (!list_empty(&bp->b_list)) {
1605 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1607 atomic_dec(&bp->b_hold);
1608 list_del(&bp->b_list);
1611 bp->b_flags |= _XBF_DELWRI_Q;
1612 list_add_tail(&bp->b_list, dwq);
1613 bp->b_queuetime = jiffies;
1621 xfs_buf_delwri_dequeue(
1624 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
1628 if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1629 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1630 list_del_init(&bp->b_list);
1633 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1639 XB_TRACE(bp, "delwri_dq", (long)dequeued);
1643 xfs_buf_runall_queues(
1644 struct workqueue_struct *queue)
1646 flush_workqueue(queue);
1656 spin_lock(&xfs_buftarg_lock);
1657 list_for_each_entry(btp, &xfs_buftarg_list, bt_list) {
1658 if (test_bit(XBT_FORCE_SLEEP, &btp->bt_flags))
1660 set_bit(XBT_FORCE_FLUSH, &btp->bt_flags);
1661 wake_up_process(btp->bt_task);
1663 spin_unlock(&xfs_buftarg_lock);
1668 * Move as many buffers as specified to the supplied list
1669 * idicating if we skipped any buffers to prevent deadlocks.
1672 xfs_buf_delwri_split(
1673 xfs_buftarg_t *target,
1674 struct list_head *list,
1678 struct list_head *dwq = &target->bt_delwrite_queue;
1679 spinlock_t *dwlk = &target->bt_delwrite_lock;
1683 force = test_and_clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1684 INIT_LIST_HEAD(list);
1686 list_for_each_entry_safe(bp, n, dwq, b_list) {
1687 XB_TRACE(bp, "walkq1", (long)xfs_buf_ispin(bp));
1688 ASSERT(bp->b_flags & XBF_DELWRI);
1690 if (!xfs_buf_ispin(bp) && !xfs_buf_cond_lock(bp)) {
1692 time_before(jiffies, bp->b_queuetime + age)) {
1697 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q|
1699 bp->b_flags |= XBF_WRITE;
1700 list_move_tail(&bp->b_list, list);
1714 struct list_head tmp;
1715 xfs_buftarg_t *target = (xfs_buftarg_t *)data;
1719 current->flags |= PF_MEMALLOC;
1724 if (unlikely(freezing(current))) {
1725 set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1728 clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1731 schedule_timeout_interruptible(
1732 xfs_buf_timer_centisecs * msecs_to_jiffies(10));
1734 xfs_buf_delwri_split(target, &tmp,
1735 xfs_buf_age_centisecs * msecs_to_jiffies(10));
1738 while (!list_empty(&tmp)) {
1739 bp = list_entry(tmp.next, xfs_buf_t, b_list);
1740 ASSERT(target == bp->b_target);
1742 list_del_init(&bp->b_list);
1743 xfs_buf_iostrategy(bp);
1747 if (as_list_len > 0)
1750 blk_run_address_space(target->bt_mapping);
1752 } while (!kthread_should_stop());
1758 * Go through all incore buffers, and release buffers if they belong to
1759 * the given device. This is used in filesystem error handling to
1760 * preserve the consistency of its metadata.
1764 xfs_buftarg_t *target,
1767 struct list_head tmp;
1771 xfs_buf_runall_queues(xfsdatad_workqueue);
1772 xfs_buf_runall_queues(xfslogd_workqueue);
1774 set_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1775 pincount = xfs_buf_delwri_split(target, &tmp, 0);
1778 * Dropped the delayed write list lock, now walk the temporary list
1780 list_for_each_entry_safe(bp, n, &tmp, b_list) {
1781 ASSERT(target == bp->b_target);
1783 bp->b_flags &= ~XBF_ASYNC;
1785 list_del_init(&bp->b_list);
1787 xfs_buf_iostrategy(bp);
1791 blk_run_address_space(target->bt_mapping);
1794 * Remaining list items must be flushed before returning
1796 while (!list_empty(&tmp)) {
1797 bp = list_entry(tmp.next, xfs_buf_t, b_list);
1799 list_del_init(&bp->b_list);
1810 #ifdef XFS_BUF_TRACE
1811 xfs_buf_trace_buf = ktrace_alloc(XFS_BUF_TRACE_SIZE, KM_NOFS);
1814 xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1815 KM_ZONE_HWALIGN, NULL);
1817 goto out_free_trace_buf;
1819 xfslogd_workqueue = create_workqueue("xfslogd");
1820 if (!xfslogd_workqueue)
1821 goto out_free_buf_zone;
1823 xfsdatad_workqueue = create_workqueue("xfsdatad");
1824 if (!xfsdatad_workqueue)
1825 goto out_destroy_xfslogd_workqueue;
1827 register_shrinker(&xfs_buf_shake);
1830 out_destroy_xfslogd_workqueue:
1831 destroy_workqueue(xfslogd_workqueue);
1833 kmem_zone_destroy(xfs_buf_zone);
1835 #ifdef XFS_BUF_TRACE
1836 ktrace_free(xfs_buf_trace_buf);
1842 xfs_buf_terminate(void)
1844 unregister_shrinker(&xfs_buf_shake);
1845 destroy_workqueue(xfsdatad_workqueue);
1846 destroy_workqueue(xfslogd_workqueue);
1847 kmem_zone_destroy(xfs_buf_zone);
1848 #ifdef XFS_BUF_TRACE
1849 ktrace_free(xfs_buf_trace_buf);
1853 #ifdef CONFIG_KDB_MODULES
1855 xfs_get_buftarg_list(void)
1857 return &xfs_buftarg_list;