[linux-2.6-block.git] / mm / backing-dev.c


#include <linux/wait.h>
#include <linux/backing-dev.h>
#include <linux/kthread.h>
#include <linux/freezer.h>
#include <linux/fs.h>
#include <linux/pagemap.h>
#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/module.h>
#include <linux/writeback.h>
#include <linux/device.h>
#include <trace/events/writeback.h>

static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0);

struct backing_dev_info noop_backing_dev_info = {
	.name		= "noop",
	.capabilities	= BDI_CAP_NO_ACCT_AND_WRITEBACK,
};

static struct class *bdi_class;

/*
 * bdi_lock protects updates to bdi_list. bdi_list has RCU reader side
 * locking.
 */
DEFINE_SPINLOCK(bdi_lock);
LIST_HEAD(bdi_list);

/* bdi_wq serves all asynchronous writeback tasks */
struct workqueue_struct *bdi_wq;

#ifdef CONFIG_DEBUG_FS
#include <linux/debugfs.h>
#include <linux/seq_file.h>

static struct dentry *bdi_debug_root;

static void bdi_debug_init(void)
{
	bdi_debug_root = debugfs_create_dir("bdi", NULL);
}

static int bdi_debug_stats_show(struct seq_file *m, void *v)
{
	struct backing_dev_info *bdi = m->private;
	struct bdi_writeback *wb = &bdi->wb;
	unsigned long background_thresh;
	unsigned long dirty_thresh;
	unsigned long bdi_thresh;
	unsigned long nr_dirty, nr_io, nr_more_io, nr_dirty_time;
	struct inode *inode;

	nr_dirty = nr_io = nr_more_io = nr_dirty_time = 0;
	spin_lock(&wb->list_lock);
	list_for_each_entry(inode, &wb->b_dirty, i_wb_list)
		nr_dirty++;
	list_for_each_entry(inode, &wb->b_io, i_wb_list)
		nr_io++;
	list_for_each_entry(inode, &wb->b_more_io, i_wb_list)
		nr_more_io++;
	list_for_each_entry(inode, &wb->b_dirty_time, i_wb_list)
		if (inode->i_state & I_DIRTY_TIME)
			nr_dirty_time++;
	spin_unlock(&wb->list_lock);

	global_dirty_limits(&background_thresh, &dirty_thresh);
	bdi_thresh = wb_dirty_limit(wb, dirty_thresh);

#define K(x) ((x) << (PAGE_SHIFT - 10))
	seq_printf(m,
		   "BdiWriteback:       %10lu kB\n"
		   "BdiReclaimable:     %10lu kB\n"
		   "BdiDirtyThresh:     %10lu kB\n"
		   "DirtyThresh:        %10lu kB\n"
		   "BackgroundThresh:   %10lu kB\n"
		   "BdiDirtied:         %10lu kB\n"
		   "BdiWritten:         %10lu kB\n"
		   "BdiWriteBandwidth:  %10lu kBps\n"
		   "b_dirty:            %10lu\n"
		   "b_io:               %10lu\n"
		   "b_more_io:          %10lu\n"
		   "b_dirty_time:       %10lu\n"
		   "bdi_list:           %10u\n"
		   "state:              %10lx\n",
		   (unsigned long) K(wb_stat(wb, WB_WRITEBACK)),
		   (unsigned long) K(wb_stat(wb, WB_RECLAIMABLE)),
		   K(bdi_thresh),
		   K(dirty_thresh),
		   K(background_thresh),
		   (unsigned long) K(wb_stat(wb, WB_DIRTIED)),
		   (unsigned long) K(wb_stat(wb, WB_WRITTEN)),
		   (unsigned long) K(wb->write_bandwidth),
		   nr_dirty,
		   nr_io,
		   nr_more_io,
		   nr_dirty_time,
		   !list_empty(&bdi->bdi_list), bdi->wb.state);
#undef K

	return 0;
}

static int bdi_debug_stats_open(struct inode *inode, struct file *file)
{
	return single_open(file, bdi_debug_stats_show, inode->i_private);
}

static const struct file_operations bdi_debug_stats_fops = {
	.open		= bdi_debug_stats_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
};

static void bdi_debug_register(struct backing_dev_info *bdi, const char *name)
{
	bdi->debug_dir = debugfs_create_dir(name, bdi_debug_root);
	bdi->debug_stats = debugfs_create_file("stats", 0444, bdi->debug_dir,
					       bdi, &bdi_debug_stats_fops);
}

static void bdi_debug_unregister(struct backing_dev_info *bdi)
{
	debugfs_remove(bdi->debug_stats);
	debugfs_remove(bdi->debug_dir);
}
#else
static inline void bdi_debug_init(void)
{
}
static inline void bdi_debug_register(struct backing_dev_info *bdi,
				      const char *name)
{
}
static inline void bdi_debug_unregister(struct backing_dev_info *bdi)
{
}
#endif

static ssize_t read_ahead_kb_store(struct device *dev,
				  struct device_attribute *attr,
				  const char *buf, size_t count)
{
	struct backing_dev_info *bdi = dev_get_drvdata(dev);
	unsigned long read_ahead_kb;
	ssize_t ret;

	ret = kstrtoul(buf, 10, &read_ahead_kb);
	if (ret < 0)
		return ret;

	bdi->ra_pages = read_ahead_kb >> (PAGE_SHIFT - 10);

	return count;
}

#define K(pages) ((pages) << (PAGE_SHIFT - 10))

#define BDI_SHOW(name, expr)						\
static ssize_t name##_show(struct device *dev,				\
			   struct device_attribute *attr, char *page)	\
{									\
	struct backing_dev_info *bdi = dev_get_drvdata(dev);		\
									\
	return snprintf(page, PAGE_SIZE-1, "%lld\n", (long long)expr);	\
}									\
static DEVICE_ATTR_RW(name);

BDI_SHOW(read_ahead_kb, K(bdi->ra_pages))

static ssize_t min_ratio_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t count)
{
	struct backing_dev_info *bdi = dev_get_drvdata(dev);
	unsigned int ratio;
	ssize_t ret;

	ret = kstrtouint(buf, 10, &ratio);
	if (ret < 0)
		return ret;

	ret = bdi_set_min_ratio(bdi, ratio);
	if (!ret)
		ret = count;

	return ret;
}
BDI_SHOW(min_ratio, bdi->min_ratio)

static ssize_t max_ratio_store(struct device *dev,
		struct device_attribute *attr, const char *buf, size_t count)
{
	struct backing_dev_info *bdi = dev_get_drvdata(dev);
	unsigned int ratio;
	ssize_t ret;

	ret = kstrtouint(buf, 10, &ratio);
	if (ret < 0)
		return ret;

	ret = bdi_set_max_ratio(bdi, ratio);
	if (!ret)
		ret = count;

	return ret;
}
BDI_SHOW(max_ratio, bdi->max_ratio)

static ssize_t stable_pages_required_show(struct device *dev,
					  struct device_attribute *attr,
					  char *page)
{
	struct backing_dev_info *bdi = dev_get_drvdata(dev);

	return snprintf(page, PAGE_SIZE-1, "%d\n",
			bdi_cap_stable_pages_required(bdi) ? 1 : 0);
}
static DEVICE_ATTR_RO(stable_pages_required);

static struct attribute *bdi_dev_attrs[] = {
	&dev_attr_read_ahead_kb.attr,
	&dev_attr_min_ratio.attr,
	&dev_attr_max_ratio.attr,
	&dev_attr_stable_pages_required.attr,
	NULL,
};
ATTRIBUTE_GROUPS(bdi_dev);

static __init int bdi_class_init(void)
{
	bdi_class = class_create(THIS_MODULE, "bdi");
	if (IS_ERR(bdi_class))
		return PTR_ERR(bdi_class);

	bdi_class->dev_groups = bdi_dev_groups;
	bdi_debug_init();
	return 0;
}
postcore_initcall(bdi_class_init);

static int __init default_bdi_init(void)
{
	int err;

	bdi_wq = alloc_workqueue("writeback", WQ_MEM_RECLAIM | WQ_FREEZABLE |
					      WQ_UNBOUND | WQ_SYSFS, 0);
	if (!bdi_wq)
		return -ENOMEM;

	err = bdi_init(&noop_backing_dev_info);

	return err;
}
subsys_initcall(default_bdi_init);

int bdi_has_dirty_io(struct backing_dev_info *bdi)
{
	return wb_has_dirty_io(&bdi->wb);
}

/*
 * This function is used when the first inode for this wb is marked dirty. It
 * wakes-up the corresponding bdi thread which should then take care of the
 * periodic background write-out of dirty inodes. Since the write-out would
 * starts only 'dirty_writeback_interval' centisecs from now anyway, we just
 * set up a timer which wakes the bdi thread up later.
 *
 * Note, we wouldn't bother setting up the timer, but this function is on the
 * fast-path (used by '__mark_inode_dirty()'), so we save few context switches
 * by delaying the wake-up.
 *
 * We have to be careful not to postpone flush work if it is scheduled for
 * earlier. Thus we use queue_delayed_work().
 */
void wb_wakeup_delayed(struct bdi_writeback *wb)
{
	unsigned long timeout;

	timeout = msecs_to_jiffies(dirty_writeback_interval * 10);
	spin_lock_bh(&wb->work_lock);
	if (test_bit(WB_registered, &wb->state))
		queue_delayed_work(bdi_wq, &wb->dwork, timeout);
	spin_unlock_bh(&wb->work_lock);
}

/*
 * Initial write bandwidth: 100 MB/s
 */
#define INIT_BW		(100 << (20 - PAGE_SHIFT))

static int wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi)
{
	int i, err;

	memset(wb, 0, sizeof(*wb));

	wb->bdi = bdi;
	wb->last_old_flush = jiffies;
	INIT_LIST_HEAD(&wb->b_dirty);
	INIT_LIST_HEAD(&wb->b_io);
	INIT_LIST_HEAD(&wb->b_more_io);
	INIT_LIST_HEAD(&wb->b_dirty_time);
	spin_lock_init(&wb->list_lock);

	wb->bw_time_stamp = jiffies;
	wb->balanced_dirty_ratelimit = INIT_BW;
	wb->dirty_ratelimit = INIT_BW;
	wb->write_bandwidth = INIT_BW;
	wb->avg_write_bandwidth = INIT_BW;

	spin_lock_init(&wb->work_lock);
	INIT_LIST_HEAD(&wb->work_list);
	INIT_DELAYED_WORK(&wb->dwork, wb_workfn);

	err = fprop_local_init_percpu(&wb->completions, GFP_KERNEL);
	if (err)
		return err;

	for (i = 0; i < NR_WB_STAT_ITEMS; i++) {
		err = percpu_counter_init(&wb->stat[i], 0, GFP_KERNEL);
		if (err) {
			while (--i)
				percpu_counter_destroy(&wb->stat[i]);
			fprop_local_destroy_percpu(&wb->completions);
			return err;
		}
	}

	return 0;
}

/*
 * Remove bdi from the global list and shutdown any threads we have running
 */
static void wb_shutdown(struct bdi_writeback *wb)
{
	/* Make sure nobody queues further work */
	spin_lock_bh(&wb->work_lock);
	if (!test_and_clear_bit(WB_registered, &wb->state)) {
		spin_unlock_bh(&wb->work_lock);
		return;
	}
	spin_unlock_bh(&wb->work_lock);

	/*
	 * Drain work list and shutdown the delayed_work.  !WB_registered
	 * tells wb_workfn() that @wb is dying and its work_list needs to
	 * be drained no matter what.
	 */
	mod_delayed_work(bdi_wq, &wb->dwork, 0);
	flush_delayed_work(&wb->dwork);
	WARN_ON(!list_empty(&wb->work_list));
}

static void wb_exit(struct bdi_writeback *wb)
{
	int i;

	WARN_ON(delayed_work_pending(&wb->dwork));

	for (i = 0; i < NR_WB_STAT_ITEMS; i++)
		percpu_counter_destroy(&wb->stat[i]);

	fprop_local_destroy_percpu(&wb->completions);
}

int bdi_init(struct backing_dev_info *bdi)
{
	int err;

	bdi->dev = NULL;

	bdi->min_ratio = 0;
	bdi->max_ratio = 100;
	bdi->max_prop_frac = FPROP_FRAC_BASE;
	INIT_LIST_HEAD(&bdi->bdi_list);

	err = wb_init(&bdi->wb, bdi);
	if (err)
		return err;

	return 0;
}
EXPORT_SYMBOL(bdi_init);

int bdi_register(struct backing_dev_info *bdi, struct device *parent,
		const char *fmt, ...)
{
	va_list args;
	struct device *dev;

	if (bdi->dev)	/* The driver needs to use separate queues per device */
		return 0;

	va_start(args, fmt);
	dev = device_create_vargs(bdi_class, parent, MKDEV(0, 0), bdi, fmt, args);
	va_end(args);
	if (IS_ERR(dev))
		return PTR_ERR(dev);

	bdi->dev = dev;

	bdi_debug_register(bdi, dev_name(dev));
	set_bit(WB_registered, &bdi->wb.state);

	spin_lock_bh(&bdi_lock);
	list_add_tail_rcu(&bdi->bdi_list, &bdi_list);
	spin_unlock_bh(&bdi_lock);

	trace_writeback_bdi_register(bdi);
	return 0;
}
EXPORT_SYMBOL(bdi_register);

int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev)
{
	return bdi_register(bdi, NULL, "%u:%u", MAJOR(dev), MINOR(dev));
}
EXPORT_SYMBOL(bdi_register_dev);

/*
 * Remove bdi from bdi_list, and ensure that it is no longer visible
 */
static void bdi_remove_from_list(struct backing_dev_info *bdi)
{
	spin_lock_bh(&bdi_lock);
	list_del_rcu(&bdi->bdi_list);
	spin_unlock_bh(&bdi_lock);

	synchronize_rcu_expedited();
}

/*
 * Called when the device behind @bdi has been removed or ejected.
 *
 * We can't really do much here except for reducing the dirty ratio at
 * the moment.  In the future we should be able to set a flag so that
 * the filesystem can handle errors at mark_inode_dirty time instead
 * of only at writeback time.
 */
void bdi_unregister(struct backing_dev_info *bdi)
{
	if (WARN_ON_ONCE(!bdi->dev))
		return;

	bdi_set_min_ratio(bdi, 0);
}
EXPORT_SYMBOL(bdi_unregister);

void bdi_destroy(struct backing_dev_info *bdi)
{
	/* make sure nobody finds us on the bdi_list anymore */
	bdi_remove_from_list(bdi);
	wb_shutdown(&bdi->wb);

	if (bdi->dev) {
		bdi_debug_unregister(bdi);
		device_unregister(bdi->dev);
		bdi->dev = NULL;
	}

	wb_exit(&bdi->wb);
}
EXPORT_SYMBOL(bdi_destroy);

/*
 * For use from filesystems to quickly init and register a bdi associated
 * with dirty writeback
 */
int bdi_setup_and_register(struct backing_dev_info *bdi, char *name)
{
	int err;

	bdi->name = name;
	bdi->capabilities = 0;
	err = bdi_init(bdi);
	if (err)
		return err;

	err = bdi_register(bdi, NULL, "%.28s-%ld", name,
			   atomic_long_inc_return(&bdi_seq));
	if (err) {
		bdi_destroy(bdi);
		return err;
	}

	return 0;
}
EXPORT_SYMBOL(bdi_setup_and_register);

static wait_queue_head_t congestion_wqh[2] = {
		__WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[0]),
		__WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1])
	};
static atomic_t nr_bdi_congested[2];

void clear_bdi_congested(struct backing_dev_info *bdi, int sync)
{
	enum wb_state bit;
	wait_queue_head_t *wqh = &congestion_wqh[sync];

	bit = sync ? WB_sync_congested : WB_async_congested;
	if (test_and_clear_bit(bit, &bdi->wb.state))
		atomic_dec(&nr_bdi_congested[sync]);
	smp_mb__after_atomic();
	if (waitqueue_active(wqh))
		wake_up(wqh);
}
EXPORT_SYMBOL(clear_bdi_congested);

void set_bdi_congested(struct backing_dev_info *bdi, int sync)
{
	enum wb_state bit;

	bit = sync ? WB_sync_congested : WB_async_congested;
	if (!test_and_set_bit(bit, &bdi->wb.state))
		atomic_inc(&nr_bdi_congested[sync]);
}
EXPORT_SYMBOL(set_bdi_congested);

/**
 * congestion_wait - wait for a backing_dev to become uncongested
 * @sync: SYNC or ASYNC IO
 * @timeout: timeout in jiffies
 *
 * Waits for up to @timeout jiffies for a backing_dev (any backing_dev) to exit
 * write congestion.  If no backing_devs are congested then just wait for the
 * next write to be completed.
 */
long congestion_wait(int sync, long timeout)
{
	long ret;
	unsigned long start = jiffies;
	DEFINE_WAIT(wait);
	wait_queue_head_t *wqh = &congestion_wqh[sync];

	prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
	ret = io_schedule_timeout(timeout);
	finish_wait(wqh, &wait);

	trace_writeback_congestion_wait(jiffies_to_usecs(timeout),
					jiffies_to_usecs(jiffies - start));

	return ret;
}
EXPORT_SYMBOL(congestion_wait);

/**
 * wait_iff_congested - Conditionally wait for a backing_dev to become uncongested or a zone to complete writes
 * @zone: A zone to check if it is heavily congested
 * @sync: SYNC or ASYNC IO
 * @timeout: timeout in jiffies
 *
 * In the event of a congested backing_dev (any backing_dev) and the given
 * @zone has experienced recent congestion, this waits for up to @timeout
 * jiffies for either a BDI to exit congestion of the given @sync queue
 * or a write to complete.
 *
 * In the absence of zone congestion, cond_resched() is called to yield
 * the processor if necessary but otherwise does not sleep.
 *
 * The return value is 0 if the sleep is for the full timeout. Otherwise,
 * it is the number of jiffies that were still remaining when the function
 * returned. return_value == timeout implies the function did not sleep.
 */
long wait_iff_congested(struct zone *zone, int sync, long timeout)
{
	long ret;
	unsigned long start = jiffies;
	DEFINE_WAIT(wait);
	wait_queue_head_t *wqh = &congestion_wqh[sync];

	/*
	 * If there is no congestion, or heavy congestion is not being
	 * encountered in the current zone, yield if necessary instead
	 * of sleeping on the congestion queue
	 */
	if (atomic_read(&nr_bdi_congested[sync]) == 0 ||
	    !test_bit(ZONE_CONGESTED, &zone->flags)) {
		cond_resched();

		/* In case we scheduled, work out time remaining */
		ret = timeout - (jiffies - start);
		if (ret < 0)
			ret = 0;

		goto out;
	}

	/* Sleep until uncongested or a write happens */
	prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
	ret = io_schedule_timeout(timeout);
	finish_wait(wqh, &wait);

out:
	trace_writeback_wait_iff_congested(jiffies_to_usecs(timeout),
					jiffies_to_usecs(jiffies - start));

	return ret;
}
EXPORT_SYMBOL(wait_iff_congested);

int pdflush_proc_obsolete(struct ctl_table *table, int write,
			void __user *buffer, size_t *lenp, loff_t *ppos)
{
	char kbuf[] = "0\n";

	if (*ppos || *lenp < sizeof(kbuf)) {
		*lenp = 0;
		return 0;
	}

	if (copy_to_user(buffer, kbuf, sizeof(kbuf)))
		return -EFAULT;
	printk_once(KERN_WARNING "%s exported in /proc is scheduled for removal\n",
			table->procname);

	*lenp = 2;
	*ppos += *lenp;
	return 2;
}
Commit	Line	Data
3fcfab16 AM	1
	2	#include <linux/wait.h>
	3	#include <linux/backing-dev.h>
03ba3782 JA	4	#include <linux/kthread.h>
03ba3782 JA	5	#include <linux/freezer.h>
3fcfab16	6	#include <linux/fs.h>
26160158	7	#include <linux/pagemap.h>
03ba3782	8	#include <linux/mm.h>
3fcfab16 AM	9	#include <linux/sched.h>
3fcfab16 AM	10	#include <linux/module.h>
cf0ca9fe PZ	11	#include <linux/writeback.h>
cf0ca9fe PZ	12	#include <linux/device.h>
455b2864	13	#include <trace/events/writeback.h>
cf0ca9fe	14
c3c53206 JA	15	static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0);
c3c53206 JA	16
5129a469 JE	17	struct backing_dev_info noop_backing_dev_info = {
5129a469 JE	18	.name = "noop",
976e48f8	19	.capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK,
5129a469	20	};
5129a469	21
cf0ca9fe	22	static struct class *bdi_class;
cfc4ba53 JA	23
cfc4ba53 JA	24	/*
181387da	25	* bdi_lock protects updates to bdi_list. bdi_list has RCU reader side
cfc4ba53 JA	26	* locking.
cfc4ba53 JA	27	*/
03ba3782	28	DEFINE_SPINLOCK(bdi_lock);
66f3b8e2	29	LIST_HEAD(bdi_list);
03ba3782	30
839a8e86 TH	31	/* bdi_wq serves all asynchronous writeback tasks */
	32	struct workqueue_struct *bdi_wq;
	33
76f1418b MS	34	#ifdef CONFIG_DEBUG_FS
	35	#include <linux/debugfs.h>
	36	#include <linux/seq_file.h>
	37
	38	static struct dentry *bdi_debug_root;
	39
	40	static void bdi_debug_init(void)
	41	{
	42	bdi_debug_root = debugfs_create_dir("bdi", NULL);
	43	}
	44
	45	static int bdi_debug_stats_show(struct seq_file m, void v)
	46	{
	47	struct backing_dev_info *bdi = m->private;
c1955ce3	48	struct bdi_writeback *wb = &bdi->wb;
364aeb28 DR	49	unsigned long background_thresh;
	50	unsigned long dirty_thresh;
	51	unsigned long bdi_thresh;
0ae45f63	52	unsigned long nr_dirty, nr_io, nr_more_io, nr_dirty_time;
f09b00d3 JA	53	struct inode *inode;
f09b00d3 JA	54
0ae45f63	55	nr_dirty = nr_io = nr_more_io = nr_dirty_time = 0;
f758eeab	56	spin_lock(&wb->list_lock);
7ccf19a8	57	list_for_each_entry(inode, &wb->b_dirty, i_wb_list)
c1955ce3	58	nr_dirty++;
7ccf19a8	59	list_for_each_entry(inode, &wb->b_io, i_wb_list)
c1955ce3	60	nr_io++;
7ccf19a8	61	list_for_each_entry(inode, &wb->b_more_io, i_wb_list)
c1955ce3	62	nr_more_io++;
0ae45f63 TT	63	list_for_each_entry(inode, &wb->b_dirty_time, i_wb_list)
	64	if (inode->i_state & I_DIRTY_TIME)
	65	nr_dirty_time++;
f758eeab	66	spin_unlock(&wb->list_lock);
76f1418b	67
16c4042f	68	global_dirty_limits(&background_thresh, &dirty_thresh);
a88a341a	69	bdi_thresh = wb_dirty_limit(wb, dirty_thresh);
76f1418b MS	70
	71	#define K(x) ((x) << (PAGE_SHIFT - 10))
	72	seq_printf(m,
00821b00 WF	73	"BdiWriteback: %10lu kB\n"
	74	"BdiReclaimable: %10lu kB\n"
	75	"BdiDirtyThresh: %10lu kB\n"
	76	"DirtyThresh: %10lu kB\n"
	77	"BackgroundThresh: %10lu kB\n"
c8e28ce0	78	"BdiDirtied: %10lu kB\n"
00821b00 WF	79	"BdiWritten: %10lu kB\n"
	80	"BdiWriteBandwidth: %10lu kBps\n"
	81	"b_dirty: %10lu\n"
	82	"b_io: %10lu\n"
	83	"b_more_io: %10lu\n"
0ae45f63	84	"b_dirty_time: %10lu\n"
00821b00 WF	85	"bdi_list: %10u\n"
00821b00 WF	86	"state: %10lx\n",
93f78d88 TH	87	(unsigned long) K(wb_stat(wb, WB_WRITEBACK)),
93f78d88 TH	88	(unsigned long) K(wb_stat(wb, WB_RECLAIMABLE)),
f7d2b1ec JK	89	K(bdi_thresh),
	90	K(dirty_thresh),
	91	K(background_thresh),
93f78d88 TH	92	(unsigned long) K(wb_stat(wb, WB_DIRTIED)),
93f78d88 TH	93	(unsigned long) K(wb_stat(wb, WB_WRITTEN)),
a88a341a	94	(unsigned long) K(wb->write_bandwidth),
f7d2b1ec JK	95	nr_dirty,
	96	nr_io,
	97	nr_more_io,
0ae45f63	98	nr_dirty_time,
4452226e	99	!list_empty(&bdi->bdi_list), bdi->wb.state);
76f1418b MS	100	#undef K
	101
	102	return 0;
	103	}
	104
	105	static int bdi_debug_stats_open(struct inode inode, struct file file)
	106	{
	107	return single_open(file, bdi_debug_stats_show, inode->i_private);
	108	}
	109
	110	static const struct file_operations bdi_debug_stats_fops = {
	111	.open = bdi_debug_stats_open,
	112	.read = seq_read,
	113	.llseek = seq_lseek,
	114	.release = single_release,
	115	};
	116
	117	static void bdi_debug_register(struct backing_dev_info bdi, const char name)
	118	{
	119	bdi->debug_dir = debugfs_create_dir(name, bdi_debug_root);
	120	bdi->debug_stats = debugfs_create_file("stats", 0444, bdi->debug_dir,
	121	bdi, &bdi_debug_stats_fops);
	122	}
	123
	124	static void bdi_debug_unregister(struct backing_dev_info *bdi)
	125	{
	126	debugfs_remove(bdi->debug_stats);
	127	debugfs_remove(bdi->debug_dir);
	128	}
	129	#else
	130	static inline void bdi_debug_init(void)
	131	{
	132	}
	133	static inline void bdi_debug_register(struct backing_dev_info *bdi,
	134	const char *name)
	135	{
	136	}
	137	static inline void bdi_debug_unregister(struct backing_dev_info *bdi)
	138	{
	139	}
	140	#endif
	141
cf0ca9fe PZ	142	static ssize_t read_ahead_kb_store(struct device *dev,
	143	struct device_attribute *attr,
	144	const char *buf, size_t count)
	145	{
	146	struct backing_dev_info *bdi = dev_get_drvdata(dev);
cf0ca9fe	147	unsigned long read_ahead_kb;
7034ed13	148	ssize_t ret;
cf0ca9fe	149
7034ed13 NJ	150	ret = kstrtoul(buf, 10, &read_ahead_kb);
	151	if (ret < 0)
	152	return ret;
	153
	154	bdi->ra_pages = read_ahead_kb >> (PAGE_SHIFT - 10);
	155
	156	return count;
cf0ca9fe PZ	157	}
	158
	159	#define K(pages) ((pages) << (PAGE_SHIFT - 10))
	160
	161	#define BDI_SHOW(name, expr) \
	162	static ssize_t name##_show(struct device *dev, \
	163	struct device_attribute attr, char page) \
	164	{ \
	165	struct backing_dev_info *bdi = dev_get_drvdata(dev); \
	166	\
	167	return snprintf(page, PAGE_SIZE-1, "%lld\n", (long long)expr); \
d9e1241e GKH	168	} \
d9e1241e GKH	169	static DEVICE_ATTR_RW(name);
cf0ca9fe PZ	170
	171	BDI_SHOW(read_ahead_kb, K(bdi->ra_pages))
	172
189d3c4a PZ	173	static ssize_t min_ratio_store(struct device *dev,
	174	struct device_attribute attr, const char buf, size_t count)
	175	{
	176	struct backing_dev_info *bdi = dev_get_drvdata(dev);
189d3c4a	177	unsigned int ratio;
7034ed13 NJ	178	ssize_t ret;
	179
	180	ret = kstrtouint(buf, 10, &ratio);
	181	if (ret < 0)
	182	return ret;
	183
	184	ret = bdi_set_min_ratio(bdi, ratio);
	185	if (!ret)
	186	ret = count;
189d3c4a	187
189d3c4a PZ	188	return ret;
	189	}
	190	BDI_SHOW(min_ratio, bdi->min_ratio)
	191
a42dde04 PZ	192	static ssize_t max_ratio_store(struct device *dev,
	193	struct device_attribute attr, const char buf, size_t count)
	194	{
	195	struct backing_dev_info *bdi = dev_get_drvdata(dev);
a42dde04	196	unsigned int ratio;
7034ed13 NJ	197	ssize_t ret;
	198
	199	ret = kstrtouint(buf, 10, &ratio);
	200	if (ret < 0)
	201	return ret;
	202
	203	ret = bdi_set_max_ratio(bdi, ratio);
	204	if (!ret)
	205	ret = count;
a42dde04	206
a42dde04 PZ	207	return ret;
	208	}
	209	BDI_SHOW(max_ratio, bdi->max_ratio)
	210
7d311cda DW	211	static ssize_t stable_pages_required_show(struct device *dev,
	212	struct device_attribute *attr,
	213	char *page)
	214	{
	215	struct backing_dev_info *bdi = dev_get_drvdata(dev);
	216
	217	return snprintf(page, PAGE_SIZE-1, "%d\n",
	218	bdi_cap_stable_pages_required(bdi) ? 1 : 0);
	219	}
d9e1241e GKH	220	static DEVICE_ATTR_RO(stable_pages_required);
	221
	222	static struct attribute *bdi_dev_attrs[] = {
	223	&dev_attr_read_ahead_kb.attr,
	224	&dev_attr_min_ratio.attr,
	225	&dev_attr_max_ratio.attr,
	226	&dev_attr_stable_pages_required.attr,
	227	NULL,
cf0ca9fe	228	};
d9e1241e	229	ATTRIBUTE_GROUPS(bdi_dev);
cf0ca9fe PZ	230
	231	static __init int bdi_class_init(void)
	232	{
	233	bdi_class = class_create(THIS_MODULE, "bdi");
14421453 AB	234	if (IS_ERR(bdi_class))
	235	return PTR_ERR(bdi_class);
	236
d9e1241e	237	bdi_class->dev_groups = bdi_dev_groups;
76f1418b	238	bdi_debug_init();
cf0ca9fe PZ	239	return 0;
cf0ca9fe PZ	240	}
76f1418b	241	postcore_initcall(bdi_class_init);
cf0ca9fe	242
26160158 JA	243	static int __init default_bdi_init(void)
	244	{
	245	int err;
	246
839a8e86	247	bdi_wq = alloc_workqueue("writeback", WQ_MEM_RECLAIM \| WQ_FREEZABLE \|
b5c872dd	248	WQ_UNBOUND \| WQ_SYSFS, 0);
839a8e86 TH	249	if (!bdi_wq)
	250	return -ENOMEM;
	251
976e48f8	252	err = bdi_init(&noop_backing_dev_info);
26160158 JA	253
	254	return err;
	255	}
	256	subsys_initcall(default_bdi_init);
	257
03ba3782 JA	258	int bdi_has_dirty_io(struct backing_dev_info *bdi)
	259	{
	260	return wb_has_dirty_io(&bdi->wb);
	261	}
	262
6467716a	263	/*
f0054bb1	264	* This function is used when the first inode for this wb is marked dirty. It
6467716a AB	265	* wakes-up the corresponding bdi thread which should then take care of the
	266	* periodic background write-out of dirty inodes. Since the write-out would
	267	* starts only 'dirty_writeback_interval' centisecs from now anyway, we just
	268	* set up a timer which wakes the bdi thread up later.
	269	*
	270	* Note, we wouldn't bother setting up the timer, but this function is on the
	271	* fast-path (used by '__mark_inode_dirty()'), so we save few context switches
	272	* by delaying the wake-up.
6ca738d6 DB	273	*
	274	* We have to be careful not to postpone flush work if it is scheduled for
	275	* earlier. Thus we use queue_delayed_work().
6467716a	276	*/
f0054bb1	277	void wb_wakeup_delayed(struct bdi_writeback *wb)
6467716a AB	278	{
	279	unsigned long timeout;
	280
	281	timeout = msecs_to_jiffies(dirty_writeback_interval * 10);
f0054bb1 TH	282	spin_lock_bh(&wb->work_lock);
	283	if (test_bit(WB_registered, &wb->state))
	284	queue_delayed_work(bdi_wq, &wb->dwork, timeout);
	285	spin_unlock_bh(&wb->work_lock);
03ba3782 JA	286	}
03ba3782 JA	287
a88a341a TH	288	/*
	289	* Initial write bandwidth: 100 MB/s
	290	*/
	291	#define INIT_BW (100 << (20 - PAGE_SHIFT))
	292
f0054bb1	293	static int wb_init(struct bdi_writeback wb, struct backing_dev_info bdi)
6467716a	294	{
93f78d88 TH	295	int i, err;
93f78d88 TH	296
6467716a AB	297	memset(wb, 0, sizeof(*wb));
	298
	299	wb->bdi = bdi;
	300	wb->last_old_flush = jiffies;
	301	INIT_LIST_HEAD(&wb->b_dirty);
	302	INIT_LIST_HEAD(&wb->b_io);
	303	INIT_LIST_HEAD(&wb->b_more_io);
0ae45f63	304	INIT_LIST_HEAD(&wb->b_dirty_time);
f758eeab	305	spin_lock_init(&wb->list_lock);
93f78d88	306
a88a341a TH	307	wb->bw_time_stamp = jiffies;
	308	wb->balanced_dirty_ratelimit = INIT_BW;
	309	wb->dirty_ratelimit = INIT_BW;
	310	wb->write_bandwidth = INIT_BW;
	311	wb->avg_write_bandwidth = INIT_BW;
	312
f0054bb1 TH	313	spin_lock_init(&wb->work_lock);
	314	INIT_LIST_HEAD(&wb->work_list);
	315	INIT_DELAYED_WORK(&wb->dwork, wb_workfn);
	316
a88a341a TH	317	err = fprop_local_init_percpu(&wb->completions, GFP_KERNEL);
	318	if (err)
	319	return err;
	320
93f78d88 TH	321	for (i = 0; i < NR_WB_STAT_ITEMS; i++) {
	322	err = percpu_counter_init(&wb->stat[i], 0, GFP_KERNEL);
	323	if (err) {
	324	while (--i)
	325	percpu_counter_destroy(&wb->stat[i]);
a88a341a	326	fprop_local_destroy_percpu(&wb->completions);
93f78d88 TH	327	return err;
	328	}
	329	}
	330
	331	return 0;
	332	}
	333
46100071 TH	334	/*
	335	* Remove bdi from the global list and shutdown any threads we have running
	336	*/
	337	static void wb_shutdown(struct bdi_writeback *wb)
	338	{
	339	/* Make sure nobody queues further work */
	340	spin_lock_bh(&wb->work_lock);
	341	if (!test_and_clear_bit(WB_registered, &wb->state)) {
	342	spin_unlock_bh(&wb->work_lock);
	343	return;
	344	}
	345	spin_unlock_bh(&wb->work_lock);
	346
	347	/*
	348	* Drain work list and shutdown the delayed_work. !WB_registered
	349	* tells wb_workfn() that @wb is dying and its work_list needs to
	350	* be drained no matter what.
	351	*/
	352	mod_delayed_work(bdi_wq, &wb->dwork, 0);
	353	flush_delayed_work(&wb->dwork);
	354	WARN_ON(!list_empty(&wb->work_list));
	355	}
	356
f0054bb1	357	static void wb_exit(struct bdi_writeback *wb)
93f78d88 TH	358	{
	359	int i;
	360
	361	WARN_ON(delayed_work_pending(&wb->dwork));
	362
	363	for (i = 0; i < NR_WB_STAT_ITEMS; i++)
	364	percpu_counter_destroy(&wb->stat[i]);
6467716a	365
a88a341a TH	366	fprop_local_destroy_percpu(&wb->completions);
a88a341a TH	367	}
e98be2d5	368
b2e8fb6e PZ	369	int bdi_init(struct backing_dev_info *bdi)
b2e8fb6e PZ	370	{
93f78d88	371	int err;
b2e8fb6e	372
cf0ca9fe PZ	373	bdi->dev = NULL;
cf0ca9fe PZ	374
189d3c4a	375	bdi->min_ratio = 0;
a42dde04	376	bdi->max_ratio = 100;
eb608e3a	377	bdi->max_prop_frac = FPROP_FRAC_BASE;
66f3b8e2	378	INIT_LIST_HEAD(&bdi->bdi_list);
03ba3782	379
f0054bb1	380	err = wb_init(&bdi->wb, bdi);
93f78d88 TH	381	if (err)
93f78d88 TH	382	return err;
04fbfdc1	383
93f78d88	384	return 0;
b2e8fb6e PZ	385	}
	386	EXPORT_SYMBOL(bdi_init);
	387
46100071 TH	388	int bdi_register(struct backing_dev_info bdi, struct device parent,
	389	const char *fmt, ...)
	390	{
	391	va_list args;
	392	struct device *dev;
	393
	394	if (bdi->dev) /* The driver needs to use separate queues per device */
	395	return 0;
	396
	397	va_start(args, fmt);
	398	dev = device_create_vargs(bdi_class, parent, MKDEV(0, 0), bdi, fmt, args);
	399	va_end(args);
	400	if (IS_ERR(dev))
	401	return PTR_ERR(dev);
	402
	403	bdi->dev = dev;
	404
	405	bdi_debug_register(bdi, dev_name(dev));
	406	set_bit(WB_registered, &bdi->wb.state);
	407
	408	spin_lock_bh(&bdi_lock);
	409	list_add_tail_rcu(&bdi->bdi_list, &bdi_list);
	410	spin_unlock_bh(&bdi_lock);
	411
	412	trace_writeback_bdi_register(bdi);
	413	return 0;
	414	}
	415	EXPORT_SYMBOL(bdi_register);
	416
	417	int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev)
	418	{
	419	return bdi_register(bdi, NULL, "%u:%u", MAJOR(dev), MINOR(dev));
	420	}
	421	EXPORT_SYMBOL(bdi_register_dev);
	422
	423	/*
	424	* Remove bdi from bdi_list, and ensure that it is no longer visible
	425	*/
	426	static void bdi_remove_from_list(struct backing_dev_info *bdi)
	427	{
	428	spin_lock_bh(&bdi_lock);
	429	list_del_rcu(&bdi->bdi_list);
	430	spin_unlock_bh(&bdi_lock);
	431
	432	synchronize_rcu_expedited();
	433	}
	434
	435	/*
	436	* Called when the device behind @bdi has been removed or ejected.
	437	*
	438	* We can't really do much here except for reducing the dirty ratio at
	439	* the moment. In the future we should be able to set a flag so that
	440	* the filesystem can handle errors at mark_inode_dirty time instead
	441	* of only at writeback time.
	442	*/
	443	void bdi_unregister(struct backing_dev_info *bdi)
	444	{
	445	if (WARN_ON_ONCE(!bdi->dev))
	446	return;
	447
	448	bdi_set_min_ratio(bdi, 0);
	449	}
	450	EXPORT_SYMBOL(bdi_unregister);
	451
b2e8fb6e PZ	452	void bdi_destroy(struct backing_dev_info *bdi)
b2e8fb6e PZ	453	{
f0054bb1 TH	454	/* make sure nobody finds us on the bdi_list anymore */
	455	bdi_remove_from_list(bdi);
	456	wb_shutdown(&bdi->wb);
7a401a97	457
c4db59d3 CH	458	if (bdi->dev) {
	459	bdi_debug_unregister(bdi);
	460	device_unregister(bdi->dev);
	461	bdi->dev = NULL;
	462	}
	463
f0054bb1	464	wb_exit(&bdi->wb);
b2e8fb6e PZ	465	}
	466	EXPORT_SYMBOL(bdi_destroy);
	467
c3c53206 JA	468	/*
	469	* For use from filesystems to quickly init and register a bdi associated
	470	* with dirty writeback
	471	*/
b4caecd4	472	int bdi_setup_and_register(struct backing_dev_info bdi, char name)
c3c53206	473	{
c3c53206 JA	474	int err;
	475
	476	bdi->name = name;
b4caecd4	477	bdi->capabilities = 0;
c3c53206 JA	478	err = bdi_init(bdi);
	479	if (err)
	480	return err;
	481
02aa2a37 KC	482	err = bdi_register(bdi, NULL, "%.28s-%ld", name,
02aa2a37 KC	483	atomic_long_inc_return(&bdi_seq));
c3c53206 JA	484	if (err) {
	485	bdi_destroy(bdi);
	486	return err;
	487	}
	488
	489	return 0;
	490	}
	491	EXPORT_SYMBOL(bdi_setup_and_register);
	492
3fcfab16 AM	493	static wait_queue_head_t congestion_wqh[2] = {
	494	__WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[0]),
	495	__WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1])
	496	};
0e093d99	497	static atomic_t nr_bdi_congested[2];
3fcfab16	498
1faa16d2	499	void clear_bdi_congested(struct backing_dev_info *bdi, int sync)
3fcfab16	500	{
4452226e	501	enum wb_state bit;
1faa16d2	502	wait_queue_head_t *wqh = &congestion_wqh[sync];
3fcfab16	503
4452226e TH	504	bit = sync ? WB_sync_congested : WB_async_congested;
4452226e TH	505	if (test_and_clear_bit(bit, &bdi->wb.state))
0e093d99	506	atomic_dec(&nr_bdi_congested[sync]);
4e857c58	507	smp_mb__after_atomic();
3fcfab16 AM	508	if (waitqueue_active(wqh))
	509	wake_up(wqh);
	510	}
	511	EXPORT_SYMBOL(clear_bdi_congested);
	512
1faa16d2	513	void set_bdi_congested(struct backing_dev_info *bdi, int sync)
3fcfab16	514	{
4452226e	515	enum wb_state bit;
3fcfab16	516
4452226e TH	517	bit = sync ? WB_sync_congested : WB_async_congested;
4452226e TH	518	if (!test_and_set_bit(bit, &bdi->wb.state))
0e093d99	519	atomic_inc(&nr_bdi_congested[sync]);
3fcfab16 AM	520	}
	521	EXPORT_SYMBOL(set_bdi_congested);
	522
	523	/**
	524	* congestion_wait - wait for a backing_dev to become uncongested
8aa7e847	525	* @sync: SYNC or ASYNC IO
3fcfab16 AM	526	* @timeout: timeout in jiffies
	527	*
	528	* Waits for up to @timeout jiffies for a backing_dev (any backing_dev) to exit
	529	* write congestion. If no backing_devs are congested then just wait for the
	530	* next write to be completed.
	531	*/
8aa7e847	532	long congestion_wait(int sync, long timeout)
3fcfab16 AM	533	{
3fcfab16 AM	534	long ret;
52bb9198	535	unsigned long start = jiffies;
3fcfab16	536	DEFINE_WAIT(wait);
8aa7e847	537	wait_queue_head_t *wqh = &congestion_wqh[sync];
3fcfab16 AM	538
	539	prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
	540	ret = io_schedule_timeout(timeout);
	541	finish_wait(wqh, &wait);
52bb9198 MG	542
	543	trace_writeback_congestion_wait(jiffies_to_usecs(timeout),
	544	jiffies_to_usecs(jiffies - start));
	545
3fcfab16 AM	546	return ret;
	547	}
	548	EXPORT_SYMBOL(congestion_wait);
04fbfdc1	549
0e093d99 MG	550	/**
	551	* wait_iff_congested - Conditionally wait for a backing_dev to become uncongested or a zone to complete writes
	552	* @zone: A zone to check if it is heavily congested
	553	* @sync: SYNC or ASYNC IO
	554	* @timeout: timeout in jiffies
	555	*
	556	* In the event of a congested backing_dev (any backing_dev) and the given
	557	* @zone has experienced recent congestion, this waits for up to @timeout
	558	* jiffies for either a BDI to exit congestion of the given @sync queue
	559	* or a write to complete.
	560	*
25985edc	561	* In the absence of zone congestion, cond_resched() is called to yield
0e093d99 MG	562	* the processor if necessary but otherwise does not sleep.
	563	*
	564	* The return value is 0 if the sleep is for the full timeout. Otherwise,
	565	* it is the number of jiffies that were still remaining when the function
	566	* returned. return_value == timeout implies the function did not sleep.
	567	*/
	568	long wait_iff_congested(struct zone *zone, int sync, long timeout)
	569	{
	570	long ret;
	571	unsigned long start = jiffies;
	572	DEFINE_WAIT(wait);
	573	wait_queue_head_t *wqh = &congestion_wqh[sync];
	574
	575	/*
	576	* If there is no congestion, or heavy congestion is not being
	577	* encountered in the current zone, yield if necessary instead
	578	* of sleeping on the congestion queue
	579	*/
	580	if (atomic_read(&nr_bdi_congested[sync]) == 0 \|\|
57054651	581	!test_bit(ZONE_CONGESTED, &zone->flags)) {
0e093d99 MG	582	cond_resched();
	583
	584	/* In case we scheduled, work out time remaining */
	585	ret = timeout - (jiffies - start);
	586	if (ret < 0)
	587	ret = 0;
	588
	589	goto out;
	590	}
	591
	592	/* Sleep until uncongested or a write happens */
	593	prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
	594	ret = io_schedule_timeout(timeout);
	595	finish_wait(wqh, &wait);
	596
	597	out:
	598	trace_writeback_wait_iff_congested(jiffies_to_usecs(timeout),
	599	jiffies_to_usecs(jiffies - start));
	600
	601	return ret;
	602	}
	603	EXPORT_SYMBOL(wait_iff_congested);
3965c9ae WL	604
	605	int pdflush_proc_obsolete(struct ctl_table *table, int write,
	606	void __user buffer, size_t lenp, loff_t *ppos)
	607	{
	608	char kbuf[] = "0\n";
	609
4c3bffc2	610	if (ppos \|\| lenp < sizeof(kbuf)) {
3965c9ae WL	611	*lenp = 0;
	612	return 0;
	613	}
	614
	615	if (copy_to_user(buffer, kbuf, sizeof(kbuf)))
	616	return -EFAULT;
	617	printk_once(KERN_WARNING "%s exported in /proc is scheduled for removal\n",
	618	table->procname);
	619
	620	*lenp = 2;
	621	ppos += lenp;
	622	return 2;
	623	}