[linux-block.git] / fs / f2fs / iostat.c

// SPDX-License-Identifier: GPL-2.0
/*
 * f2fs iostat support
 *
 * Copyright 2021 Google LLC
 * Author: Daeho Jeong <daehojeong@google.com>
 */

#include <linux/fs.h>
#include <linux/f2fs_fs.h>
#include <linux/seq_file.h>

#include "f2fs.h"
#include "iostat.h"
#include <trace/events/f2fs.h>

#define NUM_PREALLOC_IOSTAT_CTXS	128
static struct kmem_cache *bio_iostat_ctx_cache;
static mempool_t *bio_iostat_ctx_pool;

int __maybe_unused iostat_info_seq_show(struct seq_file *seq, void *offset)
{
	struct super_block *sb = seq->private;
	struct f2fs_sb_info *sbi = F2FS_SB(sb);
	time64_t now = ktime_get_real_seconds();

	if (!sbi->iostat_enable)
		return 0;

	seq_printf(seq, "time:		%-16llu\n", now);

	/* print app write IOs */
	seq_puts(seq, "[WRITE]\n");
	seq_printf(seq, "app buffered data:	%-16llu\n",
				sbi->rw_iostat[APP_BUFFERED_IO]);
	seq_printf(seq, "app direct data:	%-16llu\n",
				sbi->rw_iostat[APP_DIRECT_IO]);
	seq_printf(seq, "app mapped data:	%-16llu\n",
				sbi->rw_iostat[APP_MAPPED_IO]);
	seq_printf(seq, "app buffered cdata:	%-16llu\n",
				sbi->rw_iostat[APP_BUFFERED_CDATA_IO]);
	seq_printf(seq, "app mapped cdata:	%-16llu\n",
				sbi->rw_iostat[APP_MAPPED_CDATA_IO]);

	/* print fs write IOs */
	seq_printf(seq, "fs data:		%-16llu\n",
				sbi->rw_iostat[FS_DATA_IO]);
	seq_printf(seq, "fs cdata:		%-16llu\n",
				sbi->rw_iostat[FS_CDATA_IO]);
	seq_printf(seq, "fs node:		%-16llu\n",
				sbi->rw_iostat[FS_NODE_IO]);
	seq_printf(seq, "fs meta:		%-16llu\n",
				sbi->rw_iostat[FS_META_IO]);
	seq_printf(seq, "fs gc data:		%-16llu\n",
				sbi->rw_iostat[FS_GC_DATA_IO]);
	seq_printf(seq, "fs gc node:		%-16llu\n",
				sbi->rw_iostat[FS_GC_NODE_IO]);
	seq_printf(seq, "fs cp data:		%-16llu\n",
				sbi->rw_iostat[FS_CP_DATA_IO]);
	seq_printf(seq, "fs cp node:		%-16llu\n",
				sbi->rw_iostat[FS_CP_NODE_IO]);
	seq_printf(seq, "fs cp meta:		%-16llu\n",
				sbi->rw_iostat[FS_CP_META_IO]);

	/* print app read IOs */
	seq_puts(seq, "[READ]\n");
	seq_printf(seq, "app buffered data:	%-16llu\n",
				sbi->rw_iostat[APP_BUFFERED_READ_IO]);
	seq_printf(seq, "app direct data:	%-16llu\n",
				sbi->rw_iostat[APP_DIRECT_READ_IO]);
	seq_printf(seq, "app mapped data:	%-16llu\n",
				sbi->rw_iostat[APP_MAPPED_READ_IO]);
	seq_printf(seq, "app buffered cdata:	%-16llu\n",
				sbi->rw_iostat[APP_BUFFERED_CDATA_READ_IO]);
	seq_printf(seq, "app mapped cdata:	%-16llu\n",
				sbi->rw_iostat[APP_MAPPED_CDATA_READ_IO]);

	/* print fs read IOs */
	seq_printf(seq, "fs data:		%-16llu\n",
				sbi->rw_iostat[FS_DATA_READ_IO]);
	seq_printf(seq, "fs gc data:		%-16llu\n",
				sbi->rw_iostat[FS_GDATA_READ_IO]);
	seq_printf(seq, "fs cdata:		%-16llu\n",
				sbi->rw_iostat[FS_CDATA_READ_IO]);
	seq_printf(seq, "fs node:		%-16llu\n",
				sbi->rw_iostat[FS_NODE_READ_IO]);
	seq_printf(seq, "fs meta:		%-16llu\n",
				sbi->rw_iostat[FS_META_READ_IO]);

	/* print other IOs */
	seq_puts(seq, "[OTHER]\n");
	seq_printf(seq, "fs discard:		%-16llu\n",
				sbi->rw_iostat[FS_DISCARD]);

	return 0;
}

static inline void __record_iostat_latency(struct f2fs_sb_info *sbi)
{
	int io, idx = 0;
	unsigned int cnt;
	struct f2fs_iostat_latency iostat_lat[MAX_IO_TYPE][NR_PAGE_TYPE];
	struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
	unsigned long flags;

	spin_lock_irqsave(&sbi->iostat_lat_lock, flags);
	for (idx = 0; idx < MAX_IO_TYPE; idx++) {
		for (io = 0; io < NR_PAGE_TYPE; io++) {
			cnt = io_lat->bio_cnt[idx][io];
			iostat_lat[idx][io].peak_lat =
			   jiffies_to_msecs(io_lat->peak_lat[idx][io]);
			iostat_lat[idx][io].cnt = cnt;
			iostat_lat[idx][io].avg_lat = cnt ?
			   jiffies_to_msecs(io_lat->sum_lat[idx][io]) / cnt : 0;
			io_lat->sum_lat[idx][io] = 0;
			io_lat->peak_lat[idx][io] = 0;
			io_lat->bio_cnt[idx][io] = 0;
		}
	}
	spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags);

	trace_f2fs_iostat_latency(sbi, iostat_lat);
}

static inline void f2fs_record_iostat(struct f2fs_sb_info *sbi)
{
	unsigned long long iostat_diff[NR_IO_TYPE];
	int i;
	unsigned long flags;

	if (time_is_after_jiffies(sbi->iostat_next_period))
		return;

	/* Need double check under the lock */
	spin_lock_irqsave(&sbi->iostat_lock, flags);
	if (time_is_after_jiffies(sbi->iostat_next_period)) {
		spin_unlock_irqrestore(&sbi->iostat_lock, flags);
		return;
	}
	sbi->iostat_next_period = jiffies +
				msecs_to_jiffies(sbi->iostat_period_ms);

	for (i = 0; i < NR_IO_TYPE; i++) {
		iostat_diff[i] = sbi->rw_iostat[i] -
				sbi->prev_rw_iostat[i];
		sbi->prev_rw_iostat[i] = sbi->rw_iostat[i];
	}
	spin_unlock_irqrestore(&sbi->iostat_lock, flags);

	trace_f2fs_iostat(sbi, iostat_diff);

	__record_iostat_latency(sbi);
}

void f2fs_reset_iostat(struct f2fs_sb_info *sbi)
{
	struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
	int i;

	spin_lock_irq(&sbi->iostat_lock);
	for (i = 0; i < NR_IO_TYPE; i++) {
		sbi->rw_iostat[i] = 0;
		sbi->prev_rw_iostat[i] = 0;
	}
	spin_unlock_irq(&sbi->iostat_lock);

	spin_lock_irq(&sbi->iostat_lat_lock);
	memset(io_lat, 0, sizeof(struct iostat_lat_info));
	spin_unlock_irq(&sbi->iostat_lat_lock);
}

void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode,
			enum iostat_type type, unsigned long long io_bytes)
{
	unsigned long flags;

	if (!sbi->iostat_enable)
		return;

	spin_lock_irqsave(&sbi->iostat_lock, flags);
	sbi->rw_iostat[type] += io_bytes;

	if (type == APP_BUFFERED_IO || type == APP_DIRECT_IO)
		sbi->rw_iostat[APP_WRITE_IO] += io_bytes;

	if (type == APP_BUFFERED_READ_IO || type == APP_DIRECT_READ_IO)
		sbi->rw_iostat[APP_READ_IO] += io_bytes;

#ifdef CONFIG_F2FS_FS_COMPRESSION
	if (inode && f2fs_compressed_file(inode)) {
		if (type == APP_BUFFERED_IO)
			sbi->rw_iostat[APP_BUFFERED_CDATA_IO] += io_bytes;

		if (type == APP_BUFFERED_READ_IO)
			sbi->rw_iostat[APP_BUFFERED_CDATA_READ_IO] += io_bytes;

		if (type == APP_MAPPED_READ_IO)
			sbi->rw_iostat[APP_MAPPED_CDATA_READ_IO] += io_bytes;

		if (type == APP_MAPPED_IO)
			sbi->rw_iostat[APP_MAPPED_CDATA_IO] += io_bytes;

		if (type == FS_DATA_READ_IO)
			sbi->rw_iostat[FS_CDATA_READ_IO] += io_bytes;

		if (type == FS_DATA_IO)
			sbi->rw_iostat[FS_CDATA_IO] += io_bytes;
	}
#endif

	spin_unlock_irqrestore(&sbi->iostat_lock, flags);

	f2fs_record_iostat(sbi);
}

static inline void __update_iostat_latency(struct bio_iostat_ctx *iostat_ctx,
				int rw, bool is_sync)
{
	unsigned long ts_diff;
	unsigned int iotype = iostat_ctx->type;
	struct f2fs_sb_info *sbi = iostat_ctx->sbi;
	struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
	int idx;
	unsigned long flags;

	if (!sbi->iostat_enable)
		return;

	ts_diff = jiffies - iostat_ctx->submit_ts;
	if (iotype >= META_FLUSH)
		iotype = META;

	if (rw == 0) {
		idx = READ_IO;
	} else {
		if (is_sync)
			idx = WRITE_SYNC_IO;
		else
			idx = WRITE_ASYNC_IO;
	}

	spin_lock_irqsave(&sbi->iostat_lat_lock, flags);
	io_lat->sum_lat[idx][iotype] += ts_diff;
	io_lat->bio_cnt[idx][iotype]++;
	if (ts_diff > io_lat->peak_lat[idx][iotype])
		io_lat->peak_lat[idx][iotype] = ts_diff;
	spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags);
}

void iostat_update_and_unbind_ctx(struct bio *bio, int rw)
{
	struct bio_iostat_ctx *iostat_ctx = bio->bi_private;
	bool is_sync = bio->bi_opf & REQ_SYNC;

	if (rw == 0)
		bio->bi_private = iostat_ctx->post_read_ctx;
	else
		bio->bi_private = iostat_ctx->sbi;
	__update_iostat_latency(iostat_ctx, rw, is_sync);
	mempool_free(iostat_ctx, bio_iostat_ctx_pool);
}

void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi,
		struct bio *bio, struct bio_post_read_ctx *ctx)
{
	struct bio_iostat_ctx *iostat_ctx;
	/* Due to the mempool, this never fails. */
	iostat_ctx = mempool_alloc(bio_iostat_ctx_pool, GFP_NOFS);
	iostat_ctx->sbi = sbi;
	iostat_ctx->submit_ts = 0;
	iostat_ctx->type = 0;
	iostat_ctx->post_read_ctx = ctx;
	bio->bi_private = iostat_ctx;
}

int __init f2fs_init_iostat_processing(void)
{
	bio_iostat_ctx_cache =
		kmem_cache_create("f2fs_bio_iostat_ctx",
				  sizeof(struct bio_iostat_ctx), 0, 0, NULL);
	if (!bio_iostat_ctx_cache)
		goto fail;
	bio_iostat_ctx_pool =
		mempool_create_slab_pool(NUM_PREALLOC_IOSTAT_CTXS,
					 bio_iostat_ctx_cache);
	if (!bio_iostat_ctx_pool)
		goto fail_free_cache;
	return 0;

fail_free_cache:
	kmem_cache_destroy(bio_iostat_ctx_cache);
fail:
	return -ENOMEM;
}

void f2fs_destroy_iostat_processing(void)
{
	mempool_destroy(bio_iostat_ctx_pool);
	kmem_cache_destroy(bio_iostat_ctx_cache);
}

int f2fs_init_iostat(struct f2fs_sb_info *sbi)
{
	/* init iostat info */
	spin_lock_init(&sbi->iostat_lock);
	spin_lock_init(&sbi->iostat_lat_lock);
	sbi->iostat_enable = false;
	sbi->iostat_period_ms = DEFAULT_IOSTAT_PERIOD_MS;
	sbi->iostat_io_lat = f2fs_kzalloc(sbi, sizeof(struct iostat_lat_info),
					GFP_KERNEL);
	if (!sbi->iostat_io_lat)
		return -ENOMEM;

	return 0;
}

void f2fs_destroy_iostat(struct f2fs_sb_info *sbi)
{
	kfree(sbi->iostat_io_lat);
}
Commit	Line	Data
52118743 DJ	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* f2fs iostat support
	4	*
	5	* Copyright 2021 Google LLC
	6	* Author: Daeho Jeong <daehojeong@google.com>
	7	*/
	8
	9	#include <linux/fs.h>
	10	#include <linux/f2fs_fs.h>
	11	#include <linux/seq_file.h>
	12
	13	#include "f2fs.h"
	14	#include "iostat.h"
	15	#include <trace/events/f2fs.h>
	16
a4b68176 DJ	17	#define NUM_PREALLOC_IOSTAT_CTXS 128
	18	static struct kmem_cache *bio_iostat_ctx_cache;
	19	static mempool_t *bio_iostat_ctx_pool;
	20
52118743 DJ	21	int __maybe_unused iostat_info_seq_show(struct seq_file seq, void offset)
	22	{
	23	struct super_block *sb = seq->private;
	24	struct f2fs_sb_info *sbi = F2FS_SB(sb);
	25	time64_t now = ktime_get_real_seconds();
	26
	27	if (!sbi->iostat_enable)
	28	return 0;
	29
	30	seq_printf(seq, "time: %-16llu\n", now);
	31
	32	/* print app write IOs */
	33	seq_puts(seq, "[WRITE]\n");
34a23525	34	seq_printf(seq, "app buffered data: %-16llu\n",
52118743	35	sbi->rw_iostat[APP_BUFFERED_IO]);
34a23525	36	seq_printf(seq, "app direct data: %-16llu\n",
52118743	37	sbi->rw_iostat[APP_DIRECT_IO]);
34a23525	38	seq_printf(seq, "app mapped data: %-16llu\n",
52118743	39	sbi->rw_iostat[APP_MAPPED_IO]);
34a23525 CY	40	seq_printf(seq, "app buffered cdata: %-16llu\n",
	41	sbi->rw_iostat[APP_BUFFERED_CDATA_IO]);
	42	seq_printf(seq, "app mapped cdata: %-16llu\n",
	43	sbi->rw_iostat[APP_MAPPED_CDATA_IO]);
52118743 DJ	44
52118743 DJ	45	/* print fs write IOs */
34a23525	46	seq_printf(seq, "fs data: %-16llu\n",
52118743	47	sbi->rw_iostat[FS_DATA_IO]);
34a23525 CY	48	seq_printf(seq, "fs cdata: %-16llu\n",
	49	sbi->rw_iostat[FS_CDATA_IO]);
	50	seq_printf(seq, "fs node: %-16llu\n",
52118743	51	sbi->rw_iostat[FS_NODE_IO]);
34a23525	52	seq_printf(seq, "fs meta: %-16llu\n",
52118743	53	sbi->rw_iostat[FS_META_IO]);
34a23525	54	seq_printf(seq, "fs gc data: %-16llu\n",
52118743	55	sbi->rw_iostat[FS_GC_DATA_IO]);
34a23525	56	seq_printf(seq, "fs gc node: %-16llu\n",
52118743	57	sbi->rw_iostat[FS_GC_NODE_IO]);
34a23525	58	seq_printf(seq, "fs cp data: %-16llu\n",
52118743	59	sbi->rw_iostat[FS_CP_DATA_IO]);
34a23525	60	seq_printf(seq, "fs cp node: %-16llu\n",
52118743	61	sbi->rw_iostat[FS_CP_NODE_IO]);
34a23525	62	seq_printf(seq, "fs cp meta: %-16llu\n",
52118743 DJ	63	sbi->rw_iostat[FS_CP_META_IO]);
	64
	65	/* print app read IOs */
	66	seq_puts(seq, "[READ]\n");
34a23525	67	seq_printf(seq, "app buffered data: %-16llu\n",
52118743	68	sbi->rw_iostat[APP_BUFFERED_READ_IO]);
34a23525	69	seq_printf(seq, "app direct data: %-16llu\n",
52118743	70	sbi->rw_iostat[APP_DIRECT_READ_IO]);
34a23525	71	seq_printf(seq, "app mapped data: %-16llu\n",
52118743	72	sbi->rw_iostat[APP_MAPPED_READ_IO]);
34a23525 CY	73	seq_printf(seq, "app buffered cdata: %-16llu\n",
	74	sbi->rw_iostat[APP_BUFFERED_CDATA_READ_IO]);
	75	seq_printf(seq, "app mapped cdata: %-16llu\n",
	76	sbi->rw_iostat[APP_MAPPED_CDATA_READ_IO]);
52118743 DJ	77
52118743 DJ	78	/* print fs read IOs */
34a23525	79	seq_printf(seq, "fs data: %-16llu\n",
52118743	80	sbi->rw_iostat[FS_DATA_READ_IO]);
34a23525	81	seq_printf(seq, "fs gc data: %-16llu\n",
52118743	82	sbi->rw_iostat[FS_GDATA_READ_IO]);
34a23525	83	seq_printf(seq, "fs cdata: %-16llu\n",
52118743	84	sbi->rw_iostat[FS_CDATA_READ_IO]);
34a23525	85	seq_printf(seq, "fs node: %-16llu\n",
52118743	86	sbi->rw_iostat[FS_NODE_READ_IO]);
34a23525	87	seq_printf(seq, "fs meta: %-16llu\n",
52118743 DJ	88	sbi->rw_iostat[FS_META_READ_IO]);
	89
	90	/* print other IOs */
	91	seq_puts(seq, "[OTHER]\n");
34a23525	92	seq_printf(seq, "fs discard: %-16llu\n",
52118743 DJ	93	sbi->rw_iostat[FS_DISCARD]);
	94
	95	return 0;
	96	}
	97
a4b68176 DJ	98	static inline void __record_iostat_latency(struct f2fs_sb_info *sbi)
	99	{
	100	int io, idx = 0;
	101	unsigned int cnt;
	102	struct f2fs_iostat_latency iostat_lat[MAX_IO_TYPE][NR_PAGE_TYPE];
	103	struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
61803e98	104	unsigned long flags;
a4b68176	105
61803e98	106	spin_lock_irqsave(&sbi->iostat_lat_lock, flags);
a4b68176 DJ	107	for (idx = 0; idx < MAX_IO_TYPE; idx++) {
	108	for (io = 0; io < NR_PAGE_TYPE; io++) {
	109	cnt = io_lat->bio_cnt[idx][io];
	110	iostat_lat[idx][io].peak_lat =
	111	jiffies_to_msecs(io_lat->peak_lat[idx][io]);
	112	iostat_lat[idx][io].cnt = cnt;
	113	iostat_lat[idx][io].avg_lat = cnt ?
	114	jiffies_to_msecs(io_lat->sum_lat[idx][io]) / cnt : 0;
	115	io_lat->sum_lat[idx][io] = 0;
	116	io_lat->peak_lat[idx][io] = 0;
	117	io_lat->bio_cnt[idx][io] = 0;
	118	}
	119	}
61803e98	120	spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags);
a4b68176 DJ	121
	122	trace_f2fs_iostat_latency(sbi, iostat_lat);
	123	}
	124
52118743 DJ	125	static inline void f2fs_record_iostat(struct f2fs_sb_info *sbi)
	126	{
	127	unsigned long long iostat_diff[NR_IO_TYPE];
	128	int i;
61803e98	129	unsigned long flags;
52118743 DJ	130
	131	if (time_is_after_jiffies(sbi->iostat_next_period))
	132	return;
	133
	134	/* Need double check under the lock */
61803e98	135	spin_lock_irqsave(&sbi->iostat_lock, flags);
52118743	136	if (time_is_after_jiffies(sbi->iostat_next_period)) {
61803e98	137	spin_unlock_irqrestore(&sbi->iostat_lock, flags);
52118743 DJ	138	return;
	139	}
	140	sbi->iostat_next_period = jiffies +
	141	msecs_to_jiffies(sbi->iostat_period_ms);
	142
	143	for (i = 0; i < NR_IO_TYPE; i++) {
	144	iostat_diff[i] = sbi->rw_iostat[i] -
	145	sbi->prev_rw_iostat[i];
	146	sbi->prev_rw_iostat[i] = sbi->rw_iostat[i];
	147	}
61803e98	148	spin_unlock_irqrestore(&sbi->iostat_lock, flags);
52118743 DJ	149
52118743 DJ	150	trace_f2fs_iostat(sbi, iostat_diff);
a4b68176 DJ	151
a4b68176 DJ	152	__record_iostat_latency(sbi);
52118743 DJ	153	}
	154
	155	void f2fs_reset_iostat(struct f2fs_sb_info *sbi)
	156	{
a4b68176	157	struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
52118743 DJ	158	int i;
52118743 DJ	159
61803e98	160	spin_lock_irq(&sbi->iostat_lock);
52118743 DJ	161	for (i = 0; i < NR_IO_TYPE; i++) {
	162	sbi->rw_iostat[i] = 0;
	163	sbi->prev_rw_iostat[i] = 0;
	164	}
61803e98	165	spin_unlock_irq(&sbi->iostat_lock);
a4b68176	166
61803e98	167	spin_lock_irq(&sbi->iostat_lat_lock);
a4b68176	168	memset(io_lat, 0, sizeof(struct iostat_lat_info));
61803e98	169	spin_unlock_irq(&sbi->iostat_lat_lock);
52118743 DJ	170	}
52118743 DJ	171
34a23525	172	void f2fs_update_iostat(struct f2fs_sb_info sbi, struct inode inode,
52118743 DJ	173	enum iostat_type type, unsigned long long io_bytes)
52118743 DJ	174	{
61803e98 DJ	175	unsigned long flags;
61803e98 DJ	176
52118743 DJ	177	if (!sbi->iostat_enable)
	178	return;
	179
61803e98	180	spin_lock_irqsave(&sbi->iostat_lock, flags);
52118743 DJ	181	sbi->rw_iostat[type] += io_bytes;
52118743 DJ	182
a1e09b03 EB	183	if (type == APP_BUFFERED_IO \|\| type == APP_DIRECT_IO)
a1e09b03 EB	184	sbi->rw_iostat[APP_WRITE_IO] += io_bytes;
52118743	185
a1e09b03 EB	186	if (type == APP_BUFFERED_READ_IO \|\| type == APP_DIRECT_READ_IO)
	187	sbi->rw_iostat[APP_READ_IO] += io_bytes;
	188
34a23525 CY	189	#ifdef CONFIG_F2FS_FS_COMPRESSION
	190	if (inode && f2fs_compressed_file(inode)) {
	191	if (type == APP_BUFFERED_IO)
	192	sbi->rw_iostat[APP_BUFFERED_CDATA_IO] += io_bytes;
	193
	194	if (type == APP_BUFFERED_READ_IO)
	195	sbi->rw_iostat[APP_BUFFERED_CDATA_READ_IO] += io_bytes;
	196
	197	if (type == APP_MAPPED_READ_IO)
	198	sbi->rw_iostat[APP_MAPPED_CDATA_READ_IO] += io_bytes;
	199
	200	if (type == APP_MAPPED_IO)
	201	sbi->rw_iostat[APP_MAPPED_CDATA_IO] += io_bytes;
	202
	203	if (type == FS_DATA_READ_IO)
	204	sbi->rw_iostat[FS_CDATA_READ_IO] += io_bytes;
	205
	206	if (type == FS_DATA_IO)
	207	sbi->rw_iostat[FS_CDATA_IO] += io_bytes;
	208	}
	209	#endif
	210
61803e98	211	spin_unlock_irqrestore(&sbi->iostat_lock, flags);
52118743 DJ	212
	213	f2fs_record_iostat(sbi);
	214	}
	215
a4b68176 DJ	216	static inline void __update_iostat_latency(struct bio_iostat_ctx *iostat_ctx,
	217	int rw, bool is_sync)
	218	{
	219	unsigned long ts_diff;
	220	unsigned int iotype = iostat_ctx->type;
a4b68176 DJ	221	struct f2fs_sb_info *sbi = iostat_ctx->sbi;
	222	struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
	223	int idx;
61803e98	224	unsigned long flags;
a4b68176 DJ	225
	226	if (!sbi->iostat_enable)
	227	return;
	228
	229	ts_diff = jiffies - iostat_ctx->submit_ts;
	230	if (iotype >= META_FLUSH)
	231	iotype = META;
	232
	233	if (rw == 0) {
	234	idx = READ_IO;
	235	} else {
	236	if (is_sync)
	237	idx = WRITE_SYNC_IO;
	238	else
	239	idx = WRITE_ASYNC_IO;
	240	}
	241
61803e98	242	spin_lock_irqsave(&sbi->iostat_lat_lock, flags);
a4b68176 DJ	243	io_lat->sum_lat[idx][iotype] += ts_diff;
	244	io_lat->bio_cnt[idx][iotype]++;
	245	if (ts_diff > io_lat->peak_lat[idx][iotype])
	246	io_lat->peak_lat[idx][iotype] = ts_diff;
61803e98	247	spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags);
a4b68176 DJ	248	}
	249
	250	void iostat_update_and_unbind_ctx(struct bio *bio, int rw)
	251	{
	252	struct bio_iostat_ctx *iostat_ctx = bio->bi_private;
	253	bool is_sync = bio->bi_opf & REQ_SYNC;
	254
	255	if (rw == 0)
	256	bio->bi_private = iostat_ctx->post_read_ctx;
	257	else
	258	bio->bi_private = iostat_ctx->sbi;
	259	__update_iostat_latency(iostat_ctx, rw, is_sync);
	260	mempool_free(iostat_ctx, bio_iostat_ctx_pool);
	261	}
	262
	263	void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi,
	264	struct bio bio, struct bio_post_read_ctx ctx)
	265	{
	266	struct bio_iostat_ctx *iostat_ctx;
	267	/* Due to the mempool, this never fails. */
	268	iostat_ctx = mempool_alloc(bio_iostat_ctx_pool, GFP_NOFS);
	269	iostat_ctx->sbi = sbi;
	270	iostat_ctx->submit_ts = 0;
	271	iostat_ctx->type = 0;
	272	iostat_ctx->post_read_ctx = ctx;
	273	bio->bi_private = iostat_ctx;
	274	}
	275
	276	int __init f2fs_init_iostat_processing(void)
	277	{
	278	bio_iostat_ctx_cache =
	279	kmem_cache_create("f2fs_bio_iostat_ctx",
	280	sizeof(struct bio_iostat_ctx), 0, 0, NULL);
	281	if (!bio_iostat_ctx_cache)
	282	goto fail;
	283	bio_iostat_ctx_pool =
	284	mempool_create_slab_pool(NUM_PREALLOC_IOSTAT_CTXS,
	285	bio_iostat_ctx_cache);
	286	if (!bio_iostat_ctx_pool)
	287	goto fail_free_cache;
	288	return 0;
	289
	290	fail_free_cache:
	291	kmem_cache_destroy(bio_iostat_ctx_cache);
	292	fail:
	293	return -ENOMEM;
	294	}
	295
	296	void f2fs_destroy_iostat_processing(void)
	297	{
	298	mempool_destroy(bio_iostat_ctx_pool);
	299	kmem_cache_destroy(bio_iostat_ctx_cache);
	300	}
	301
52118743 DJ	302	int f2fs_init_iostat(struct f2fs_sb_info *sbi)
	303	{
	304	/* init iostat info */
	305	spin_lock_init(&sbi->iostat_lock);
a4b68176	306	spin_lock_init(&sbi->iostat_lat_lock);
52118743 DJ	307	sbi->iostat_enable = false;
52118743 DJ	308	sbi->iostat_period_ms = DEFAULT_IOSTAT_PERIOD_MS;
a4b68176 DJ	309	sbi->iostat_io_lat = f2fs_kzalloc(sbi, sizeof(struct iostat_lat_info),
	310	GFP_KERNEL);
	311	if (!sbi->iostat_io_lat)
	312	return -ENOMEM;
52118743 DJ	313
	314	return 0;
	315	}
a4b68176 DJ	316
	317	void f2fs_destroy_iostat(struct f2fs_sb_info *sbi)
	318	{
	319	kfree(sbi->iostat_io_lat);
	320	}