[linux-block.git] / fs / f2fs / iostat.c

// SPDX-License-Identifier: GPL-2.0
/*
 * f2fs iostat support
 *
 * Copyright 2021 Google LLC
 * Author: Daeho Jeong <daehojeong@google.com>
 */

#include <linux/fs.h>
#include <linux/f2fs_fs.h>
#include <linux/seq_file.h>

#include "f2fs.h"
#include "iostat.h"
#include <trace/events/f2fs.h>

#define NUM_PREALLOC_IOSTAT_CTXS	128
static struct kmem_cache *bio_iostat_ctx_cache;
static mempool_t *bio_iostat_ctx_pool;

int __maybe_unused iostat_info_seq_show(struct seq_file *seq, void *offset)
{
	struct super_block *sb = seq->private;
	struct f2fs_sb_info *sbi = F2FS_SB(sb);
	time64_t now = ktime_get_real_seconds();

	if (!sbi->iostat_enable)
		return 0;

	seq_printf(seq, "time:		%-16llu\n", now);

	/* print app write IOs */
	seq_puts(seq, "[WRITE]\n");
	seq_printf(seq, "app buffered:	%-16llu\n",
				sbi->rw_iostat[APP_BUFFERED_IO]);
	seq_printf(seq, "app direct:	%-16llu\n",
				sbi->rw_iostat[APP_DIRECT_IO]);
	seq_printf(seq, "app mapped:	%-16llu\n",
				sbi->rw_iostat[APP_MAPPED_IO]);

	/* print fs write IOs */
	seq_printf(seq, "fs data:	%-16llu\n",
				sbi->rw_iostat[FS_DATA_IO]);
	seq_printf(seq, "fs node:	%-16llu\n",
				sbi->rw_iostat[FS_NODE_IO]);
	seq_printf(seq, "fs meta:	%-16llu\n",
				sbi->rw_iostat[FS_META_IO]);
	seq_printf(seq, "fs gc data:	%-16llu\n",
				sbi->rw_iostat[FS_GC_DATA_IO]);
	seq_printf(seq, "fs gc node:	%-16llu\n",
				sbi->rw_iostat[FS_GC_NODE_IO]);
	seq_printf(seq, "fs cp data:	%-16llu\n",
				sbi->rw_iostat[FS_CP_DATA_IO]);
	seq_printf(seq, "fs cp node:	%-16llu\n",
				sbi->rw_iostat[FS_CP_NODE_IO]);
	seq_printf(seq, "fs cp meta:	%-16llu\n",
				sbi->rw_iostat[FS_CP_META_IO]);

	/* print app read IOs */
	seq_puts(seq, "[READ]\n");
	seq_printf(seq, "app buffered:	%-16llu\n",
				sbi->rw_iostat[APP_BUFFERED_READ_IO]);
	seq_printf(seq, "app direct:	%-16llu\n",
				sbi->rw_iostat[APP_DIRECT_READ_IO]);
	seq_printf(seq, "app mapped:	%-16llu\n",
				sbi->rw_iostat[APP_MAPPED_READ_IO]);

	/* print fs read IOs */
	seq_printf(seq, "fs data:	%-16llu\n",
				sbi->rw_iostat[FS_DATA_READ_IO]);
	seq_printf(seq, "fs gc data:	%-16llu\n",
				sbi->rw_iostat[FS_GDATA_READ_IO]);
	seq_printf(seq, "fs compr_data:	%-16llu\n",
				sbi->rw_iostat[FS_CDATA_READ_IO]);
	seq_printf(seq, "fs node:	%-16llu\n",
				sbi->rw_iostat[FS_NODE_READ_IO]);
	seq_printf(seq, "fs meta:	%-16llu\n",
				sbi->rw_iostat[FS_META_READ_IO]);

	/* print other IOs */
	seq_puts(seq, "[OTHER]\n");
	seq_printf(seq, "fs discard:	%-16llu\n",
				sbi->rw_iostat[FS_DISCARD]);

	return 0;
}

static inline void __record_iostat_latency(struct f2fs_sb_info *sbi)
{
	int io, idx = 0;
	unsigned int cnt;
	struct f2fs_iostat_latency iostat_lat[MAX_IO_TYPE][NR_PAGE_TYPE];
	struct iostat_lat_info *io_lat = sbi->iostat_io_lat;

	spin_lock_bh(&sbi->iostat_lat_lock);
	for (idx = 0; idx < MAX_IO_TYPE; idx++) {
		for (io = 0; io < NR_PAGE_TYPE; io++) {
			cnt = io_lat->bio_cnt[idx][io];
			iostat_lat[idx][io].peak_lat =
			   jiffies_to_msecs(io_lat->peak_lat[idx][io]);
			iostat_lat[idx][io].cnt = cnt;
			iostat_lat[idx][io].avg_lat = cnt ?
			   jiffies_to_msecs(io_lat->sum_lat[idx][io]) / cnt : 0;
			io_lat->sum_lat[idx][io] = 0;
			io_lat->peak_lat[idx][io] = 0;
			io_lat->bio_cnt[idx][io] = 0;
		}
	}
	spin_unlock_bh(&sbi->iostat_lat_lock);

	trace_f2fs_iostat_latency(sbi, iostat_lat);
}

static inline void f2fs_record_iostat(struct f2fs_sb_info *sbi)
{
	unsigned long long iostat_diff[NR_IO_TYPE];
	int i;

	if (time_is_after_jiffies(sbi->iostat_next_period))
		return;

	/* Need double check under the lock */
	spin_lock_bh(&sbi->iostat_lock);
	if (time_is_after_jiffies(sbi->iostat_next_period)) {
		spin_unlock_bh(&sbi->iostat_lock);
		return;
	}
	sbi->iostat_next_period = jiffies +
				msecs_to_jiffies(sbi->iostat_period_ms);

	for (i = 0; i < NR_IO_TYPE; i++) {
		iostat_diff[i] = sbi->rw_iostat[i] -
				sbi->prev_rw_iostat[i];
		sbi->prev_rw_iostat[i] = sbi->rw_iostat[i];
	}
	spin_unlock_bh(&sbi->iostat_lock);

	trace_f2fs_iostat(sbi, iostat_diff);

	__record_iostat_latency(sbi);
}

void f2fs_reset_iostat(struct f2fs_sb_info *sbi)
{
	struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
	int i;

	spin_lock_bh(&sbi->iostat_lock);
	for (i = 0; i < NR_IO_TYPE; i++) {
		sbi->rw_iostat[i] = 0;
		sbi->prev_rw_iostat[i] = 0;
	}
	spin_unlock_bh(&sbi->iostat_lock);

	spin_lock_bh(&sbi->iostat_lat_lock);
	memset(io_lat, 0, sizeof(struct iostat_lat_info));
	spin_unlock_bh(&sbi->iostat_lat_lock);
}

void f2fs_update_iostat(struct f2fs_sb_info *sbi,
			enum iostat_type type, unsigned long long io_bytes)
{
	if (!sbi->iostat_enable)
		return;

	spin_lock_bh(&sbi->iostat_lock);
	sbi->rw_iostat[type] += io_bytes;

	if (type == APP_BUFFERED_IO || type == APP_DIRECT_IO)
		sbi->rw_iostat[APP_WRITE_IO] += io_bytes;

	if (type == APP_BUFFERED_READ_IO || type == APP_DIRECT_READ_IO)
		sbi->rw_iostat[APP_READ_IO] += io_bytes;

	spin_unlock_bh(&sbi->iostat_lock);

	f2fs_record_iostat(sbi);
}

static inline void __update_iostat_latency(struct bio_iostat_ctx *iostat_ctx,
				int rw, bool is_sync)
{
	unsigned long ts_diff;
	unsigned int iotype = iostat_ctx->type;
	struct f2fs_sb_info *sbi = iostat_ctx->sbi;
	struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
	int idx;

	if (!sbi->iostat_enable)
		return;

	ts_diff = jiffies - iostat_ctx->submit_ts;
	if (iotype >= META_FLUSH)
		iotype = META;

	if (rw == 0) {
		idx = READ_IO;
	} else {
		if (is_sync)
			idx = WRITE_SYNC_IO;
		else
			idx = WRITE_ASYNC_IO;
	}

	spin_lock_bh(&sbi->iostat_lat_lock);
	io_lat->sum_lat[idx][iotype] += ts_diff;
	io_lat->bio_cnt[idx][iotype]++;
	if (ts_diff > io_lat->peak_lat[idx][iotype])
		io_lat->peak_lat[idx][iotype] = ts_diff;
	spin_unlock_bh(&sbi->iostat_lat_lock);
}

void iostat_update_and_unbind_ctx(struct bio *bio, int rw)
{
	struct bio_iostat_ctx *iostat_ctx = bio->bi_private;
	bool is_sync = bio->bi_opf & REQ_SYNC;

	if (rw == 0)
		bio->bi_private = iostat_ctx->post_read_ctx;
	else
		bio->bi_private = iostat_ctx->sbi;
	__update_iostat_latency(iostat_ctx, rw, is_sync);
	mempool_free(iostat_ctx, bio_iostat_ctx_pool);
}

void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi,
		struct bio *bio, struct bio_post_read_ctx *ctx)
{
	struct bio_iostat_ctx *iostat_ctx;
	/* Due to the mempool, this never fails. */
	iostat_ctx = mempool_alloc(bio_iostat_ctx_pool, GFP_NOFS);
	iostat_ctx->sbi = sbi;
	iostat_ctx->submit_ts = 0;
	iostat_ctx->type = 0;
	iostat_ctx->post_read_ctx = ctx;
	bio->bi_private = iostat_ctx;
}

int __init f2fs_init_iostat_processing(void)
{
	bio_iostat_ctx_cache =
		kmem_cache_create("f2fs_bio_iostat_ctx",
				  sizeof(struct bio_iostat_ctx), 0, 0, NULL);
	if (!bio_iostat_ctx_cache)
		goto fail;
	bio_iostat_ctx_pool =
		mempool_create_slab_pool(NUM_PREALLOC_IOSTAT_CTXS,
					 bio_iostat_ctx_cache);
	if (!bio_iostat_ctx_pool)
		goto fail_free_cache;
	return 0;

fail_free_cache:
	kmem_cache_destroy(bio_iostat_ctx_cache);
fail:
	return -ENOMEM;
}

void f2fs_destroy_iostat_processing(void)
{
	mempool_destroy(bio_iostat_ctx_pool);
	kmem_cache_destroy(bio_iostat_ctx_cache);
}

int f2fs_init_iostat(struct f2fs_sb_info *sbi)
{
	/* init iostat info */
	spin_lock_init(&sbi->iostat_lock);
	spin_lock_init(&sbi->iostat_lat_lock);
	sbi->iostat_enable = false;
	sbi->iostat_period_ms = DEFAULT_IOSTAT_PERIOD_MS;
	sbi->iostat_io_lat = f2fs_kzalloc(sbi, sizeof(struct iostat_lat_info),
					GFP_KERNEL);
	if (!sbi->iostat_io_lat)
		return -ENOMEM;

	return 0;
}

void f2fs_destroy_iostat(struct f2fs_sb_info *sbi)
{
	kfree(sbi->iostat_io_lat);
}
Commit	Line	Data
52118743 DJ	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* f2fs iostat support
	4	*
	5	* Copyright 2021 Google LLC
	6	* Author: Daeho Jeong <daehojeong@google.com>
	7	*/
	8
	9	#include <linux/fs.h>
	10	#include <linux/f2fs_fs.h>
	11	#include <linux/seq_file.h>
	12
	13	#include "f2fs.h"
	14	#include "iostat.h"
	15	#include <trace/events/f2fs.h>
	16
a4b68176 DJ	17	#define NUM_PREALLOC_IOSTAT_CTXS 128
	18	static struct kmem_cache *bio_iostat_ctx_cache;
	19	static mempool_t *bio_iostat_ctx_pool;
	20
52118743 DJ	21	int __maybe_unused iostat_info_seq_show(struct seq_file seq, void offset)
	22	{
	23	struct super_block *sb = seq->private;
	24	struct f2fs_sb_info *sbi = F2FS_SB(sb);
	25	time64_t now = ktime_get_real_seconds();
	26
	27	if (!sbi->iostat_enable)
	28	return 0;
	29
	30	seq_printf(seq, "time: %-16llu\n", now);
	31
	32	/* print app write IOs */
	33	seq_puts(seq, "[WRITE]\n");
	34	seq_printf(seq, "app buffered: %-16llu\n",
	35	sbi->rw_iostat[APP_BUFFERED_IO]);
	36	seq_printf(seq, "app direct: %-16llu\n",
	37	sbi->rw_iostat[APP_DIRECT_IO]);
	38	seq_printf(seq, "app mapped: %-16llu\n",
	39	sbi->rw_iostat[APP_MAPPED_IO]);
	40
	41	/* print fs write IOs */
	42	seq_printf(seq, "fs data: %-16llu\n",
	43	sbi->rw_iostat[FS_DATA_IO]);
	44	seq_printf(seq, "fs node: %-16llu\n",
	45	sbi->rw_iostat[FS_NODE_IO]);
	46	seq_printf(seq, "fs meta: %-16llu\n",
	47	sbi->rw_iostat[FS_META_IO]);
	48	seq_printf(seq, "fs gc data: %-16llu\n",
	49	sbi->rw_iostat[FS_GC_DATA_IO]);
	50	seq_printf(seq, "fs gc node: %-16llu\n",
	51	sbi->rw_iostat[FS_GC_NODE_IO]);
	52	seq_printf(seq, "fs cp data: %-16llu\n",
	53	sbi->rw_iostat[FS_CP_DATA_IO]);
	54	seq_printf(seq, "fs cp node: %-16llu\n",
	55	sbi->rw_iostat[FS_CP_NODE_IO]);
	56	seq_printf(seq, "fs cp meta: %-16llu\n",
	57	sbi->rw_iostat[FS_CP_META_IO]);
	58
	59	/* print app read IOs */
	60	seq_puts(seq, "[READ]\n");
	61	seq_printf(seq, "app buffered: %-16llu\n",
	62	sbi->rw_iostat[APP_BUFFERED_READ_IO]);
	63	seq_printf(seq, "app direct: %-16llu\n",
	64	sbi->rw_iostat[APP_DIRECT_READ_IO]);
	65	seq_printf(seq, "app mapped: %-16llu\n",
	66	sbi->rw_iostat[APP_MAPPED_READ_IO]);
	67
	68	/* print fs read IOs */
	69	seq_printf(seq, "fs data: %-16llu\n",
	70	sbi->rw_iostat[FS_DATA_READ_IO]);
	71	seq_printf(seq, "fs gc data: %-16llu\n",
	72	sbi->rw_iostat[FS_GDATA_READ_IO]);
	73	seq_printf(seq, "fs compr_data: %-16llu\n",
	74	sbi->rw_iostat[FS_CDATA_READ_IO]);
	75	seq_printf(seq, "fs node: %-16llu\n",
	76	sbi->rw_iostat[FS_NODE_READ_IO]);
	77	seq_printf(seq, "fs meta: %-16llu\n",
	78	sbi->rw_iostat[FS_META_READ_IO]);
	79
	80	/* print other IOs */
	81	seq_puts(seq, "[OTHER]\n");
	82	seq_printf(seq, "fs discard: %-16llu\n",
	83	sbi->rw_iostat[FS_DISCARD]);
	84
85	return 0;
86	}
87
a4b68176 DJ	88	static inline void __record_iostat_latency(struct f2fs_sb_info *sbi)
	89	{
	90	int io, idx = 0;
	91	unsigned int cnt;
	92	struct f2fs_iostat_latency iostat_lat[MAX_IO_TYPE][NR_PAGE_TYPE];
	93	struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
	94
a1e09b03	95	spin_lock_bh(&sbi->iostat_lat_lock);
a4b68176 DJ	96	for (idx = 0; idx < MAX_IO_TYPE; idx++) {
	97	for (io = 0; io < NR_PAGE_TYPE; io++) {
	98	cnt = io_lat->bio_cnt[idx][io];
	99	iostat_lat[idx][io].peak_lat =
	100	jiffies_to_msecs(io_lat->peak_lat[idx][io]);
	101	iostat_lat[idx][io].cnt = cnt;
	102	iostat_lat[idx][io].avg_lat = cnt ?
	103	jiffies_to_msecs(io_lat->sum_lat[idx][io]) / cnt : 0;
	104	io_lat->sum_lat[idx][io] = 0;
	105	io_lat->peak_lat[idx][io] = 0;
	106	io_lat->bio_cnt[idx][io] = 0;
	107	}
	108	}
a1e09b03	109	spin_unlock_bh(&sbi->iostat_lat_lock);
a4b68176 DJ	110
	111	trace_f2fs_iostat_latency(sbi, iostat_lat);
	112	}
	113
52118743 DJ	114	static inline void f2fs_record_iostat(struct f2fs_sb_info *sbi)
	115	{
	116	unsigned long long iostat_diff[NR_IO_TYPE];
	117	int i;
	118
	119	if (time_is_after_jiffies(sbi->iostat_next_period))
	120	return;
	121
	122	/* Need double check under the lock */
a1e09b03	123	spin_lock_bh(&sbi->iostat_lock);
52118743	124	if (time_is_after_jiffies(sbi->iostat_next_period)) {
a1e09b03	125	spin_unlock_bh(&sbi->iostat_lock);
52118743 DJ	126	return;
	127	}
	128	sbi->iostat_next_period = jiffies +
	129	msecs_to_jiffies(sbi->iostat_period_ms);
	130
	131	for (i = 0; i < NR_IO_TYPE; i++) {
	132	iostat_diff[i] = sbi->rw_iostat[i] -
	133	sbi->prev_rw_iostat[i];
	134	sbi->prev_rw_iostat[i] = sbi->rw_iostat[i];
	135	}
a1e09b03	136	spin_unlock_bh(&sbi->iostat_lock);
52118743 DJ	137
52118743 DJ	138	trace_f2fs_iostat(sbi, iostat_diff);
a4b68176 DJ	139
a4b68176 DJ	140	__record_iostat_latency(sbi);
52118743 DJ	141	}
	142
	143	void f2fs_reset_iostat(struct f2fs_sb_info *sbi)
	144	{
a4b68176	145	struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
52118743 DJ	146	int i;
52118743 DJ	147
a1e09b03	148	spin_lock_bh(&sbi->iostat_lock);
52118743 DJ	149	for (i = 0; i < NR_IO_TYPE; i++) {
	150	sbi->rw_iostat[i] = 0;
	151	sbi->prev_rw_iostat[i] = 0;
	152	}
a1e09b03	153	spin_unlock_bh(&sbi->iostat_lock);
a4b68176	154
a1e09b03	155	spin_lock_bh(&sbi->iostat_lat_lock);
a4b68176	156	memset(io_lat, 0, sizeof(struct iostat_lat_info));
a1e09b03	157	spin_unlock_bh(&sbi->iostat_lat_lock);
52118743 DJ	158	}
	159
	160	void f2fs_update_iostat(struct f2fs_sb_info *sbi,
	161	enum iostat_type type, unsigned long long io_bytes)
	162	{
	163	if (!sbi->iostat_enable)
	164	return;
	165
a1e09b03	166	spin_lock_bh(&sbi->iostat_lock);
52118743 DJ	167	sbi->rw_iostat[type] += io_bytes;
52118743 DJ	168
a1e09b03 EB	169	if (type == APP_BUFFERED_IO \|\| type == APP_DIRECT_IO)
a1e09b03 EB	170	sbi->rw_iostat[APP_WRITE_IO] += io_bytes;
52118743	171
a1e09b03 EB	172	if (type == APP_BUFFERED_READ_IO \|\| type == APP_DIRECT_READ_IO)
	173	sbi->rw_iostat[APP_READ_IO] += io_bytes;
	174
	175	spin_unlock_bh(&sbi->iostat_lock);
52118743 DJ	176
	177	f2fs_record_iostat(sbi);
	178	}
	179
a4b68176 DJ	180	static inline void __update_iostat_latency(struct bio_iostat_ctx *iostat_ctx,
	181	int rw, bool is_sync)
	182	{
	183	unsigned long ts_diff;
	184	unsigned int iotype = iostat_ctx->type;
a4b68176 DJ	185	struct f2fs_sb_info *sbi = iostat_ctx->sbi;
	186	struct iostat_lat_info *io_lat = sbi->iostat_io_lat;
	187	int idx;
	188
	189	if (!sbi->iostat_enable)
	190	return;
	191
	192	ts_diff = jiffies - iostat_ctx->submit_ts;
	193	if (iotype >= META_FLUSH)
	194	iotype = META;
	195
	196	if (rw == 0) {
	197	idx = READ_IO;
	198	} else {
	199	if (is_sync)
	200	idx = WRITE_SYNC_IO;
	201	else
	202	idx = WRITE_ASYNC_IO;
	203	}
	204
a1e09b03	205	spin_lock_bh(&sbi->iostat_lat_lock);
a4b68176 DJ	206	io_lat->sum_lat[idx][iotype] += ts_diff;
	207	io_lat->bio_cnt[idx][iotype]++;
	208	if (ts_diff > io_lat->peak_lat[idx][iotype])
	209	io_lat->peak_lat[idx][iotype] = ts_diff;
a1e09b03	210	spin_unlock_bh(&sbi->iostat_lat_lock);
a4b68176 DJ	211	}
	212
	213	void iostat_update_and_unbind_ctx(struct bio *bio, int rw)
	214	{
	215	struct bio_iostat_ctx *iostat_ctx = bio->bi_private;
	216	bool is_sync = bio->bi_opf & REQ_SYNC;
	217
	218	if (rw == 0)
	219	bio->bi_private = iostat_ctx->post_read_ctx;
	220	else
	221	bio->bi_private = iostat_ctx->sbi;
	222	__update_iostat_latency(iostat_ctx, rw, is_sync);
	223	mempool_free(iostat_ctx, bio_iostat_ctx_pool);
	224	}
	225
	226	void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi,
	227	struct bio bio, struct bio_post_read_ctx ctx)
	228	{
	229	struct bio_iostat_ctx *iostat_ctx;
	230	/* Due to the mempool, this never fails. */
	231	iostat_ctx = mempool_alloc(bio_iostat_ctx_pool, GFP_NOFS);
	232	iostat_ctx->sbi = sbi;
	233	iostat_ctx->submit_ts = 0;
	234	iostat_ctx->type = 0;
	235	iostat_ctx->post_read_ctx = ctx;
	236	bio->bi_private = iostat_ctx;
	237	}
	238
	239	int __init f2fs_init_iostat_processing(void)
	240	{
	241	bio_iostat_ctx_cache =
	242	kmem_cache_create("f2fs_bio_iostat_ctx",
	243	sizeof(struct bio_iostat_ctx), 0, 0, NULL);
	244	if (!bio_iostat_ctx_cache)
	245	goto fail;
	246	bio_iostat_ctx_pool =
	247	mempool_create_slab_pool(NUM_PREALLOC_IOSTAT_CTXS,
	248	bio_iostat_ctx_cache);
	249	if (!bio_iostat_ctx_pool)
	250	goto fail_free_cache;
	251	return 0;
	252
	253	fail_free_cache:
	254	kmem_cache_destroy(bio_iostat_ctx_cache);
	255	fail:
	256	return -ENOMEM;
	257	}
	258
	259	void f2fs_destroy_iostat_processing(void)
	260	{
	261	mempool_destroy(bio_iostat_ctx_pool);
	262	kmem_cache_destroy(bio_iostat_ctx_cache);
	263	}
	264
52118743 DJ	265	int f2fs_init_iostat(struct f2fs_sb_info *sbi)
	266	{
	267	/* init iostat info */
	268	spin_lock_init(&sbi->iostat_lock);
a4b68176	269	spin_lock_init(&sbi->iostat_lat_lock);
52118743 DJ	270	sbi->iostat_enable = false;
52118743 DJ	271	sbi->iostat_period_ms = DEFAULT_IOSTAT_PERIOD_MS;
a4b68176 DJ	272	sbi->iostat_io_lat = f2fs_kzalloc(sbi, sizeof(struct iostat_lat_info),
	273	GFP_KERNEL);
	274	if (!sbi->iostat_io_lat)
	275	return -ENOMEM;
52118743 DJ	276
	277	return 0;
	278	}
a4b68176 DJ	279
	280	void f2fs_destroy_iostat(struct f2fs_sb_info *sbi)
	281	{
	282	kfree(sbi->iostat_io_lat);
	283	}