[fio.git] / iolog.h

#ifndef FIO_IOLOG_H
#define FIO_IOLOG_H

#include <stdio.h>

#include "lib/rbtree.h"
#include "lib/ieee754.h"
#include "flist.h"
#include "ioengines.h"

/*
 * Use for maintaining statistics
 */
struct io_stat {
	uint64_t max_val;
	uint64_t min_val;
	uint64_t samples;

	fio_fp64_t mean;
	fio_fp64_t S;
};

struct io_hist {
	uint64_t samples;
	unsigned long hist_last;
	struct flist_head list;
};


union io_sample_data {
	uint64_t val;
	struct io_u_plat_entry *plat_entry;
};

#define sample_val(value) ((union io_sample_data) { .val = value })
#define sample_plat(plat) ((union io_sample_data) { .plat_entry = plat })

/*
 * A single data sample
 */
struct io_sample {
	uint64_t time;
	union io_sample_data data;
	uint32_t __ddir;
	uint16_t priority;
	uint64_t bs;
};

struct io_sample_offset {
	struct io_sample s;
	uint64_t offset;
};

enum {
	IO_LOG_TYPE_LAT = 1,
	IO_LOG_TYPE_CLAT,
	IO_LOG_TYPE_SLAT,
	IO_LOG_TYPE_BW,
	IO_LOG_TYPE_IOPS,
	IO_LOG_TYPE_HIST,
};

#define DEF_LOG_ENTRIES		1024
#define MAX_LOG_ENTRIES		(1024 * DEF_LOG_ENTRIES)

struct io_logs {
	struct flist_head list;
	uint64_t nr_samples;
	uint64_t max_samples;
	void *log;
};

/*
 * Dynamically growing data sample log
 */
struct io_log {
	/*
	 * Entries already logged
	 */
	struct flist_head io_logs;
	uint32_t cur_log_max;

	/*
	 * When the current log runs out of space, store events here until
	 * we have a chance to regrow
	 */
	struct io_logs *pending;

	unsigned int log_ddir_mask;

	char *filename;

	struct thread_data *td;

	unsigned int log_type;

	/*
	 * If we fail extending the log, stop collecting more entries.
	 */
	bool disabled;

	/*
	 * Log offsets
	 */
	unsigned int log_offset;

	/*
	 * Log I/O priorities
	 */
	unsigned int log_prio;

	/*
	 * Max size of log entries before a chunk is compressed
	 */
	unsigned int log_gz;

	/*
	 * Don't deflate for storing, just store the compressed bits
	 */
	unsigned int log_gz_store;

	/*
	 * Windowed average, for logging single entries average over some
	 * period of time.
	 */
	struct io_stat avg_window[DDIR_RWDIR_CNT];
	unsigned long avg_msec;
	unsigned long avg_last[DDIR_RWDIR_CNT];

	/*
	 * Windowed latency histograms, for keeping track of when we need to
	 * save a copy of the histogram every approximately hist_msec
	 * milliseconds.
	 */
	struct io_hist hist_window[DDIR_RWDIR_CNT];
	unsigned long hist_msec;
	unsigned int hist_coarseness;

	pthread_mutex_t chunk_lock;
	unsigned int chunk_seq;
	struct flist_head chunk_list;

	pthread_mutex_t deferred_free_lock;
#define IOLOG_MAX_DEFER	8
	void *deferred_items[IOLOG_MAX_DEFER];
	unsigned int deferred;
};

/*
 * If the upper bit is set, then we have the offset as well
 */
#define LOG_OFFSET_SAMPLE_BIT	0x80000000U
/*
 * If the bit following the upper bit is set, then we have the priority
 */
#define LOG_PRIO_SAMPLE_BIT	0x40000000U

#define LOG_SAMPLE_BITS		(LOG_OFFSET_SAMPLE_BIT | LOG_PRIO_SAMPLE_BIT)
#define io_sample_ddir(io)	((io)->__ddir & ~LOG_SAMPLE_BITS)

static inline void io_sample_set_ddir(struct io_log *log,
				      struct io_sample *io,
				      enum fio_ddir ddir)
{
	io->__ddir = ddir | log->log_ddir_mask;
}

static inline size_t __log_entry_sz(int log_offset)
{
	if (log_offset)
		return sizeof(struct io_sample_offset);
	else
		return sizeof(struct io_sample);
}

static inline size_t log_entry_sz(struct io_log *log)
{
	return __log_entry_sz(log->log_offset);
}

static inline size_t log_sample_sz(struct io_log *log, struct io_logs *cur_log)
{
	return cur_log->nr_samples * log_entry_sz(log);
}

static inline struct io_sample *__get_sample(void *samples, int log_offset,
					     uint64_t sample)
{
	uint64_t sample_offset = sample * __log_entry_sz(log_offset);
	return (struct io_sample *) ((char *) samples + sample_offset);
}

struct io_logs *iolog_cur_log(struct io_log *);
uint64_t iolog_nr_samples(struct io_log *);
void regrow_logs(struct thread_data *);
void regrow_agg_logs(void);

static inline struct io_sample *get_sample(struct io_log *iolog,
					   struct io_logs *cur_log,
					   uint64_t sample)
{
	return __get_sample(cur_log->log, iolog->log_offset, sample);
}

enum {
	IP_F_ONRB	= 1,
	IP_F_ONLIST	= 2,
	IP_F_TRIMMED	= 4,
	IP_F_IN_FLIGHT	= 8,
};

/*
 * When logging io actions, this matches a single sent io_u
 */
struct io_piece {
	union {
		struct fio_rb_node rb_node;
		struct flist_head list;
	};
	struct flist_head trim_list;
	union {
		int fileno;
		struct fio_file *file;
	};
	unsigned long long offset;
	unsigned short numberio;
	unsigned long len;
	unsigned int flags;
	enum fio_ddir ddir;
	union {
		unsigned long delay;
		unsigned int file_action;
	};
};

/*
 * Log exports
 */
enum file_log_act {
	FIO_LOG_ADD_FILE,
	FIO_LOG_OPEN_FILE,
	FIO_LOG_CLOSE_FILE,
	FIO_LOG_UNLINK_FILE,
};

struct io_u;
extern int __must_check read_iolog_get(struct thread_data *, struct io_u *);
extern void log_io_u(const struct thread_data *, const struct io_u *);
extern void log_file(struct thread_data *, struct fio_file *, enum file_log_act);
extern bool __must_check init_iolog(struct thread_data *td);
extern void log_io_piece(struct thread_data *, struct io_u *);
extern void unlog_io_piece(struct thread_data *, struct io_u *);
extern void trim_io_piece(const struct io_u *);
extern void queue_io_piece(struct thread_data *, struct io_piece *);
extern void prune_io_piece_log(struct thread_data *);
extern void write_iolog_close(struct thread_data *);
extern int iolog_compress_init(struct thread_data *, struct sk_out *);
extern void iolog_compress_exit(struct thread_data *);
extern size_t log_chunk_sizes(struct io_log *);
extern int init_io_u_buffers(struct thread_data *);

#ifdef CONFIG_ZLIB
extern int iolog_file_inflate(const char *);
#endif

/*
 * Logging
 */
struct log_params {
	struct thread_data *td;
	unsigned long avg_msec;
	unsigned long hist_msec;
	int hist_coarseness;
	int log_type;
	int log_offset;
	int log_prio;
	int log_gz;
	int log_gz_store;
	int log_compress;
};

static inline bool per_unit_log(struct io_log *log)
{
	return log && (!log->avg_msec || log->log_gz || log->log_gz_store);
}

static inline bool inline_log(struct io_log *log)
{
	return log->log_type == IO_LOG_TYPE_LAT ||
		log->log_type == IO_LOG_TYPE_CLAT ||
		log->log_type == IO_LOG_TYPE_SLAT;
}

static inline void ipo_bytes_align(unsigned int replay_align, struct io_piece *ipo)
{
	if (!replay_align)
		return;

	ipo->offset &= ~(replay_align - (uint64_t)1);
}

extern void finalize_logs(struct thread_data *td, bool);
extern void setup_log(struct io_log **, struct log_params *, const char *);
extern void flush_log(struct io_log *, bool);
extern void flush_samples(FILE *, void *, uint64_t);
extern uint64_t hist_sum(int, int, uint64_t *, uint64_t *);
extern void free_log(struct io_log *);
extern void fio_writeout_logs(bool);
extern void td_writeout_logs(struct thread_data *, bool);
extern int iolog_cur_flush(struct io_log *, struct io_logs *);

static inline void init_ipo(struct io_piece *ipo)
{
	INIT_FLIST_HEAD(&ipo->list);
	INIT_FLIST_HEAD(&ipo->trim_list);
}

struct iolog_compress {
	struct flist_head list;
	void *buf;
	size_t len;
	unsigned int seq;
};

#endif
Commit	Line	Data
5995a6a4 JA	1	#ifndef FIO_IOLOG_H
	2	#define FIO_IOLOG_H
	3
3d2d14bc SW	4	#include <stdio.h>
3d2d14bc SW	5
05775438	6	#include "lib/rbtree.h"
c7c6cb4c	7	#include "lib/ieee754.h"
836fcc0f	8	#include "flist.h"
618ee94c	9	#include "ioengines.h"
802ad4a8	10
5995a6a4 JA	11	/*
	12	* Use for maintaining statistics
	13	*/
	14	struct io_stat {
7b9f733a JA	15	uint64_t max_val;
	16	uint64_t min_val;
	17	uint64_t samples;
5995a6a4	18
802ad4a8 JA	19	fio_fp64_t mean;
802ad4a8 JA	20	fio_fp64_t S;
5995a6a4 JA	21	};
5995a6a4 JA	22
1e613c9c KC	23	struct io_hist {
	24	uint64_t samples;
	25	unsigned long hist_last;
d730bc58	26	struct flist_head list;
1e613c9c KC	27	};
1e613c9c KC	28
af2fde19 SW	29
	30	union io_sample_data {
	31	uint64_t val;
	32	struct io_u_plat_entry *plat_entry;
	33	};
	34
	35	#define sample_val(value) ((union io_sample_data) { .val = value })
	36	#define sample_plat(plat) ((union io_sample_data) { .plat_entry = plat })
	37
5995a6a4 JA	38	/*
	39	* A single data sample
	40	*/
	41	struct io_sample {
1b42725f	42	uint64_t time;
af2fde19	43	union io_sample_data data;
b26317c9	44	uint32_t __ddir;
03ec570f	45	uint16_t priority;
5fff9543	46	uint64_t bs;
5995a6a4 JA	47	};
5995a6a4 JA	48
ae588852 JA	49	struct io_sample_offset {
	50	struct io_sample s;
	51	uint64_t offset;
	52	};
	53
ea51b956 JA	54	enum {
	55	IO_LOG_TYPE_LAT = 1,
	56	IO_LOG_TYPE_CLAT,
	57	IO_LOG_TYPE_SLAT,
	58	IO_LOG_TYPE_BW,
	59	IO_LOG_TYPE_IOPS,
1e613c9c	60	IO_LOG_TYPE_HIST,
ea51b956 JA	61	};
ea51b956 JA	62
6d37f67f	63	#define DEF_LOG_ENTRIES 1024
7e419452 JA	64	#define MAX_LOG_ENTRIES (1024 * DEF_LOG_ENTRIES)
7e419452 JA	65
7e419452 JA	66	struct io_logs {
	67	struct flist_head list;
	68	uint64_t nr_samples;
	69	uint64_t max_samples;
	70	void *log;
	71	};
6d37f67f	72
5995a6a4 JA	73	/*
	74	* Dynamically growing data sample log
	75	*/
	76	struct io_log {
b8bc8cba JA	77	/*
	78	* Entries already logged
	79	*/
7e419452 JA	80	struct flist_head io_logs;
7e419452 JA	81	uint32_t cur_log_max;
b8bc8cba	82
1fed2080 JA	83	/*
	84	* When the current log runs out of space, store events here until
	85	* we have a chance to regrow
	86	*/
	87	struct io_logs *pending;
	88
b26317c9 JA	89	unsigned int log_ddir_mask;
b26317c9 JA	90
cb7e0ace JA	91	char *filename;
cb7e0ace JA	92
aee2ab67 JA	93	struct thread_data *td;
aee2ab67 JA	94
1b42725f	95	unsigned int log_type;
ea51b956	96
3c568239 JA	97	/*
	98	* If we fail extending the log, stop collecting more entries.
	99	*/
7e419452	100	bool disabled;
3c568239	101
ae588852 JA	102	/*
	103	* Log offsets
	104	*/
	105	unsigned int log_offset;
	106
03ec570f DLM	107	/*
	108	* Log I/O priorities
	109	*/
	110	unsigned int log_prio;
	111
aee2ab67 JA	112	/*
	113	* Max size of log entries before a chunk is compressed
	114	*/
	115	unsigned int log_gz;
	116
b26317c9 JA	117	/*
	118	* Don't deflate for storing, just store the compressed bits
	119	*/
	120	unsigned int log_gz_store;
	121
b8bc8cba JA	122	/*
	123	* Windowed average, for logging single entries average over some
	124	* period of time.
	125	*/
6eaf09d6	126	struct io_stat avg_window[DDIR_RWDIR_CNT];
b8bc8cba	127	unsigned long avg_msec;
8355119b	128	unsigned long avg_last[DDIR_RWDIR_CNT];
aee2ab67	129
8aa89d70 JA	130	/*
	131	* Windowed latency histograms, for keeping track of when we need to
	132	* save a copy of the histogram every approximately hist_msec
	133	* milliseconds.
	134	*/
1e613c9c KC	135	struct io_hist hist_window[DDIR_RWDIR_CNT];
1e613c9c KC	136	unsigned long hist_msec;
868f6f03	137	unsigned int hist_coarseness;
1e613c9c	138
aee2ab67 JA	139	pthread_mutex_t chunk_lock;
	140	unsigned int chunk_seq;
	141	struct flist_head chunk_list;
858bce70 JA	142
	143	pthread_mutex_t deferred_free_lock;
	144	#define IOLOG_MAX_DEFER 8
	145	void *deferred_items[IOLOG_MAX_DEFER];
	146	unsigned int deferred;
5995a6a4 JA	147	};
5995a6a4 JA	148
49e98daa JA	149	/*
	150	* If the upper bit is set, then we have the offset as well
	151	*/
	152	#define LOG_OFFSET_SAMPLE_BIT 0x80000000U
03ec570f DLM	153	/*
	154	* If the bit following the upper bit is set, then we have the priority
	155	*/
	156	#define LOG_PRIO_SAMPLE_BIT 0x40000000U
	157
	158	#define LOG_SAMPLE_BITS (LOG_OFFSET_SAMPLE_BIT \| LOG_PRIO_SAMPLE_BIT)
	159	#define io_sample_ddir(io) ((io)->__ddir & ~LOG_SAMPLE_BITS)
b26317c9 JA	160
	161	static inline void io_sample_set_ddir(struct io_log *log,
	162	struct io_sample *io,
	163	enum fio_ddir ddir)
	164	{
	165	io->__ddir = ddir \| log->log_ddir_mask;
	166	}
	167
ae588852 JA	168	static inline size_t __log_entry_sz(int log_offset)
	169	{
	170	if (log_offset)
	171	return sizeof(struct io_sample_offset);
	172	else
	173	return sizeof(struct io_sample);
	174	}
	175
	176	static inline size_t log_entry_sz(struct io_log *log)
	177	{
	178	return __log_entry_sz(log->log_offset);
	179	}
	180
868f6f03 KC	181	static inline size_t log_sample_sz(struct io_log log, struct io_logs cur_log)
	182	{
	183	return cur_log->nr_samples * log_entry_sz(log);
	184	}
	185
ae588852 JA	186	static inline struct io_sample __get_sample(void samples, int log_offset,
	187	uint64_t sample)
	188	{
6f9bbb55 CB	189	uint64_t sample_offset = sample * __log_entry_sz(log_offset);
6f9bbb55 CB	190	return (struct io_sample ) ((char ) samples + sample_offset);
ae588852 JA	191	}
ae588852 JA	192
7e419452 JA	193	struct io_logs iolog_cur_log(struct io_log );
7e419452 JA	194	uint64_t iolog_nr_samples(struct io_log *);
1fed2080	195	void regrow_logs(struct thread_data *);
76204de3	196	void regrow_agg_logs(void);
7e419452	197
ae588852	198	static inline struct io_sample get_sample(struct io_log iolog,
7e419452	199	struct io_logs *cur_log,
ae588852 JA	200	uint64_t sample)
ae588852 JA	201	{
7e419452	202	return __get_sample(cur_log->log, iolog->log_offset, sample);
ae588852 JA	203	}
ae588852 JA	204
0d29de83 JA	205	enum {
	206	IP_F_ONRB = 1,
	207	IP_F_ONLIST = 2,
	208	IP_F_TRIMMED = 4,
f9401285	209	IP_F_IN_FLIGHT = 8,
0d29de83 JA	210	};
0d29de83 JA	211
5995a6a4 JA	212	/*
	213	* When logging io actions, this matches a single sent io_u
	214	*/
	215	struct io_piece {
	216	union {
e96c0125	217	struct fio_rb_node rb_node;
5995a6a4 JA	218	struct flist_head list;
5995a6a4 JA	219	};
0d29de83	220	struct flist_head trim_list;
5995a6a4 JA	221	union {
	222	int fileno;
	223	struct fio_file *file;
	224	};
	225	unsigned long long offset;
da0a7bd2	226	unsigned short numberio;
5995a6a4	227	unsigned long len;
a917a8b3	228	unsigned int flags;
5995a6a4 JA	229	enum fio_ddir ddir;
	230	union {
	231	unsigned long delay;
	232	unsigned int file_action;
	233	};
	234	};
	235
	236	/*
	237	* Log exports
	238	*/
	239	enum file_log_act {
	240	FIO_LOG_ADD_FILE,
	241	FIO_LOG_OPEN_FILE,
	242	FIO_LOG_CLOSE_FILE,
	243	FIO_LOG_UNLINK_FILE,
	244	};
	245
8062f527	246	struct io_u;
5995a6a4	247	extern int __must_check read_iolog_get(struct thread_data , struct io_u );
1f440ece	248	extern void log_io_u(const struct thread_data , const struct io_u );
5995a6a4	249	extern void log_file(struct thread_data , struct fio_file , enum file_log_act);
b153f94a	250	extern bool __must_check init_iolog(struct thread_data *td);
5995a6a4	251	extern void log_io_piece(struct thread_data , struct io_u );
890b6656	252	extern void unlog_io_piece(struct thread_data , struct io_u );
94a78d52	253	extern void trim_io_piece(const struct io_u *);
5995a6a4 JA	254	extern void queue_io_piece(struct thread_data , struct io_piece );
	255	extern void prune_io_piece_log(struct thread_data *);
	256	extern void write_iolog_close(struct thread_data *);
24660963	257	extern int iolog_compress_init(struct thread_data , struct sk_out );
155f2f02	258	extern void iolog_compress_exit(struct thread_data *);
0cba0f91	259	extern size_t log_chunk_sizes(struct io_log *);
71e6e5a2	260	extern int init_io_u_buffers(struct thread_data *);
5995a6a4	261
b26317c9 JA	262	#ifdef CONFIG_ZLIB
	263	extern int iolog_file_inflate(const char *);
	264	#endif
	265
5995a6a4 JA	266	/*
	267	* Logging
	268	*/
aee2ab67 JA	269	struct log_params {
	270	struct thread_data *td;
	271	unsigned long avg_msec;
1e613c9c KC	272	unsigned long hist_msec;
1e613c9c KC	273	int hist_coarseness;
aee2ab67 JA	274	int log_type;
aee2ab67 JA	275	int log_offset;
03ec570f	276	int log_prio;
aee2ab67	277	int log_gz;
b26317c9	278	int log_gz_store;
aee2ab67 JA	279	int log_compress;
	280	};
	281
a47591e4 JA	282	static inline bool per_unit_log(struct io_log *log)
a47591e4 JA	283	{
83636545	284	return log && (!log->avg_msec \|\| log->log_gz \|\| log->log_gz_store);
a47591e4 JA	285	}
a47591e4 JA	286
b392f36d JA	287	static inline bool inline_log(struct io_log *log)
	288	{
	289	return log->log_type == IO_LOG_TYPE_LAT \|\|
	290	log->log_type == IO_LOG_TYPE_CLAT \|\|
	291	log->log_type == IO_LOG_TYPE_SLAT;
	292	}
	293
a79f17bf SW	294	static inline void ipo_bytes_align(unsigned int replay_align, struct io_piece *ipo)
a79f17bf SW	295	{
3dd2e8aa	296	if (!replay_align)
a79f17bf SW	297	return;
	298
	299	ipo->offset &= ~(replay_align - (uint64_t)1);
	300	}
	301
a47591e4	302	extern void finalize_logs(struct thread_data *td, bool);
aee2ab67	303	extern void setup_log(struct io_log *, struct log_params , const char *);
60a25727	304	extern void flush_log(struct io_log *, bool);
bead01d7	305	extern void flush_samples(FILE , void , uint64_t);
6cc0e5aa	306	extern uint64_t hist_sum(int, int, uint64_t , uint64_t );
518dac09	307	extern void free_log(struct io_log *);
a47591e4 JA	308	extern void fio_writeout_logs(bool);
a47591e4 JA	309	extern void td_writeout_logs(struct thread_data *, bool);
7e419452	310	extern int iolog_cur_flush(struct io_log , struct io_logs );
5995a6a4	311
0d29de83 JA	312	static inline void init_ipo(struct io_piece *ipo)
0d29de83 JA	313	{
8812d7f2	314	INIT_FLIST_HEAD(&ipo->list);
0d29de83 JA	315	INIT_FLIST_HEAD(&ipo->trim_list);
	316	}
	317
0cba0f91 JA	318	struct iolog_compress {
	319	struct flist_head list;
	320	void *buf;
	321	size_t len;
	322	unsigned int seq;
	323	};
	324
5995a6a4	325	#endif