[fio.git] / iolog.h

#ifndef FIO_IOLOG_H
#define FIO_IOLOG_H

#include <stdio.h>

#include "lib/rbtree.h"
#include "lib/ieee754.h"
#include "flist.h"
#include "ioengines.h"

/*
 * Use for maintaining statistics
 */
struct io_stat {
	uint64_t max_val;
	uint64_t min_val;
	uint64_t samples;

	fio_fp64_t mean;
	fio_fp64_t S;
};

struct io_hist {
	uint64_t samples;
	unsigned long hist_last;
	struct flist_head list;
};

enum {
	IO_LOG_SAMPLE_AVG = 0,
	IO_LOG_SAMPLE_MAX,
	IO_LOG_SAMPLE_BOTH,
};

struct io_sample_value {
	uint64_t val0;
	uint64_t val1;
};

union io_sample_data {
	struct io_sample_value val;
	struct io_u_plat_entry *plat_entry;
};

#define sample_val(value) ((union io_sample_data) { .val.val0 = value })
#define sample_plat(plat) ((union io_sample_data) { .plat_entry = plat })

/*
 * A single data sample
 */
struct io_sample {
	uint64_t time;
	union io_sample_data data;
	uint32_t __ddir;
	uint16_t priority;
	uint64_t bs;
	uint64_t aux[];
};

/*
 * Enumerate indexes of auxiliary log data in struct io_sample aux[] array
 */
enum {
	IOS_AUX_OFFSET_INDEX,
	IOS_AUX_ISSUE_TIME_INDEX,
};

enum {
	IO_LOG_TYPE_LAT = 1,
	IO_LOG_TYPE_CLAT,
	IO_LOG_TYPE_SLAT,
	IO_LOG_TYPE_BW,
	IO_LOG_TYPE_IOPS,
	IO_LOG_TYPE_HIST,
};

#define DEF_LOG_ENTRIES		1024
#define MAX_LOG_ENTRIES		(1024 * DEF_LOG_ENTRIES)

struct io_logs {
	struct flist_head list;
	uint64_t nr_samples;
	uint64_t max_samples;
	void *log;
};

/*
 * Dynamically growing data sample log
 */
struct io_log {
	/*
	 * Entries already logged
	 */
	struct flist_head io_logs;
	uint32_t cur_log_max;

	/*
	 * When the current log runs out of space, store events here until
	 * we have a chance to regrow
	 */
	struct io_logs *pending;

	unsigned int log_ddir_mask;

	char *filename;

	struct thread_data *td;

	unsigned int log_type;

	/*
	 * If we fail extending the log, stop collecting more entries.
	 */
	bool disabled;

	/*
	 * Log offsets
	 */
	unsigned int log_offset;

	/*
	 * Log I/O priorities
	 */
	unsigned int log_prio;

	/*
	 * Log I/O issuing time
	 */
	unsigned int log_issue_time;

	/*
	 * Max size of log entries before a chunk is compressed
	 */
	unsigned int log_gz;

	/*
	 * Don't deflate for storing, just store the compressed bits
	 */
	unsigned int log_gz_store;

	/*
	 * Windowed average, for logging single entries average over some
	 * period of time.
	 */
	struct io_stat avg_window[DDIR_RWDIR_CNT];
	unsigned long avg_msec;
	unsigned long avg_last[DDIR_RWDIR_CNT];

	/*
	 * Windowed latency histograms, for keeping track of when we need to
	 * save a copy of the histogram every approximately hist_msec
	 * milliseconds.
	 */
	struct io_hist hist_window[DDIR_RWDIR_CNT];
	unsigned long hist_msec;
	unsigned int hist_coarseness;

	pthread_mutex_t chunk_lock;
	unsigned int chunk_seq;
	struct flist_head chunk_list;

	pthread_mutex_t deferred_free_lock;
#define IOLOG_MAX_DEFER	8
	void *deferred_items[IOLOG_MAX_DEFER];
	unsigned int deferred;
};

/*
 * If the upper bit is set, then we have the offset as well
 */
#define LOG_OFFSET_SAMPLE_BIT	0x80000000U
/*
 * If the bit following the upper bit is set, then we have the priority
 */
#define LOG_PRIO_SAMPLE_BIT	0x40000000U
/*
 * If the bit following prioity sample vit is set, we report both avg and max
 */
#define LOG_AVG_MAX_SAMPLE_BIT	0x20000000U
/*
 * If the bit following AVG_MAX_SAMPLE_BIT is set, we report the issue time also
 */
#define LOG_ISSUE_TIME_SAMPLE_BIT	0x10000000U

#define LOG_SAMPLE_BITS		(LOG_OFFSET_SAMPLE_BIT | LOG_PRIO_SAMPLE_BIT |\
					LOG_AVG_MAX_SAMPLE_BIT |\
					LOG_ISSUE_TIME_SAMPLE_BIT)
#define io_sample_ddir(io)	((io)->__ddir & ~LOG_SAMPLE_BITS)

static inline void io_sample_set_ddir(struct io_log *log,
				      struct io_sample *io,
				      enum fio_ddir ddir)
{
	io->__ddir = ddir | log->log_ddir_mask;
}

static inline size_t __log_entry_sz(bool log_offset, bool log_issue_time)
{
	size_t ret = sizeof(struct io_sample);

	if (log_offset)
		ret += sizeof(uint64_t);

	if (log_issue_time)
		ret += sizeof(uint64_t);

	return ret;
}

static inline size_t log_entry_sz(struct io_log *log)
{
	return __log_entry_sz(log->log_offset, log->log_issue_time);
}

static inline size_t log_sample_sz(struct io_log *log, struct io_logs *cur_log)
{
	return cur_log->nr_samples * log_entry_sz(log);
}

static inline struct io_sample *__get_sample(void *samples, bool log_offset,
					     bool log_issue_time,
					     uint64_t sample)
{
	uint64_t sample_offset = sample *
		__log_entry_sz(log_offset, log_issue_time);
	return (struct io_sample *) ((char *) samples + sample_offset);
}

struct io_logs *iolog_cur_log(struct io_log *);
uint64_t iolog_nr_samples(struct io_log *);
void regrow_logs(struct thread_data *);
void regrow_agg_logs(void);

static inline struct io_sample *get_sample(struct io_log *iolog,
					   struct io_logs *cur_log,
					   uint64_t sample)
{
	return __get_sample(cur_log->log,
			    iolog->log_offset, iolog->log_issue_time, sample);
}

enum {
	IP_F_ONRB	= 1,
	IP_F_ONLIST	= 2,
	IP_F_TRIMMED	= 4,
	IP_F_IN_FLIGHT	= 8,
};

/*
 * When logging io actions, this matches a single sent io_u
 */
struct io_piece {
	union {
		struct fio_rb_node rb_node;
		struct flist_head list;
	};
	struct flist_head trim_list;
	union {
		int fileno;
		struct fio_file *file;
	};
	unsigned long long offset;
	unsigned short numberio;
	unsigned long len;
	unsigned int flags;
	enum fio_ddir ddir;
	unsigned long delay;
	unsigned int file_action;
};

/*
 * Log exports
 */
enum file_log_act {
	FIO_LOG_ADD_FILE,
	FIO_LOG_OPEN_FILE,
	FIO_LOG_CLOSE_FILE,
	FIO_LOG_UNLINK_FILE,
};

struct io_u;
extern int __must_check read_iolog_get(struct thread_data *, struct io_u *);
extern void log_io_u(const struct thread_data *, const struct io_u *);
extern void log_file(struct thread_data *, struct fio_file *, enum file_log_act);
extern bool __must_check init_iolog(struct thread_data *td);
extern void log_io_piece(struct thread_data *, struct io_u *);
extern void unlog_io_piece(struct thread_data *, struct io_u *);
extern void trim_io_piece(const struct io_u *);
extern void queue_io_piece(struct thread_data *, struct io_piece *);
extern void prune_io_piece_log(struct thread_data *);
extern void write_iolog_close(struct thread_data *);
int64_t iolog_items_to_fetch(struct thread_data *td);
extern int iolog_compress_init(struct thread_data *, struct sk_out *);
extern void iolog_compress_exit(struct thread_data *);
extern size_t log_chunk_sizes(struct io_log *);
extern int init_io_u_buffers(struct thread_data *);
extern unsigned long long delay_since_ttime(const struct thread_data *,
					     unsigned long long);

#ifdef CONFIG_ZLIB
extern int iolog_file_inflate(const char *);
#endif

/*
 * Logging
 */
struct log_params {
	struct thread_data *td;
	unsigned long avg_msec;
	unsigned long hist_msec;
	int hist_coarseness;
	int log_type;
	int log_offset;
	int log_prio;
	int log_issue_time;
	int log_gz;
	int log_gz_store;
	int log_compress;
};

static inline bool per_unit_log(struct io_log *log)
{
	return log && (!log->avg_msec || log->log_gz || log->log_gz_store);
}

static inline bool inline_log(struct io_log *log)
{
	return log->log_type == IO_LOG_TYPE_LAT ||
		log->log_type == IO_LOG_TYPE_CLAT ||
		log->log_type == IO_LOG_TYPE_SLAT;
}

static inline void ipo_bytes_align(unsigned int replay_align, struct io_piece *ipo)
{
	if (!replay_align)
		return;

	ipo->offset &= ~(replay_align - (uint64_t)1);
}

extern void finalize_logs(struct thread_data *td, bool);
extern void setup_log(struct io_log **, struct log_params *, const char *);
extern void flush_log(struct io_log *, bool);
extern void flush_samples(FILE *, void *, uint64_t);
extern uint64_t hist_sum(int, int, uint64_t *, uint64_t *);
extern void free_log(struct io_log *);
extern void fio_writeout_logs(bool);
extern void td_writeout_logs(struct thread_data *, bool);
extern int iolog_cur_flush(struct io_log *, struct io_logs *);

static inline void init_ipo(struct io_piece *ipo)
{
	INIT_FLIST_HEAD(&ipo->list);
	INIT_FLIST_HEAD(&ipo->trim_list);
}

struct iolog_compress {
	struct flist_head list;
	void *buf;
	size_t len;
	unsigned int seq;
};

#endif
Commit	Line	Data
	1	#ifndef FIO_IOLOG_H
	2	#define FIO_IOLOG_H
	3
	4	#include <stdio.h>
	5
	6	#include "lib/rbtree.h"
	7	#include "lib/ieee754.h"
	8	#include "flist.h"
	9	#include "ioengines.h"
	10
	11	/*
	12	* Use for maintaining statistics
	13	*/
	14	struct io_stat {
	15	uint64_t max_val;
	16	uint64_t min_val;
	17	uint64_t samples;
	18
	19	fio_fp64_t mean;
	20	fio_fp64_t S;
	21	};
	22
	23	struct io_hist {
	24	uint64_t samples;
	25	unsigned long hist_last;
	26	struct flist_head list;
	27	};
	28
	29	enum {
	30	IO_LOG_SAMPLE_AVG = 0,
	31	IO_LOG_SAMPLE_MAX,
	32	IO_LOG_SAMPLE_BOTH,
	33	};
	34
	35	struct io_sample_value {
	36	uint64_t val0;
	37	uint64_t val1;
	38	};
	39
	40	union io_sample_data {
	41	struct io_sample_value val;
	42	struct io_u_plat_entry *plat_entry;
	43	};
	44
	45	#define sample_val(value) ((union io_sample_data) { .val.val0 = value })
	46	#define sample_plat(plat) ((union io_sample_data) { .plat_entry = plat })
	47
	48	/*
	49	* A single data sample
	50	*/
	51	struct io_sample {
	52	uint64_t time;
	53	union io_sample_data data;
	54	uint32_t __ddir;
	55	uint16_t priority;
	56	uint64_t bs;
	57	uint64_t aux[];
	58	};
	59
	60	/*
	61	* Enumerate indexes of auxiliary log data in struct io_sample aux[] array
	62	*/
	63	enum {
	64	IOS_AUX_OFFSET_INDEX,
	65	IOS_AUX_ISSUE_TIME_INDEX,
	66	};
	67
	68	enum {
	69	IO_LOG_TYPE_LAT = 1,
	70	IO_LOG_TYPE_CLAT,
	71	IO_LOG_TYPE_SLAT,
	72	IO_LOG_TYPE_BW,
	73	IO_LOG_TYPE_IOPS,
	74	IO_LOG_TYPE_HIST,
	75	};
	76
	77	#define DEF_LOG_ENTRIES 1024
	78	#define MAX_LOG_ENTRIES (1024 * DEF_LOG_ENTRIES)
	79
	80	struct io_logs {
	81	struct flist_head list;
	82	uint64_t nr_samples;
	83	uint64_t max_samples;
	84	void *log;
	85	};
	86
	87	/*
	88	* Dynamically growing data sample log
	89	*/
	90	struct io_log {
	91	/*
	92	* Entries already logged
	93	*/
	94	struct flist_head io_logs;
	95	uint32_t cur_log_max;
	96
	97	/*
	98	* When the current log runs out of space, store events here until
	99	* we have a chance to regrow
	100	*/
	101	struct io_logs *pending;
	102
	103	unsigned int log_ddir_mask;
	104
	105	char *filename;
	106
	107	struct thread_data *td;
	108
	109	unsigned int log_type;
	110
	111	/*
	112	* If we fail extending the log, stop collecting more entries.
	113	*/
	114	bool disabled;
	115
	116	/*
	117	* Log offsets
	118	*/
	119	unsigned int log_offset;
	120
	121	/*
	122	* Log I/O priorities
	123	*/
	124	unsigned int log_prio;
	125
	126	/*
	127	* Log I/O issuing time
	128	*/
	129	unsigned int log_issue_time;
	130
	131	/*
	132	* Max size of log entries before a chunk is compressed
	133	*/
	134	unsigned int log_gz;
	135
	136	/*
	137	* Don't deflate for storing, just store the compressed bits
	138	*/
	139	unsigned int log_gz_store;
	140
	141	/*
	142	* Windowed average, for logging single entries average over some
	143	* period of time.
	144	*/
	145	struct io_stat avg_window[DDIR_RWDIR_CNT];
	146	unsigned long avg_msec;
	147	unsigned long avg_last[DDIR_RWDIR_CNT];
	148
	149	/*
	150	* Windowed latency histograms, for keeping track of when we need to
	151	* save a copy of the histogram every approximately hist_msec
	152	* milliseconds.
	153	*/
	154	struct io_hist hist_window[DDIR_RWDIR_CNT];
	155	unsigned long hist_msec;
	156	unsigned int hist_coarseness;
	157
	158	pthread_mutex_t chunk_lock;
	159	unsigned int chunk_seq;
	160	struct flist_head chunk_list;
	161
	162	pthread_mutex_t deferred_free_lock;
	163	#define IOLOG_MAX_DEFER 8
	164	void *deferred_items[IOLOG_MAX_DEFER];
	165	unsigned int deferred;
	166	};
	167
	168	/*
	169	* If the upper bit is set, then we have the offset as well
	170	*/
	171	#define LOG_OFFSET_SAMPLE_BIT 0x80000000U
	172	/*
	173	* If the bit following the upper bit is set, then we have the priority
	174	*/
	175	#define LOG_PRIO_SAMPLE_BIT 0x40000000U
	176	/*
	177	* If the bit following prioity sample vit is set, we report both avg and max
	178	*/
	179	#define LOG_AVG_MAX_SAMPLE_BIT 0x20000000U
	180	/*
	181	* If the bit following AVG_MAX_SAMPLE_BIT is set, we report the issue time also
	182	*/
	183	#define LOG_ISSUE_TIME_SAMPLE_BIT 0x10000000U
	184
	185	#define LOG_SAMPLE_BITS (LOG_OFFSET_SAMPLE_BIT \| LOG_PRIO_SAMPLE_BIT \|\
	186	LOG_AVG_MAX_SAMPLE_BIT \|\
	187	LOG_ISSUE_TIME_SAMPLE_BIT)
	188	#define io_sample_ddir(io) ((io)->__ddir & ~LOG_SAMPLE_BITS)
	189
	190	static inline void io_sample_set_ddir(struct io_log *log,
	191	struct io_sample *io,
	192	enum fio_ddir ddir)
	193	{
	194	io->__ddir = ddir \| log->log_ddir_mask;
	195	}
	196
	197	static inline size_t __log_entry_sz(bool log_offset, bool log_issue_time)
	198	{
	199	size_t ret = sizeof(struct io_sample);
	200
	201	if (log_offset)
	202	ret += sizeof(uint64_t);
	203
	204	if (log_issue_time)
	205	ret += sizeof(uint64_t);
	206
	207	return ret;
	208	}
	209
	210	static inline size_t log_entry_sz(struct io_log *log)
	211	{
	212	return __log_entry_sz(log->log_offset, log->log_issue_time);
	213	}
	214
	215	static inline size_t log_sample_sz(struct io_log log, struct io_logs cur_log)
	216	{
	217	return cur_log->nr_samples * log_entry_sz(log);
	218	}
	219
	220	static inline struct io_sample __get_sample(void samples, bool log_offset,
	221	bool log_issue_time,
	222	uint64_t sample)
	223	{
	224	uint64_t sample_offset = sample *
	225	__log_entry_sz(log_offset, log_issue_time);
	226	return (struct io_sample ) ((char ) samples + sample_offset);
	227	}
	228
	229	struct io_logs iolog_cur_log(struct io_log );
	230	uint64_t iolog_nr_samples(struct io_log *);
	231	void regrow_logs(struct thread_data *);
	232	void regrow_agg_logs(void);
	233
	234	static inline struct io_sample get_sample(struct io_log iolog,
	235	struct io_logs *cur_log,
	236	uint64_t sample)
	237	{
	238	return __get_sample(cur_log->log,
	239	iolog->log_offset, iolog->log_issue_time, sample);
	240	}
	241
	242	enum {
	243	IP_F_ONRB = 1,
	244	IP_F_ONLIST = 2,
	245	IP_F_TRIMMED = 4,
	246	IP_F_IN_FLIGHT = 8,
	247	};
	248
	249	/*
	250	* When logging io actions, this matches a single sent io_u
	251	*/
	252	struct io_piece {
	253	union {
	254	struct fio_rb_node rb_node;
	255	struct flist_head list;
	256	};
	257	struct flist_head trim_list;
	258	union {
	259	int fileno;
	260	struct fio_file *file;
	261	};
	262	unsigned long long offset;
	263	unsigned short numberio;
	264	unsigned long len;
	265	unsigned int flags;
	266	enum fio_ddir ddir;
	267	unsigned long delay;
	268	unsigned int file_action;
	269	};
	270
	271	/*
	272	* Log exports
	273	*/
	274	enum file_log_act {
	275	FIO_LOG_ADD_FILE,
	276	FIO_LOG_OPEN_FILE,
	277	FIO_LOG_CLOSE_FILE,
	278	FIO_LOG_UNLINK_FILE,
	279	};
	280
	281	struct io_u;
	282	extern int __must_check read_iolog_get(struct thread_data , struct io_u );
	283	extern void log_io_u(const struct thread_data , const struct io_u );
	284	extern void log_file(struct thread_data , struct fio_file , enum file_log_act);
	285	extern bool __must_check init_iolog(struct thread_data *td);
	286	extern void log_io_piece(struct thread_data , struct io_u );
	287	extern void unlog_io_piece(struct thread_data , struct io_u );
	288	extern void trim_io_piece(const struct io_u *);
	289	extern void queue_io_piece(struct thread_data , struct io_piece );
	290	extern void prune_io_piece_log(struct thread_data *);
	291	extern void write_iolog_close(struct thread_data *);
	292	int64_t iolog_items_to_fetch(struct thread_data *td);
	293	extern int iolog_compress_init(struct thread_data , struct sk_out );
	294	extern void iolog_compress_exit(struct thread_data *);
	295	extern size_t log_chunk_sizes(struct io_log *);
	296	extern int init_io_u_buffers(struct thread_data *);
	297	extern unsigned long long delay_since_ttime(const struct thread_data *,
	298	unsigned long long);
	299
	300	#ifdef CONFIG_ZLIB
	301	extern int iolog_file_inflate(const char *);
	302	#endif
	303
	304	/*
	305	* Logging
	306	*/
	307	struct log_params {
	308	struct thread_data *td;
	309	unsigned long avg_msec;
	310	unsigned long hist_msec;
	311	int hist_coarseness;
	312	int log_type;
	313	int log_offset;
	314	int log_prio;
	315	int log_issue_time;
	316	int log_gz;
	317	int log_gz_store;
	318	int log_compress;
	319	};
	320
	321	static inline bool per_unit_log(struct io_log *log)
	322	{
	323	return log && (!log->avg_msec \|\| log->log_gz \|\| log->log_gz_store);
	324	}
	325
	326	static inline bool inline_log(struct io_log *log)
	327	{
	328	return log->log_type == IO_LOG_TYPE_LAT \|\|
	329	log->log_type == IO_LOG_TYPE_CLAT \|\|
	330	log->log_type == IO_LOG_TYPE_SLAT;
	331	}
	332
	333	static inline void ipo_bytes_align(unsigned int replay_align, struct io_piece *ipo)
	334	{
	335	if (!replay_align)
	336	return;
	337
	338	ipo->offset &= ~(replay_align - (uint64_t)1);
	339	}
	340
	341	extern void finalize_logs(struct thread_data *td, bool);
	342	extern void setup_log(struct io_log *, struct log_params , const char *);
	343	extern void flush_log(struct io_log *, bool);
	344	extern void flush_samples(FILE , void , uint64_t);
	345	extern uint64_t hist_sum(int, int, uint64_t , uint64_t );
	346	extern void free_log(struct io_log *);
	347	extern void fio_writeout_logs(bool);
	348	extern void td_writeout_logs(struct thread_data *, bool);
	349	extern int iolog_cur_flush(struct io_log , struct io_logs );
	350
	351	static inline void init_ipo(struct io_piece *ipo)
	352	{
	353	INIT_FLIST_HEAD(&ipo->list);
	354	INIT_FLIST_HEAD(&ipo->trim_list);
	355	}
	356
	357	struct iolog_compress {
	358	struct flist_head list;
	359	void *buf;
	360	size_t len;
	361	unsigned int seq;
	362	};
	363
	364	#endif