[fio.git] / t / io_uring.c

#include <stdio.h>
#include <errno.h>
#include <assert.h>
#include <stdlib.h>
#include <stddef.h>
#include <signal.h>
#include <inttypes.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <sys/syscall.h>
#include <sys/resource.h>
#include <sys/mman.h>
#include <sys/uio.h>
#include <linux/fs.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <pthread.h>
#include <sched.h>

#include "../arch/arch.h"
#include "../lib/types.h"
#include "../os/io_uring.h"

#define barrier()	__asm__ __volatile__("": : :"memory")

#define min(a, b)		((a < b) ? (a) : (b))

struct io_sq_ring {
	unsigned *head;
	unsigned *tail;
	unsigned *ring_mask;
	unsigned *ring_entries;
	unsigned *array;
};

struct io_cq_ring {
	unsigned *head;
	unsigned *tail;
	unsigned *ring_mask;
	unsigned *ring_entries;
	struct io_uring_cqe *cqes;
};

#define DEPTH			128

#define BATCH_SUBMIT		64
#define BATCH_COMPLETE		64

#define BS			4096

#define MAX_FDS			16

static unsigned sq_ring_mask, cq_ring_mask;

struct file {
	unsigned long max_blocks;
	unsigned pending_ios;
	int fd;
};

struct submitter {
	pthread_t thread;
	int ring_fd;
	struct drand48_data rand;
	struct io_sq_ring sq_ring;
	struct io_uring_sqe *sqes;
	struct iovec iovecs[DEPTH];
	struct io_cq_ring cq_ring;
	int inflight;
	unsigned long reaps;
	unsigned long done;
	unsigned long calls;
	unsigned long cachehit, cachemiss;
	volatile int finish;

	struct file files[MAX_FDS];
	unsigned nr_files;
	unsigned cur_file;
};

static struct submitter submitters[1];
static volatile int finish;

static int polled = 1;		/* use IO polling */
static int fixedbufs = 1;	/* use fixed user buffers */
static int buffered = 0;	/* use buffered IO, not O_DIRECT */
static int sq_thread_poll = 0;	/* use kernel submission/poller thread */
static int sq_thread_cpu = -1;	/* pin above thread to this CPU */

static int io_uring_register_buffers(struct submitter *s)
{
	struct io_uring_register_buffers reg = {
		.iovecs = s->iovecs,
		.nr_iovecs = DEPTH
	};

	return syscall(__NR_sys_io_uring_register, s->ring_fd,
			IORING_REGISTER_BUFFERS, &reg);
}

static int io_uring_setup(unsigned entries, struct io_uring_params *p)
{
	return syscall(__NR_sys_io_uring_setup, entries, p);
}

static int io_uring_enter(struct submitter *s, unsigned int to_submit,
			  unsigned int min_complete, unsigned int flags)
{
	return syscall(__NR_sys_io_uring_enter, s->ring_fd, to_submit,
			min_complete, flags);
}

static int gettid(void)
{
	return syscall(__NR_gettid);
}

static unsigned file_depth(struct submitter *s)
{
	return (DEPTH + s->nr_files - 1) / s->nr_files;
}

static void init_io(struct submitter *s, unsigned index)
{
	struct io_uring_sqe *sqe = &s->sqes[index];
	unsigned long offset;
	struct file *f;
	long r;

	if (s->nr_files == 1) {
		f = &s->files[0];
	} else {
		f = &s->files[s->cur_file];
		if (f->pending_ios >= file_depth(s)) {
			s->cur_file++;
			if (s->cur_file == s->nr_files)
				s->cur_file = 0;
		}
	}
	f->pending_ios++;

	lrand48_r(&s->rand, &r);
	offset = (r % (f->max_blocks - 1)) * BS;

	sqe->flags = 0;
	sqe->opcode = IORING_OP_READV;
	if (fixedbufs) {
		sqe->addr = s->iovecs[index].iov_base;
		sqe->len = BS;
		sqe->buf_index = index;
		sqe->flags |= IOSQE_FIXED_BUFFER;
	} else {
		sqe->addr = &s->iovecs[index];
		sqe->len = 1;
		sqe->buf_index = 0;
	}
	sqe->ioprio = 0;
	sqe->fd = f->fd;
	sqe->off = offset;
	sqe->data = (unsigned long) f;
}

static int prep_more_ios(struct submitter *s, int max_ios)
{
	struct io_sq_ring *ring = &s->sq_ring;
	unsigned index, tail, next_tail, prepped = 0;

	next_tail = tail = *ring->tail;
	do {
		next_tail++;
		barrier();
		if (next_tail == *ring->head)
			break;

		index = tail & sq_ring_mask;
		init_io(s, index);
		ring->array[index] = index;
		prepped++;
		tail = next_tail;
	} while (prepped < max_ios);

	if (*ring->tail != tail) {
		/* order tail store with writes to sqes above */
		barrier();
		*ring->tail = tail;
		barrier();
	}
	return prepped;
}

static int get_file_size(struct file *f)
{
	struct stat st;

	if (fstat(f->fd, &st) < 0)
		return -1;
	if (S_ISBLK(st.st_mode)) {
		unsigned long long bytes;

		if (ioctl(f->fd, BLKGETSIZE64, &bytes) != 0)
			return -1;

		f->max_blocks = bytes / BS;
		return 0;
	} else if (S_ISREG(st.st_mode)) {
		f->max_blocks = st.st_size / BS;
		return 0;
	}

	return -1;
}

static int reap_events(struct submitter *s)
{
	struct io_cq_ring *ring = &s->cq_ring;
	struct io_uring_cqe *cqe;
	unsigned head, reaped = 0;

	head = *ring->head;
	do {
		struct file *f;

		barrier();
		if (head == *ring->tail)
			break;
		cqe = &ring->cqes[head & cq_ring_mask];
		f = (struct file *) cqe->data;
		f->pending_ios--;
		if (cqe->res != BS) {
			printf("io: unexpected ret=%d\n", cqe->res);
			return -1;
		}
		if (cqe->flags & IOCQE_FLAG_CACHEHIT)
			s->cachehit++;
		else
			s->cachemiss++;
		reaped++;
		head++;
	} while (1);

	s->inflight -= reaped;
	*ring->head = head;
	barrier();
	return reaped;
}

static void *submitter_fn(void *data)
{
	struct submitter *s = data;
	int ret, prepped;

	printf("submitter=%d\n", gettid());

	srand48_r(pthread_self(), &s->rand);

	prepped = 0;
	do {
		int to_wait, to_submit, this_reap, to_prep;

		if (!prepped && s->inflight < DEPTH) {
			to_prep = min(DEPTH - s->inflight, BATCH_SUBMIT);
			prepped = prep_more_ios(s, to_prep);
		}
		s->inflight += prepped;
submit_more:
		to_submit = prepped;
submit:
		if (s->inflight + BATCH_SUBMIT < DEPTH)
			to_wait = 0;
		else
			to_wait = min(s->inflight + to_submit, BATCH_COMPLETE);

		ret = io_uring_enter(s, to_submit, to_wait,
					IORING_ENTER_GETEVENTS);
		s->calls++;

		this_reap = reap_events(s);
		if (this_reap == -1)
			break;
		s->reaps += this_reap;

		if (ret >= 0) {
			if (!ret) {
				to_submit = 0;
				if (s->inflight)
					goto submit;
				continue;
			} else if (ret < to_submit) {
				int diff = to_submit - ret;

				s->done += ret;
				prepped -= diff;
				goto submit_more;
			}
			s->done += ret;
			prepped = 0;
			continue;
		} else if (ret < 0) {
			if (errno == EAGAIN) {
				if (s->finish)
					break;
				if (this_reap)
					goto submit;
				to_submit = 0;
				goto submit;
			}
			printf("io_submit: %s\n", strerror(errno));
			break;
		}
	} while (!s->finish);

	finish = 1;
	return NULL;
}

static void sig_int(int sig)
{
	printf("Exiting on signal %d\n", sig);
	submitters[0].finish = 1;
	finish = 1;
}

static void arm_sig_int(void)
{
	struct sigaction act;

	memset(&act, 0, sizeof(act));
	act.sa_handler = sig_int;
	act.sa_flags = SA_RESTART;
	sigaction(SIGINT, &act, NULL);
}

static int setup_ring(struct submitter *s)
{
	struct io_sq_ring *sring = &s->sq_ring;
	struct io_cq_ring *cring = &s->cq_ring;
	struct io_uring_params p;
	int ret, fd;
	void *ptr;

	memset(&p, 0, sizeof(p));

	if (polled)
		p.flags |= IORING_SETUP_IOPOLL;
	if (sq_thread_poll) {
		p.flags |= IORING_SETUP_SQPOLL;
		if (sq_thread_cpu != -1)
			p.flags |= IORING_SETUP_SQ_AFF;
	}

	fd = io_uring_setup(DEPTH, &p);
	if (fd < 0) {
		perror("io_uring_setup");
		return 1;
	}
	s->ring_fd = fd;

	if (fixedbufs) {
		ret = io_uring_register_buffers(s);
		if (ret < 0) {
			perror("io_uring_register");
			return 1;
		}
	}

	ptr = mmap(0, p.sq_off.array + p.sq_entries * sizeof(__u32),
			PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
			IORING_OFF_SQ_RING);
	printf("sq_ring ptr = 0x%p\n", ptr);
	sring->head = ptr + p.sq_off.head;
	sring->tail = ptr + p.sq_off.tail;
	sring->ring_mask = ptr + p.sq_off.ring_mask;
	sring->ring_entries = ptr + p.sq_off.ring_entries;
	sring->array = ptr + p.sq_off.array;
	sq_ring_mask = *sring->ring_mask;

	s->sqes = mmap(0, p.sq_entries * sizeof(struct io_uring_sqe),
			PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
			IORING_OFF_SQES);
	printf("sqes ptr    = 0x%p\n", s->sqes);

	ptr = mmap(0, p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe),
			PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
			IORING_OFF_CQ_RING);
	printf("cq_ring ptr = 0x%p\n", ptr);
	cring->head = ptr + p.cq_off.head;
	cring->tail = ptr + p.cq_off.tail;
	cring->ring_mask = ptr + p.cq_off.ring_mask;
	cring->ring_entries = ptr + p.cq_off.ring_entries;
	cring->cqes = ptr + p.cq_off.cqes;
	cq_ring_mask = *cring->ring_mask;
	return 0;
}

int main(int argc, char *argv[])
{
	struct submitter *s = &submitters[0];
	unsigned long done, calls, reap, cache_hit, cache_miss;
	int err, i, flags, fd;
	struct rlimit rlim;
	void *ret;

	if (argc < 2) {
		printf("%s: filename\n", argv[0]);
		return 1;
	}

	flags = O_RDONLY | O_NOATIME;
	if (!buffered)
		flags |= O_DIRECT;

	i = 1;
	while (i < argc) {
		struct file *f = &s->files[s->nr_files];

		fd = open(argv[i], flags);
		if (fd < 0) {
			perror("open");
			return 1;
		}
		f->fd = fd;
		if (get_file_size(f)) {
			printf("failed getting size of device/file\n");
			return 1;
		}
		if (f->max_blocks <= 1) {
			printf("Zero file/device size?\n");
			return 1;
		}
		f->max_blocks--;

		printf("Added file %s\n", argv[i]);
		s->nr_files++;
		i++;
	}

	rlim.rlim_cur = RLIM_INFINITY;
	rlim.rlim_max = RLIM_INFINITY;
	if (setrlimit(RLIMIT_MEMLOCK, &rlim) < 0) {
		perror("setrlimit");
		return 1;
	}

	arm_sig_int();

	for (i = 0; i < DEPTH; i++) {
		void *buf;

		if (posix_memalign(&buf, BS, BS)) {
			printf("failed alloc\n");
			return 1;
		}
		s->iovecs[i].iov_base = buf;
		s->iovecs[i].iov_len = BS;
	}

	err = setup_ring(s);
	if (err) {
		printf("ring setup failed: %s, %d\n", strerror(errno), err);
		return 1;
	}
	printf("polled=%d, fixedbufs=%d, buffered=%d", polled, fixedbufs, buffered);
	printf(" QD=%d, sq_ring=%d, cq_ring=%d\n", DEPTH, *s->sq_ring.ring_entries, *s->cq_ring.ring_entries);

	pthread_create(&s->thread, NULL, submitter_fn, s);

	cache_hit = cache_miss = reap = calls = done = 0;
	do {
		unsigned long this_done = 0;
		unsigned long this_reap = 0;
		unsigned long this_call = 0;
		unsigned long this_cache_hit = 0;
		unsigned long this_cache_miss = 0;
		unsigned long rpc = 0, ipc = 0;
		double hit = 0.0;

		sleep(1);
		this_done += s->done;
		this_call += s->calls;
		this_reap += s->reaps;
		this_cache_hit += s->cachehit;
		this_cache_miss += s->cachemiss;
		if (this_cache_hit && this_cache_miss) {
			unsigned long hits, total;

			hits = this_cache_hit - cache_hit;
			total = hits + this_cache_miss - cache_miss;
			hit = (double) hits / (double) total;
			hit *= 100.0;
		}
		if (this_call - calls) {
			rpc = (this_done - done) / (this_call - calls);
			ipc = (this_reap - reap) / (this_call - calls);
		}
		printf("IOPS=%lu, IOS/call=%lu/%lu, inflight=%u (head=%u tail=%u), Cachehit=%0.2f%%\n",
				this_done - done, rpc, ipc, s->inflight,
				*s->cq_ring.head, *s->cq_ring.tail, hit);
		done = this_done;
		calls = this_call;
		reap = this_reap;
		cache_hit = s->cachehit;
		cache_miss = s->cachemiss;
	} while (!finish);

	pthread_join(s->thread, &ret);
	close(s->ring_fd);
	return 0;
}
Commit	Line	Data
c9fb4c5b JA	1	#include <stdio.h>
	2	#include <errno.h>
	3	#include <assert.h>
	4	#include <stdlib.h>
	5	#include <stddef.h>
	6	#include <signal.h>
	7	#include <inttypes.h>
	8
	9	#include <sys/types.h>
	10	#include <sys/stat.h>
	11	#include <sys/ioctl.h>
	12	#include <sys/syscall.h>
	13	#include <sys/resource.h>
c3e2fc25	14	#include <sys/mman.h>
e31b8288	15	#include <sys/uio.h>
c9fb4c5b JA	16	#include <linux/fs.h>
	17	#include <fcntl.h>
	18	#include <unistd.h>
c9fb4c5b JA	19	#include <string.h>
	20	#include <pthread.h>
	21	#include <sched.h>
	22
74efb029	23	#include "../arch/arch.h"
57fa61f0	24	#include "../lib/types.h"
e31b8288	25	#include "../os/io_uring.h"
ac122fea	26
e31b8288	27	#define barrier() __asm__ __volatile__("": : :"memory")
c3e2fc25	28
e31b8288	29	#define min(a, b) ((a < b) ? (a) : (b))
c3e2fc25	30
e31b8288	31	struct io_sq_ring {
e2239016 JA	32	unsigned *head;
	33	unsigned *tail;
	34	unsigned *ring_mask;
	35	unsigned *ring_entries;
	36	unsigned *array;
c9fb4c5b JA	37	};
c9fb4c5b JA	38
e31b8288	39	struct io_cq_ring {
e2239016 JA	40	unsigned *head;
	41	unsigned *tail;
	42	unsigned *ring_mask;
	43	unsigned *ring_entries;
f0403f94	44	struct io_uring_cqe *cqes;
c9fb4c5b JA	45	};
c9fb4c5b JA	46
701d1277	47	#define DEPTH 128
c9fb4c5b	48
701d1277 JA	49	#define BATCH_SUBMIT 64
701d1277 JA	50	#define BATCH_COMPLETE 64
c9fb4c5b JA	51
	52	#define BS 4096
	53
a7086591 JA	54	#define MAX_FDS 16
a7086591 JA	55
c3e2fc25	56	static unsigned sq_ring_mask, cq_ring_mask;
e39c34dc	57
a7086591 JA	58	struct file {
a7086591 JA	59	unsigned long max_blocks;
701d1277	60	unsigned pending_ios;
a7086591 JA	61	int fd;
	62	};
	63
c9fb4c5b JA	64	struct submitter {
c9fb4c5b JA	65	pthread_t thread;
f310970e	66	int ring_fd;
c9fb4c5b	67	struct drand48_data rand;
e31b8288	68	struct io_sq_ring sq_ring;
f0403f94	69	struct io_uring_sqe *sqes;
c3e2fc25	70	struct iovec iovecs[DEPTH];
e31b8288	71	struct io_cq_ring cq_ring;
c9fb4c5b JA	72	int inflight;
	73	unsigned long reaps;
	74	unsigned long done;
	75	unsigned long calls;
305c06ce	76	unsigned long cachehit, cachemiss;
c9fb4c5b	77	volatile int finish;
701d1277	78
a7086591 JA	79	struct file files[MAX_FDS];
	80	unsigned nr_files;
	81	unsigned cur_file;
c9fb4c5b JA	82	};
	83
	84	static struct submitter submitters[1];
	85	static volatile int finish;
	86
f0403f94	87	static int polled = 1; /* use IO polling */
701d1277	88	static int fixedbufs = 1; /* use fixed user buffers */
f0403f94	89	static int buffered = 0; /* use buffered IO, not O_DIRECT */
3d7d00a3 JA	90	static int sq_thread_poll = 0; /* use kernel submission/poller thread */
3d7d00a3 JA	91	static int sq_thread_cpu = -1; /* pin above thread to this CPU */
c9fb4c5b	92
2ea53ca3	93	static int io_uring_register_buffers(struct submitter *s)
c9fb4c5b	94	{
2ea53ca3 JA	95	struct io_uring_register_buffers reg = {
	96	.iovecs = s->iovecs,
	97	.nr_iovecs = DEPTH
	98	};
	99
	100	return syscall(__NR_sys_io_uring_register, s->ring_fd,
	101	IORING_REGISTER_BUFFERS, &reg);
	102	}
	103
	104	static int io_uring_setup(unsigned entries, struct io_uring_params *p)
	105	{
	106	return syscall(__NR_sys_io_uring_setup, entries, p);
c9fb4c5b JA	107	}
c9fb4c5b JA	108
c3e2fc25 JA	109	static int io_uring_enter(struct submitter *s, unsigned int to_submit,
c3e2fc25 JA	110	unsigned int min_complete, unsigned int flags)
c9fb4c5b	111	{
f310970e JA	112	return syscall(__NR_sys_io_uring_enter, s->ring_fd, to_submit,
f310970e JA	113	min_complete, flags);
c9fb4c5b JA	114	}
	115
	116	static int gettid(void)
	117	{
	118	return syscall(__NR_gettid);
	119	}
	120
701d1277 JA	121	static unsigned file_depth(struct submitter *s)
	122	{
	123	return (DEPTH + s->nr_files - 1) / s->nr_files;
	124	}
	125
a7086591	126	static void init_io(struct submitter *s, unsigned index)
c9fb4c5b	127	{
f0403f94	128	struct io_uring_sqe *sqe = &s->sqes[index];
c9fb4c5b	129	unsigned long offset;
a7086591	130	struct file *f;
c9fb4c5b JA	131	long r;
c9fb4c5b JA	132
701d1277 JA	133	if (s->nr_files == 1) {
	134	f = &s->files[0];
	135	} else {
	136	f = &s->files[s->cur_file];
	137	if (f->pending_ios >= file_depth(s)) {
	138	s->cur_file++;
	139	if (s->cur_file == s->nr_files)
	140	s->cur_file = 0;
	141	}
	142	}
	143	f->pending_ios++;
a7086591	144
c9fb4c5b	145	lrand48_r(&s->rand, &r);
a7086591	146	offset = (r % (f->max_blocks - 1)) * BS;
c9fb4c5b	147
2ea53ca3 JA	148	sqe->flags = 0;
2ea53ca3 JA	149	sqe->opcode = IORING_OP_READV;
f0403f94	150	if (fixedbufs) {
f0403f94 JA	151	sqe->addr = s->iovecs[index].iov_base;
f0403f94 JA	152	sqe->len = BS;
2ea53ca3 JA	153	sqe->buf_index = index;
2ea53ca3 JA	154	sqe->flags \|= IOSQE_FIXED_BUFFER;
f0403f94	155	} else {
f0403f94 JA	156	sqe->addr = &s->iovecs[index];
f0403f94 JA	157	sqe->len = 1;
2ea53ca3	158	sqe->buf_index = 0;
f0403f94	159	}
f0403f94	160	sqe->ioprio = 0;
a7086591	161	sqe->fd = f->fd;
f0403f94	162	sqe->off = offset;
701d1277	163	sqe->data = (unsigned long) f;
c9fb4c5b JA	164	}
c9fb4c5b JA	165
a7086591	166	static int prep_more_ios(struct submitter *s, int max_ios)
c9fb4c5b	167	{
e31b8288	168	struct io_sq_ring *ring = &s->sq_ring;
e2239016	169	unsigned index, tail, next_tail, prepped = 0;
c9fb4c5b	170
c3e2fc25	171	next_tail = tail = *ring->tail;
c9fb4c5b JA	172	do {
c9fb4c5b JA	173	next_tail++;
c9fb4c5b	174	barrier();
c3e2fc25	175	if (next_tail == *ring->head)
c9fb4c5b JA	176	break;
c9fb4c5b JA	177
e39c34dc	178	index = tail & sq_ring_mask;
a7086591	179	init_io(s, index);
c3e2fc25	180	ring->array[index] = index;
c9fb4c5b JA	181	prepped++;
	182	tail = next_tail;
	183	} while (prepped < max_ios);
	184
c3e2fc25	185	if (*ring->tail != tail) {
f0403f94	186	/* order tail store with writes to sqes above */
c9fb4c5b	187	barrier();
c3e2fc25	188	*ring->tail = tail;
c9fb4c5b JA	189	barrier();
	190	}
	191	return prepped;
	192	}
	193
a7086591	194	static int get_file_size(struct file *f)
c9fb4c5b JA	195	{
	196	struct stat st;
	197
a7086591	198	if (fstat(f->fd, &st) < 0)
c9fb4c5b JA	199	return -1;
	200	if (S_ISBLK(st.st_mode)) {
	201	unsigned long long bytes;
	202
a7086591	203	if (ioctl(f->fd, BLKGETSIZE64, &bytes) != 0)
c9fb4c5b JA	204	return -1;
c9fb4c5b JA	205
a7086591	206	f->max_blocks = bytes / BS;
c9fb4c5b JA	207	return 0;
c9fb4c5b JA	208	} else if (S_ISREG(st.st_mode)) {
a7086591	209	f->max_blocks = st.st_size / BS;
c9fb4c5b JA	210	return 0;
	211	}
	212
	213	return -1;
	214	}
	215
	216	static int reap_events(struct submitter *s)
	217	{
e31b8288	218	struct io_cq_ring *ring = &s->cq_ring;
f0403f94	219	struct io_uring_cqe *cqe;
e2239016	220	unsigned head, reaped = 0;
c9fb4c5b	221
c3e2fc25	222	head = *ring->head;
c9fb4c5b	223	do {
701d1277 JA	224	struct file *f;
701d1277 JA	225
c9fb4c5b	226	barrier();
c3e2fc25	227	if (head == *ring->tail)
c9fb4c5b	228	break;
f0403f94	229	cqe = &ring->cqes[head & cq_ring_mask];
701d1277 JA	230	f = (struct file *) cqe->data;
701d1277 JA	231	f->pending_ios--;
f0403f94	232	if (cqe->res != BS) {
f0403f94	233	printf("io: unexpected ret=%d\n", cqe->res);
c9fb4c5b JA	234	return -1;
c9fb4c5b JA	235	}
f0403f94	236	if (cqe->flags & IOCQE_FLAG_CACHEHIT)
305c06ce JA	237	s->cachehit++;
	238	else
	239	s->cachemiss++;
c9fb4c5b JA	240	reaped++;
c9fb4c5b JA	241	head++;
c9fb4c5b JA	242	} while (1);
	243
	244	s->inflight -= reaped;
c3e2fc25	245	*ring->head = head;
c9fb4c5b JA	246	barrier();
	247	return reaped;
	248	}
	249
	250	static void submitter_fn(void data)
	251	{
	252	struct submitter *s = data;
a7086591	253	int ret, prepped;
c9fb4c5b JA	254
	255	printf("submitter=%d\n", gettid());
	256
c9fb4c5b JA	257	srand48_r(pthread_self(), &s->rand);
	258
	259	prepped = 0;
	260	do {
f310970e	261	int to_wait, to_submit, this_reap, to_prep;
c9fb4c5b	262
f310970e JA	263	if (!prepped && s->inflight < DEPTH) {
f310970e JA	264	to_prep = min(DEPTH - s->inflight, BATCH_SUBMIT);
a7086591	265	prepped = prep_more_ios(s, to_prep);
f310970e	266	}
c9fb4c5b JA	267	s->inflight += prepped;
	268	submit_more:
	269	to_submit = prepped;
	270	submit:
	271	if (s->inflight + BATCH_SUBMIT < DEPTH)
	272	to_wait = 0;
	273	else
	274	to_wait = min(s->inflight + to_submit, BATCH_COMPLETE);
	275
f310970e JA	276	ret = io_uring_enter(s, to_submit, to_wait,
f310970e JA	277	IORING_ENTER_GETEVENTS);
c9fb4c5b JA	278	s->calls++;
	279
	280	this_reap = reap_events(s);
	281	if (this_reap == -1)
	282	break;
	283	s->reaps += this_reap;
	284
	285	if (ret >= 0) {
	286	if (!ret) {
	287	to_submit = 0;
	288	if (s->inflight)
	289	goto submit;
	290	continue;
	291	} else if (ret < to_submit) {
	292	int diff = to_submit - ret;
	293
	294	s->done += ret;
	295	prepped -= diff;
	296	goto submit_more;
	297	}
	298	s->done += ret;
	299	prepped = 0;
	300	continue;
	301	} else if (ret < 0) {
ac122fea	302	if (errno == EAGAIN) {
c9fb4c5b JA	303	if (s->finish)
	304	break;
	305	if (this_reap)
	306	goto submit;
c9fb4c5b JA	307	to_submit = 0;
	308	goto submit;
	309	}
ac122fea	310	printf("io_submit: %s\n", strerror(errno));
c9fb4c5b JA	311	break;
	312	}
	313	} while (!s->finish);
a7086591	314
c9fb4c5b JA	315	finish = 1;
	316	return NULL;
	317	}
	318
	319	static void sig_int(int sig)
	320	{
	321	printf("Exiting on signal %d\n", sig);
	322	submitters[0].finish = 1;
	323	finish = 1;
	324	}
	325
	326	static void arm_sig_int(void)
	327	{
	328	struct sigaction act;
	329
	330	memset(&act, 0, sizeof(act));
	331	act.sa_handler = sig_int;
	332	act.sa_flags = SA_RESTART;
	333	sigaction(SIGINT, &act, NULL);
	334	}
	335
c3e2fc25 JA	336	static int setup_ring(struct submitter *s)
c3e2fc25 JA	337	{
e31b8288 JA	338	struct io_sq_ring *sring = &s->sq_ring;
	339	struct io_cq_ring *cring = &s->cq_ring;
	340	struct io_uring_params p;
2ea53ca3	341	int ret, fd;
c3e2fc25	342	void *ptr;
c3e2fc25 JA	343
	344	memset(&p, 0, sizeof(p));
	345
0e47f11b JA	346	if (polled)
0e47f11b JA	347	p.flags \|= IORING_SETUP_IOPOLL;
3d7d00a3	348	if (sq_thread_poll) {
2ea53ca3	349	p.flags \|= IORING_SETUP_SQPOLL;
3d7d00a3 JA	350	if (sq_thread_cpu != -1)
3d7d00a3 JA	351	p.flags \|= IORING_SETUP_SQ_AFF;
c3e2fc25 JA	352	}
c3e2fc25 JA	353
2ea53ca3	354	fd = io_uring_setup(DEPTH, &p);
c3e2fc25 JA	355	if (fd < 0) {
	356	perror("io_uring_setup");
	357	return 1;
	358	}
f310970e	359	s->ring_fd = fd;
2ea53ca3 JA	360
	361	if (fixedbufs) {
	362	ret = io_uring_register_buffers(s);
	363	if (ret < 0) {
	364	perror("io_uring_register");
	365	return 1;
	366	}
	367	}
	368
e2239016	369	ptr = mmap(0, p.sq_off.array + p.sq_entries * sizeof(__u32),
f310970e JA	370	PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd,
f310970e JA	371	IORING_OFF_SQ_RING);
c3e2fc25 JA	372	printf("sq_ring ptr = 0x%p\n", ptr);
	373	sring->head = ptr + p.sq_off.head;
	374	sring->tail = ptr + p.sq_off.tail;
	375	sring->ring_mask = ptr + p.sq_off.ring_mask;
	376	sring->ring_entries = ptr + p.sq_off.ring_entries;
ac122fea	377	sring->array = ptr + p.sq_off.array;
c3e2fc25 JA	378	sq_ring_mask = *sring->ring_mask;
c3e2fc25 JA	379
f0403f94	380	s->sqes = mmap(0, p.sq_entries * sizeof(struct io_uring_sqe),
f310970e	381	PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd,
f0403f94 JA	382	IORING_OFF_SQES);
f0403f94 JA	383	printf("sqes ptr = 0x%p\n", s->sqes);
c3e2fc25	384
f0403f94	385	ptr = mmap(0, p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe),
f310970e JA	386	PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd,
f310970e JA	387	IORING_OFF_CQ_RING);
c3e2fc25 JA	388	printf("cq_ring ptr = 0x%p\n", ptr);
	389	cring->head = ptr + p.cq_off.head;
	390	cring->tail = ptr + p.cq_off.tail;
	391	cring->ring_mask = ptr + p.cq_off.ring_mask;
	392	cring->ring_entries = ptr + p.cq_off.ring_entries;
f0403f94	393	cring->cqes = ptr + p.cq_off.cqes;
c3e2fc25 JA	394	cq_ring_mask = *cring->ring_mask;
	395	return 0;
	396	}
	397
c9fb4c5b JA	398	int main(int argc, char *argv[])
	399	{
	400	struct submitter *s = &submitters[0];
305c06ce	401	unsigned long done, calls, reap, cache_hit, cache_miss;
a7086591	402	int err, i, flags, fd;
c9fb4c5b	403	struct rlimit rlim;
c3e2fc25	404	void *ret;
c9fb4c5b JA	405
	406	if (argc < 2) {
	407	printf("%s: filename\n", argv[0]);
	408	return 1;
	409	}
	410
701d1277	411	flags = O_RDONLY \| O_NOATIME;
a7086591 JA	412	if (!buffered)
	413	flags \|= O_DIRECT;
	414
	415	i = 1;
	416	while (i < argc) {
	417	struct file *f = &s->files[s->nr_files];
	418
	419	fd = open(argv[i], flags);
	420	if (fd < 0) {
	421	perror("open");
	422	return 1;
	423	}
	424	f->fd = fd;
	425	if (get_file_size(f)) {
	426	printf("failed getting size of device/file\n");
	427	return 1;
	428	}
	429	if (f->max_blocks <= 1) {
	430	printf("Zero file/device size?\n");
	431	return 1;
	432	}
	433	f->max_blocks--;
	434
	435	printf("Added file %s\n", argv[i]);
	436	s->nr_files++;
	437	i++;
	438	}
	439
c9fb4c5b JA	440	rlim.rlim_cur = RLIM_INFINITY;
	441	rlim.rlim_max = RLIM_INFINITY;
	442	if (setrlimit(RLIMIT_MEMLOCK, &rlim) < 0) {
	443	perror("setrlimit");
	444	return 1;
	445	}
	446
	447	arm_sig_int();
	448
c3e2fc25 JA	449	for (i = 0; i < DEPTH; i++) {
c3e2fc25 JA	450	void *buf;
c9fb4c5b	451
c3e2fc25	452	if (posix_memalign(&buf, BS, BS)) {
c9fb4c5b JA	453	printf("failed alloc\n");
	454	return 1;
	455	}
c3e2fc25 JA	456	s->iovecs[i].iov_base = buf;
c3e2fc25 JA	457	s->iovecs[i].iov_len = BS;
c9fb4c5b JA	458	}
c9fb4c5b JA	459
c3e2fc25	460	err = setup_ring(s);
c9fb4c5b	461	if (err) {
c3e2fc25	462	printf("ring setup failed: %s, %d\n", strerror(errno), err);
c9fb4c5b JA	463	return 1;
c9fb4c5b JA	464	}
c3e2fc25 JA	465	printf("polled=%d, fixedbufs=%d, buffered=%d", polled, fixedbufs, buffered);
c3e2fc25 JA	466	printf(" QD=%d, sq_ring=%d, cq_ring=%d\n", DEPTH, s->sq_ring.ring_entries, s->cq_ring.ring_entries);
c9fb4c5b JA	467
	468	pthread_create(&s->thread, NULL, submitter_fn, s);
	469
305c06ce	470	cache_hit = cache_miss = reap = calls = done = 0;
c9fb4c5b JA	471	do {
	472	unsigned long this_done = 0;
	473	unsigned long this_reap = 0;
	474	unsigned long this_call = 0;
305c06ce JA	475	unsigned long this_cache_hit = 0;
305c06ce JA	476	unsigned long this_cache_miss = 0;
c9fb4c5b	477	unsigned long rpc = 0, ipc = 0;
305c06ce	478	double hit = 0.0;
c9fb4c5b JA	479
	480	sleep(1);
	481	this_done += s->done;
	482	this_call += s->calls;
	483	this_reap += s->reaps;
305c06ce JA	484	this_cache_hit += s->cachehit;
	485	this_cache_miss += s->cachemiss;
	486	if (this_cache_hit && this_cache_miss) {
	487	unsigned long hits, total;
	488
	489	hits = this_cache_hit - cache_hit;
	490	total = hits + this_cache_miss - cache_miss;
	491	hit = (double) hits / (double) total;
	492	hit *= 100.0;
	493	}
c9fb4c5b JA	494	if (this_call - calls) {
	495	rpc = (this_done - done) / (this_call - calls);
	496	ipc = (this_reap - reap) / (this_call - calls);
	497	}
10c4d131	498	printf("IOPS=%lu, IOS/call=%lu/%lu, inflight=%u (head=%u tail=%u), Cachehit=%0.2f%%\n",
c9fb4c5b	499	this_done - done, rpc, ipc, s->inflight,
c3e2fc25	500	s->cq_ring.head, s->cq_ring.tail, hit);
c9fb4c5b JA	501	done = this_done;
	502	calls = this_call;
	503	reap = this_reap;
305c06ce JA	504	cache_hit = s->cachehit;
305c06ce JA	505	cache_miss = s->cachemiss;
c9fb4c5b JA	506	} while (!finish);
	507
	508	pthread_join(s->thread, &ret);
e31b8288	509	close(s->ring_fd);
c9fb4c5b JA	510	return 0;
c9fb4c5b JA	511	}