[fio.git] / t / io_uring.c

#include <stdio.h>
#include <errno.h>
#include <assert.h>
#include <stdlib.h>
#include <stddef.h>
#include <signal.h>
#include <inttypes.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <sys/syscall.h>
#include <sys/resource.h>
#include <sys/mman.h>
#include <sys/uio.h>
#include <linux/fs.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <pthread.h>
#include <sched.h>

#include "../arch/arch.h"
#include "../lib/types.h"
#include "../os/linux/io_uring.h"

#define min(a, b)		((a < b) ? (a) : (b))

struct io_sq_ring {
	unsigned *head;
	unsigned *tail;
	unsigned *ring_mask;
	unsigned *ring_entries;
	unsigned *flags;
	unsigned *array;
};

struct io_cq_ring {
	unsigned *head;
	unsigned *tail;
	unsigned *ring_mask;
	unsigned *ring_entries;
	struct io_uring_cqe *cqes;
};

#define DEPTH			128

#define BATCH_SUBMIT		32
#define BATCH_COMPLETE		32

#define BS			4096

#define MAX_FDS			16

static unsigned sq_ring_mask, cq_ring_mask;

struct file {
	unsigned long max_blocks;
	unsigned pending_ios;
	int real_fd;
	int fixed_fd;
};

struct submitter {
	pthread_t thread;
	int ring_fd;
	struct drand48_data rand;
	struct io_sq_ring sq_ring;
	struct io_uring_sqe *sqes;
	struct iovec iovecs[DEPTH];
	struct io_cq_ring cq_ring;
	int inflight;
	unsigned long reaps;
	unsigned long done;
	unsigned long calls;
	unsigned long cachehit, cachemiss;
	volatile int finish;

	__s32 *fds;

	struct file files[MAX_FDS];
	unsigned nr_files;
	unsigned cur_file;
};

static struct submitter submitters[1];
static volatile int finish;

static int polled = 1;		/* use IO polling */
static int fixedbufs = 1;	/* use fixed user buffers */
static int register_files = 1;	/* use fixed files */
static int buffered = 0;	/* use buffered IO, not O_DIRECT */
static int sq_thread_poll = 0;	/* use kernel submission/poller thread */
static int sq_thread_cpu = -1;	/* pin above thread to this CPU */
static int do_nop = 0;		/* no-op SQ ring commands */

static int io_uring_register_buffers(struct submitter *s)
{
	struct io_uring_register_buffers reg = {
		.iovecs = s->iovecs,
		.nr_iovecs = DEPTH
	};

	if (do_nop)
		return 0;

	return syscall(__NR_sys_io_uring_register, s->ring_fd,
			IORING_REGISTER_BUFFERS, &reg);
}

static int io_uring_register_files(struct submitter *s)
{
	struct io_uring_register_files reg;
	int i;

	if (do_nop)
		return 0;

	s->fds = calloc(s->nr_files, sizeof(__s32));
	for (i = 0; i < s->nr_files; i++) {
		s->fds[i] = s->files[i].real_fd;
		s->files[i].fixed_fd = i;
	}
	reg.fds = s->fds;
	reg.nr_fds = s->nr_files;

	return syscall(__NR_sys_io_uring_register, s->ring_fd,
			IORING_REGISTER_FILES, &reg);
}

static int io_uring_setup(unsigned entries, struct io_uring_params *p)
{
	return syscall(__NR_sys_io_uring_setup, entries, p);
}

static int io_uring_enter(struct submitter *s, unsigned int to_submit,
			  unsigned int min_complete, unsigned int flags)
{
	return syscall(__NR_sys_io_uring_enter, s->ring_fd, to_submit,
			min_complete, flags);
}

static int gettid(void)
{
	return syscall(__NR_gettid);
}

static unsigned file_depth(struct submitter *s)
{
	return (DEPTH + s->nr_files - 1) / s->nr_files;
}

static void init_io(struct submitter *s, unsigned index)
{
	struct io_uring_sqe *sqe = &s->sqes[index];
	unsigned long offset;
	struct file *f;
	long r;

	if (do_nop) {
		sqe->opcode = IORING_OP_NOP;
		return;
	}

	if (s->nr_files == 1) {
		f = &s->files[0];
	} else {
		f = &s->files[s->cur_file];
		if (f->pending_ios >= file_depth(s)) {
			s->cur_file++;
			if (s->cur_file == s->nr_files)
				s->cur_file = 0;
			f = &s->files[s->cur_file];
		}
	}
	f->pending_ios++;

	lrand48_r(&s->rand, &r);
	offset = (r % (f->max_blocks - 1)) * BS;

	if (register_files) {
		sqe->flags = IOSQE_FIXED_FILE;
		sqe->fd = f->fixed_fd;
	} else {
		sqe->flags = 0;
		sqe->fd = f->real_fd;
	}
	if (fixedbufs) {
		sqe->opcode = IORING_OP_READ_FIXED;
		sqe->addr = s->iovecs[index].iov_base;
		sqe->len = BS;
		sqe->buf_index = index;
	} else {
		sqe->opcode = IORING_OP_READV;
		sqe->addr = &s->iovecs[index];
		sqe->len = 1;
		sqe->buf_index = 0;
	}
	sqe->ioprio = 0;
	sqe->off = offset;
	sqe->user_data = (unsigned long) f;
}

static int prep_more_ios(struct submitter *s, int max_ios)
{
	struct io_sq_ring *ring = &s->sq_ring;
	unsigned index, tail, next_tail, prepped = 0;

	next_tail = tail = *ring->tail;
	do {
		next_tail++;
		read_barrier();
		if (next_tail == *ring->head)
			break;

		index = tail & sq_ring_mask;
		init_io(s, index);
		ring->array[index] = index;
		prepped++;
		tail = next_tail;
	} while (prepped < max_ios);

	if (*ring->tail != tail) {
		/* order tail store with writes to sqes above */
		write_barrier();
		*ring->tail = tail;
		write_barrier();
	}
	return prepped;
}

static int get_file_size(struct file *f)
{
	struct stat st;

	if (fstat(f->real_fd, &st) < 0)
		return -1;
	if (S_ISBLK(st.st_mode)) {
		unsigned long long bytes;

		if (ioctl(f->real_fd, BLKGETSIZE64, &bytes) != 0)
			return -1;

		f->max_blocks = bytes / BS;
		return 0;
	} else if (S_ISREG(st.st_mode)) {
		f->max_blocks = st.st_size / BS;
		return 0;
	}

	return -1;
}

static int reap_events(struct submitter *s)
{
	struct io_cq_ring *ring = &s->cq_ring;
	struct io_uring_cqe *cqe;
	unsigned head, reaped = 0;

	head = *ring->head;
	do {
		struct file *f;

		read_barrier();
		if (head == *ring->tail)
			break;
		cqe = &ring->cqes[head & cq_ring_mask];
		if (!do_nop) {
			f = (struct file *) (uintptr_t) cqe->user_data;
			f->pending_ios--;
			if (cqe->res != BS) {
				printf("io: unexpected ret=%d\n", cqe->res);
				return -1;
			}
		}
		if (cqe->flags & IOCQE_FLAG_CACHEHIT)
			s->cachehit++;
		else
			s->cachemiss++;
		reaped++;
		head++;
	} while (1);

	s->inflight -= reaped;
	*ring->head = head;
	write_barrier();
	return reaped;
}

static void *submitter_fn(void *data)
{
	struct submitter *s = data;
	struct io_sq_ring *ring = &s->sq_ring;
	int ret, prepped;

	printf("submitter=%d\n", gettid());

	srand48_r(pthread_self(), &s->rand);

	prepped = 0;
	do {
		int to_wait, to_submit, this_reap, to_prep;

		if (!prepped && s->inflight < DEPTH) {
			to_prep = min(DEPTH - s->inflight, BATCH_SUBMIT);
			prepped = prep_more_ios(s, to_prep);
		}
		s->inflight += prepped;
submit_more:
		to_submit = prepped;
submit:
		if (to_submit && (s->inflight + to_submit <= DEPTH))
			to_wait = 0;
		else
			to_wait = min(s->inflight + to_submit, BATCH_COMPLETE);

		/*
		 * Only need to call io_uring_enter if we're not using SQ thread
		 * poll, or if IORING_SQ_NEED_WAKEUP is set.
		 */
		if (!sq_thread_poll || (*ring->flags & IORING_SQ_NEED_WAKEUP)) {
			unsigned flags = 0;

			if (to_wait)
				flags = IORING_ENTER_GETEVENTS;
			ret = io_uring_enter(s, to_submit, to_wait, flags);
			s->calls++;
		}

		/*
		 * For non SQ thread poll, we already got the events we needed
		 * through the io_uring_enter() above. For SQ thread poll, we
		 * need to loop here until we find enough events.
		 */
		this_reap = 0;
		do {
			int r;
			r = reap_events(s);
			if (r == -1) {
				s->finish = 1;
				break;
			} else if (r > 0)
				this_reap += r;
		} while (sq_thread_poll && this_reap < to_wait);
		s->reaps += this_reap;

		if (ret >= 0) {
			if (!ret) {
				to_submit = 0;
				if (s->inflight)
					goto submit;
				continue;
			} else if (ret < to_submit) {
				int diff = to_submit - ret;

				s->done += ret;
				prepped -= diff;
				goto submit_more;
			}
			s->done += ret;
			prepped = 0;
			continue;
		} else if (ret < 0) {
			if (errno == EAGAIN) {
				if (s->finish)
					break;
				if (this_reap)
					goto submit;
				to_submit = 0;
				goto submit;
			}
			printf("io_submit: %s\n", strerror(errno));
			break;
		}
	} while (!s->finish);

	finish = 1;
	return NULL;
}

static void sig_int(int sig)
{
	printf("Exiting on signal %d\n", sig);
	submitters[0].finish = 1;
	finish = 1;
}

static void arm_sig_int(void)
{
	struct sigaction act;

	memset(&act, 0, sizeof(act));
	act.sa_handler = sig_int;
	act.sa_flags = SA_RESTART;
	sigaction(SIGINT, &act, NULL);
}

static int setup_ring(struct submitter *s)
{
	struct io_sq_ring *sring = &s->sq_ring;
	struct io_cq_ring *cring = &s->cq_ring;
	struct io_uring_params p;
	int ret, fd;
	void *ptr;

	memset(&p, 0, sizeof(p));

	if (polled && !do_nop)
		p.flags |= IORING_SETUP_IOPOLL;
	if (sq_thread_poll) {
		p.flags |= IORING_SETUP_SQPOLL;
		if (sq_thread_cpu != -1) {
			p.flags |= IORING_SETUP_SQ_AFF;
			p.sq_thread_cpu = sq_thread_cpu;
		}
	}

	fd = io_uring_setup(DEPTH, &p);
	if (fd < 0) {
		perror("io_uring_setup");
		return 1;
	}
	s->ring_fd = fd;

	if (fixedbufs) {
		ret = io_uring_register_buffers(s);
		if (ret < 0) {
			perror("io_uring_register_buffers");
			return 1;
		}
	}

	if (register_files) {
		ret = io_uring_register_files(s);
		if (ret < 0) {
			perror("io_uring_register_files");
			return 1;
		}
	}

	ptr = mmap(0, p.sq_off.array + p.sq_entries * sizeof(__u32),
			PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
			IORING_OFF_SQ_RING);
	printf("sq_ring ptr = 0x%p\n", ptr);
	sring->head = ptr + p.sq_off.head;
	sring->tail = ptr + p.sq_off.tail;
	sring->ring_mask = ptr + p.sq_off.ring_mask;
	sring->ring_entries = ptr + p.sq_off.ring_entries;
	sring->flags = ptr + p.sq_off.flags;
	sring->array = ptr + p.sq_off.array;
	sq_ring_mask = *sring->ring_mask;

	s->sqes = mmap(0, p.sq_entries * sizeof(struct io_uring_sqe),
			PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
			IORING_OFF_SQES);
	printf("sqes ptr    = 0x%p\n", s->sqes);

	ptr = mmap(0, p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe),
			PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
			IORING_OFF_CQ_RING);
	printf("cq_ring ptr = 0x%p\n", ptr);
	cring->head = ptr + p.cq_off.head;
	cring->tail = ptr + p.cq_off.tail;
	cring->ring_mask = ptr + p.cq_off.ring_mask;
	cring->ring_entries = ptr + p.cq_off.ring_entries;
	cring->cqes = ptr + p.cq_off.cqes;
	cq_ring_mask = *cring->ring_mask;
	return 0;
}

static void file_depths(char *buf)
{
	struct submitter *s = &submitters[0];
	char *p;
	int i;

	buf[0] = '\0';
	p = buf;
	for (i = 0; i < s->nr_files; i++) {
		struct file *f = &s->files[i];

		if (i + 1 == s->nr_files)
			p += sprintf(p, "%d", f->pending_ios);
		else
			p += sprintf(p, "%d, ", f->pending_ios);
	}
}

int main(int argc, char *argv[])
{
	struct submitter *s = &submitters[0];
	unsigned long done, calls, reap, cache_hit, cache_miss;
	int err, i, flags, fd;
	char *fdepths;
	void *ret;

	if (!do_nop && argc < 2) {
		printf("%s: filename\n", argv[0]);
		return 1;
	}

	flags = O_RDONLY | O_NOATIME;
	if (!buffered)
		flags |= O_DIRECT;

	i = 1;
	while (!do_nop && i < argc) {
		struct file *f = &s->files[s->nr_files];

		fd = open(argv[i], flags);
		if (fd < 0) {
			perror("open");
			return 1;
		}
		f->real_fd = fd;
		if (get_file_size(f)) {
			printf("failed getting size of device/file\n");
			return 1;
		}
		if (f->max_blocks <= 1) {
			printf("Zero file/device size?\n");
			return 1;
		}
		f->max_blocks--;

		printf("Added file %s\n", argv[i]);
		s->nr_files++;
		i++;
	}

	if (fixedbufs) {
		struct rlimit rlim;

		rlim.rlim_cur = RLIM_INFINITY;
		rlim.rlim_max = RLIM_INFINITY;
		if (setrlimit(RLIMIT_MEMLOCK, &rlim) < 0) {
			perror("setrlimit");
			return 1;
		}
	}

	arm_sig_int();

	for (i = 0; i < DEPTH; i++) {
		void *buf;

		if (posix_memalign(&buf, BS, BS)) {
			printf("failed alloc\n");
			return 1;
		}
		s->iovecs[i].iov_base = buf;
		s->iovecs[i].iov_len = BS;
	}

	err = setup_ring(s);
	if (err) {
		printf("ring setup failed: %s, %d\n", strerror(errno), err);
		return 1;
	}
	printf("polled=%d, fixedbufs=%d, buffered=%d", polled, fixedbufs, buffered);
	printf(" QD=%d, sq_ring=%d, cq_ring=%d\n", DEPTH, *s->sq_ring.ring_entries, *s->cq_ring.ring_entries);

	pthread_create(&s->thread, NULL, submitter_fn, s);

	fdepths = malloc(8 * s->nr_files);
	cache_hit = cache_miss = reap = calls = done = 0;
	do {
		unsigned long this_done = 0;
		unsigned long this_reap = 0;
		unsigned long this_call = 0;
		unsigned long this_cache_hit = 0;
		unsigned long this_cache_miss = 0;
		unsigned long rpc = 0, ipc = 0;
		double hit = 0.0;

		sleep(1);
		this_done += s->done;
		this_call += s->calls;
		this_reap += s->reaps;
		this_cache_hit += s->cachehit;
		this_cache_miss += s->cachemiss;
		if (this_cache_hit && this_cache_miss) {
			unsigned long hits, total;

			hits = this_cache_hit - cache_hit;
			total = hits + this_cache_miss - cache_miss;
			hit = (double) hits / (double) total;
			hit *= 100.0;
		}
		if (this_call - calls) {
			rpc = (this_done - done) / (this_call - calls);
			ipc = (this_reap - reap) / (this_call - calls);
		} else
			rpc = ipc = -1;
		file_depths(fdepths);
		printf("IOPS=%lu, IOS/call=%ld/%ld, inflight=%u (%s), Cachehit=%0.2f%%\n",
				this_done - done, rpc, ipc, s->inflight,
				fdepths, hit);
		done = this_done;
		calls = this_call;
		reap = this_reap;
		cache_hit = s->cachehit;
		cache_miss = s->cachemiss;
	} while (!finish);

	pthread_join(s->thread, &ret);
	close(s->ring_fd);
	free(fdepths);
	return 0;
}
Commit	Line	Data
c9fb4c5b JA	1	#include <stdio.h>
	2	#include <errno.h>
	3	#include <assert.h>
	4	#include <stdlib.h>
	5	#include <stddef.h>
	6	#include <signal.h>
	7	#include <inttypes.h>
	8
	9	#include <sys/types.h>
	10	#include <sys/stat.h>
	11	#include <sys/ioctl.h>
	12	#include <sys/syscall.h>
	13	#include <sys/resource.h>
c3e2fc25	14	#include <sys/mman.h>
e31b8288	15	#include <sys/uio.h>
c9fb4c5b JA	16	#include <linux/fs.h>
	17	#include <fcntl.h>
	18	#include <unistd.h>
c9fb4c5b JA	19	#include <string.h>
	20	#include <pthread.h>
	21	#include <sched.h>
	22
74efb029	23	#include "../arch/arch.h"
57fa61f0	24	#include "../lib/types.h"
f3e769a4	25	#include "../os/linux/io_uring.h"
ac122fea	26
e31b8288	27	#define min(a, b) ((a < b) ? (a) : (b))
c3e2fc25	28
e31b8288	29	struct io_sq_ring {
e2239016 JA	30	unsigned *head;
	31	unsigned *tail;
	32	unsigned *ring_mask;
	33	unsigned *ring_entries;
ce1705de	34	unsigned *flags;
e2239016	35	unsigned *array;
c9fb4c5b JA	36	};
c9fb4c5b JA	37
e31b8288	38	struct io_cq_ring {
e2239016 JA	39	unsigned *head;
	40	unsigned *tail;
	41	unsigned *ring_mask;
	42	unsigned *ring_entries;
f0403f94	43	struct io_uring_cqe *cqes;
c9fb4c5b JA	44	};
c9fb4c5b JA	45
701d1277	46	#define DEPTH 128
c9fb4c5b	47
2e7888ef JA	48	#define BATCH_SUBMIT 32
2e7888ef JA	49	#define BATCH_COMPLETE 32
c9fb4c5b JA	50
	51	#define BS 4096
	52
a7086591 JA	53	#define MAX_FDS 16
a7086591 JA	54
c3e2fc25	55	static unsigned sq_ring_mask, cq_ring_mask;
e39c34dc	56
a7086591 JA	57	struct file {
a7086591 JA	58	unsigned long max_blocks;
701d1277	59	unsigned pending_ios;
48e698fa JA	60	int real_fd;
48e698fa JA	61	int fixed_fd;
a7086591 JA	62	};
a7086591 JA	63
c9fb4c5b JA	64	struct submitter {
c9fb4c5b JA	65	pthread_t thread;
f310970e	66	int ring_fd;
c9fb4c5b	67	struct drand48_data rand;
e31b8288	68	struct io_sq_ring sq_ring;
f0403f94	69	struct io_uring_sqe *sqes;
c3e2fc25	70	struct iovec iovecs[DEPTH];
e31b8288	71	struct io_cq_ring cq_ring;
c9fb4c5b JA	72	int inflight;
	73	unsigned long reaps;
	74	unsigned long done;
	75	unsigned long calls;
305c06ce	76	unsigned long cachehit, cachemiss;
c9fb4c5b	77	volatile int finish;
701d1277	78
48e698fa JA	79	__s32 *fds;
48e698fa JA	80
a7086591 JA	81	struct file files[MAX_FDS];
	82	unsigned nr_files;
	83	unsigned cur_file;
c9fb4c5b JA	84	};
	85
	86	static struct submitter submitters[1];
	87	static volatile int finish;
	88
f0403f94	89	static int polled = 1; /* use IO polling */
701d1277	90	static int fixedbufs = 1; /* use fixed user buffers */
8c5fa755	91	static int register_files = 1; /* use fixed files */
f0403f94	92	static int buffered = 0; /* use buffered IO, not O_DIRECT */
3d7d00a3 JA	93	static int sq_thread_poll = 0; /* use kernel submission/poller thread */
3d7d00a3 JA	94	static int sq_thread_cpu = -1; /* pin above thread to this CPU */
8025517d	95	static int do_nop = 0; /* no-op SQ ring commands */
c9fb4c5b	96
2ea53ca3	97	static int io_uring_register_buffers(struct submitter *s)
c9fb4c5b	98	{
2ea53ca3 JA	99	struct io_uring_register_buffers reg = {
	100	.iovecs = s->iovecs,
	101	.nr_iovecs = DEPTH
	102	};
	103
8025517d JA	104	if (do_nop)
	105	return 0;
	106
2ea53ca3 JA	107	return syscall(__NR_sys_io_uring_register, s->ring_fd,
	108	IORING_REGISTER_BUFFERS, &reg);
	109	}
	110
a7abc9fb JA	111	static int io_uring_register_files(struct submitter *s)
	112	{
	113	struct io_uring_register_files reg;
48e698fa	114	int i;
a7abc9fb	115
8025517d JA	116	if (do_nop)
	117	return 0;
	118
48e698fa JA	119	s->fds = calloc(s->nr_files, sizeof(__s32));
	120	for (i = 0; i < s->nr_files; i++) {
	121	s->fds[i] = s->files[i].real_fd;
	122	s->files[i].fixed_fd = i;
	123	}
	124	reg.fds = s->fds;
a7abc9fb JA	125	reg.nr_fds = s->nr_files;
a7abc9fb JA	126
48e698fa	127	return syscall(__NR_sys_io_uring_register, s->ring_fd,
a7abc9fb	128	IORING_REGISTER_FILES, &reg);
a7abc9fb JA	129	}
a7abc9fb JA	130
2ea53ca3 JA	131	static int io_uring_setup(unsigned entries, struct io_uring_params *p)
	132	{
	133	return syscall(__NR_sys_io_uring_setup, entries, p);
c9fb4c5b JA	134	}
c9fb4c5b JA	135
c3e2fc25 JA	136	static int io_uring_enter(struct submitter *s, unsigned int to_submit,
c3e2fc25 JA	137	unsigned int min_complete, unsigned int flags)
c9fb4c5b	138	{
f310970e JA	139	return syscall(__NR_sys_io_uring_enter, s->ring_fd, to_submit,
f310970e JA	140	min_complete, flags);
c9fb4c5b JA	141	}
	142
	143	static int gettid(void)
	144	{
	145	return syscall(__NR_gettid);
	146	}
	147
701d1277 JA	148	static unsigned file_depth(struct submitter *s)
	149	{
	150	return (DEPTH + s->nr_files - 1) / s->nr_files;
	151	}
	152
a7086591	153	static void init_io(struct submitter *s, unsigned index)
c9fb4c5b	154	{
f0403f94	155	struct io_uring_sqe *sqe = &s->sqes[index];
c9fb4c5b	156	unsigned long offset;
a7086591	157	struct file *f;
c9fb4c5b JA	158	long r;
c9fb4c5b JA	159
8025517d JA	160	if (do_nop) {
	161	sqe->opcode = IORING_OP_NOP;
	162	return;
	163	}
	164
701d1277 JA	165	if (s->nr_files == 1) {
	166	f = &s->files[0];
	167	} else {
	168	f = &s->files[s->cur_file];
	169	if (f->pending_ios >= file_depth(s)) {
	170	s->cur_file++;
	171	if (s->cur_file == s->nr_files)
	172	s->cur_file = 0;
93d1811c	173	f = &s->files[s->cur_file];
701d1277 JA	174	}
	175	}
	176	f->pending_ios++;
a7086591	177
c9fb4c5b	178	lrand48_r(&s->rand, &r);
a7086591	179	offset = (r % (f->max_blocks - 1)) * BS;
c9fb4c5b	180
8c5fa755 JA	181	if (register_files) {
	182	sqe->flags = IOSQE_FIXED_FILE;
	183	sqe->fd = f->fixed_fd;
	184	} else {
	185	sqe->flags = 0;
	186	sqe->fd = f->real_fd;
	187	}
f0403f94	188	if (fixedbufs) {
48e698fa	189	sqe->opcode = IORING_OP_READ_FIXED;
f0403f94 JA	190	sqe->addr = s->iovecs[index].iov_base;
f0403f94 JA	191	sqe->len = BS;
2ea53ca3	192	sqe->buf_index = index;
f0403f94	193	} else {
48e698fa	194	sqe->opcode = IORING_OP_READV;
f0403f94 JA	195	sqe->addr = &s->iovecs[index];
f0403f94 JA	196	sqe->len = 1;
2ea53ca3	197	sqe->buf_index = 0;
f0403f94	198	}
f0403f94	199	sqe->ioprio = 0;
f0403f94	200	sqe->off = offset;
48e698fa	201	sqe->user_data = (unsigned long) f;
c9fb4c5b JA	202	}
c9fb4c5b JA	203
a7086591	204	static int prep_more_ios(struct submitter *s, int max_ios)
c9fb4c5b	205	{
e31b8288	206	struct io_sq_ring *ring = &s->sq_ring;
e2239016	207	unsigned index, tail, next_tail, prepped = 0;
c9fb4c5b	208
c3e2fc25	209	next_tail = tail = *ring->tail;
c9fb4c5b JA	210	do {
c9fb4c5b JA	211	next_tail++;
679d8352	212	read_barrier();
c3e2fc25	213	if (next_tail == *ring->head)
c9fb4c5b JA	214	break;
c9fb4c5b JA	215
e39c34dc	216	index = tail & sq_ring_mask;
a7086591	217	init_io(s, index);
c3e2fc25	218	ring->array[index] = index;
c9fb4c5b JA	219	prepped++;
	220	tail = next_tail;
	221	} while (prepped < max_ios);
	222
c3e2fc25	223	if (*ring->tail != tail) {
f0403f94	224	/* order tail store with writes to sqes above */
679d8352	225	write_barrier();
c3e2fc25	226	*ring->tail = tail;
679d8352	227	write_barrier();
c9fb4c5b JA	228	}
	229	return prepped;
	230	}
	231
a7086591	232	static int get_file_size(struct file *f)
c9fb4c5b JA	233	{
	234	struct stat st;
	235
48e698fa	236	if (fstat(f->real_fd, &st) < 0)
c9fb4c5b JA	237	return -1;
	238	if (S_ISBLK(st.st_mode)) {
	239	unsigned long long bytes;
	240
48e698fa	241	if (ioctl(f->real_fd, BLKGETSIZE64, &bytes) != 0)
c9fb4c5b JA	242	return -1;
c9fb4c5b JA	243
a7086591	244	f->max_blocks = bytes / BS;
c9fb4c5b JA	245	return 0;
c9fb4c5b JA	246	} else if (S_ISREG(st.st_mode)) {
a7086591	247	f->max_blocks = st.st_size / BS;
c9fb4c5b JA	248	return 0;
	249	}
	250
	251	return -1;
	252	}
	253
	254	static int reap_events(struct submitter *s)
	255	{
e31b8288	256	struct io_cq_ring *ring = &s->cq_ring;
f0403f94	257	struct io_uring_cqe *cqe;
e2239016	258	unsigned head, reaped = 0;
c9fb4c5b	259
c3e2fc25	260	head = *ring->head;
c9fb4c5b	261	do {
701d1277 JA	262	struct file *f;
701d1277 JA	263
679d8352	264	read_barrier();
c3e2fc25	265	if (head == *ring->tail)
c9fb4c5b	266	break;
f0403f94	267	cqe = &ring->cqes[head & cq_ring_mask];
8025517d	268	if (!do_nop) {
e3466352	269	f = (struct file *) (uintptr_t) cqe->user_data;
8025517d JA	270	f->pending_ios--;
	271	if (cqe->res != BS) {
	272	printf("io: unexpected ret=%d\n", cqe->res);
	273	return -1;
	274	}
c9fb4c5b	275	}
f0403f94	276	if (cqe->flags & IOCQE_FLAG_CACHEHIT)
305c06ce JA	277	s->cachehit++;
	278	else
	279	s->cachemiss++;
c9fb4c5b JA	280	reaped++;
c9fb4c5b JA	281	head++;
c9fb4c5b JA	282	} while (1);
	283
	284	s->inflight -= reaped;
c3e2fc25	285	*ring->head = head;
679d8352	286	write_barrier();
c9fb4c5b JA	287	return reaped;
	288	}
	289
	290	static void submitter_fn(void data)
	291	{
	292	struct submitter *s = data;
ce1705de	293	struct io_sq_ring *ring = &s->sq_ring;
a7086591	294	int ret, prepped;
c9fb4c5b JA	295
	296	printf("submitter=%d\n", gettid());
	297
c9fb4c5b JA	298	srand48_r(pthread_self(), &s->rand);
	299
	300	prepped = 0;
	301	do {
f310970e	302	int to_wait, to_submit, this_reap, to_prep;
c9fb4c5b	303
f310970e JA	304	if (!prepped && s->inflight < DEPTH) {
f310970e JA	305	to_prep = min(DEPTH - s->inflight, BATCH_SUBMIT);
a7086591	306	prepped = prep_more_ios(s, to_prep);
f310970e	307	}
c9fb4c5b JA	308	s->inflight += prepped;
	309	submit_more:
	310	to_submit = prepped;
	311	submit:
e4db5bd9	312	if (to_submit && (s->inflight + to_submit <= DEPTH))
c9fb4c5b JA	313	to_wait = 0;
	314	else
	315	to_wait = min(s->inflight + to_submit, BATCH_COMPLETE);
	316
ce1705de JA	317	/*
	318	* Only need to call io_uring_enter if we're not using SQ thread
	319	* poll, or if IORING_SQ_NEED_WAKEUP is set.
	320	*/
	321	if (!sq_thread_poll \|\| (*ring->flags & IORING_SQ_NEED_WAKEUP)) {
e0abe388 JA	322	unsigned flags = 0;
	323
	324	if (to_wait)
	325	flags = IORING_ENTER_GETEVENTS;
	326	ret = io_uring_enter(s, to_submit, to_wait, flags);
ce1705de JA	327	s->calls++;
ce1705de JA	328	}
c9fb4c5b	329
ce1705de JA	330	/*
	331	* For non SQ thread poll, we already got the events we needed
	332	* through the io_uring_enter() above. For SQ thread poll, we
	333	* need to loop here until we find enough events.
	334	*/
	335	this_reap = 0;
	336	do {
	337	int r;
	338	r = reap_events(s);
7d1435e6 JA	339	if (r == -1) {
7d1435e6 JA	340	s->finish = 1;
ce1705de	341	break;
7d1435e6	342	} else if (r > 0)
ce1705de JA	343	this_reap += r;
ce1705de JA	344	} while (sq_thread_poll && this_reap < to_wait);
c9fb4c5b JA	345	s->reaps += this_reap;
	346
	347	if (ret >= 0) {
	348	if (!ret) {
	349	to_submit = 0;
	350	if (s->inflight)
	351	goto submit;
	352	continue;
	353	} else if (ret < to_submit) {
	354	int diff = to_submit - ret;
	355
	356	s->done += ret;
	357	prepped -= diff;
	358	goto submit_more;
	359	}
	360	s->done += ret;
	361	prepped = 0;
	362	continue;
	363	} else if (ret < 0) {
ac122fea	364	if (errno == EAGAIN) {
c9fb4c5b JA	365	if (s->finish)
	366	break;
	367	if (this_reap)
	368	goto submit;
c9fb4c5b JA	369	to_submit = 0;
	370	goto submit;
	371	}
ac122fea	372	printf("io_submit: %s\n", strerror(errno));
c9fb4c5b JA	373	break;
	374	}
	375	} while (!s->finish);
a7086591	376
c9fb4c5b JA	377	finish = 1;
	378	return NULL;
	379	}
	380
	381	static void sig_int(int sig)
	382	{
	383	printf("Exiting on signal %d\n", sig);
	384	submitters[0].finish = 1;
	385	finish = 1;
	386	}
	387
	388	static void arm_sig_int(void)
	389	{
	390	struct sigaction act;
	391
	392	memset(&act, 0, sizeof(act));
	393	act.sa_handler = sig_int;
	394	act.sa_flags = SA_RESTART;
	395	sigaction(SIGINT, &act, NULL);
	396	}
	397
c3e2fc25 JA	398	static int setup_ring(struct submitter *s)
c3e2fc25 JA	399	{
e31b8288 JA	400	struct io_sq_ring *sring = &s->sq_ring;
	401	struct io_cq_ring *cring = &s->cq_ring;
	402	struct io_uring_params p;
2ea53ca3	403	int ret, fd;
c3e2fc25	404	void *ptr;
c3e2fc25 JA	405
	406	memset(&p, 0, sizeof(p));
	407
7d1435e6	408	if (polled && !do_nop)
0e47f11b	409	p.flags \|= IORING_SETUP_IOPOLL;
3d7d00a3	410	if (sq_thread_poll) {
2ea53ca3	411	p.flags \|= IORING_SETUP_SQPOLL;
3ade18a3	412	if (sq_thread_cpu != -1) {
3d7d00a3	413	p.flags \|= IORING_SETUP_SQ_AFF;
3ade18a3 JA	414	p.sq_thread_cpu = sq_thread_cpu;
3ade18a3 JA	415	}
c3e2fc25 JA	416	}
c3e2fc25 JA	417
2ea53ca3	418	fd = io_uring_setup(DEPTH, &p);
c3e2fc25 JA	419	if (fd < 0) {
	420	perror("io_uring_setup");
	421	return 1;
	422	}
f310970e	423	s->ring_fd = fd;
2ea53ca3 JA	424
	425	if (fixedbufs) {
	426	ret = io_uring_register_buffers(s);
	427	if (ret < 0) {
a7abc9fb	428	perror("io_uring_register_buffers");
2ea53ca3 JA	429	return 1;
	430	}
	431	}
	432
8c5fa755 JA	433	if (register_files) {
	434	ret = io_uring_register_files(s);
	435	if (ret < 0) {
	436	perror("io_uring_register_files");
	437	return 1;
	438	}
a7abc9fb JA	439	}
a7abc9fb JA	440
e2239016	441	ptr = mmap(0, p.sq_off.array + p.sq_entries * sizeof(__u32),
f310970e JA	442	PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd,
f310970e JA	443	IORING_OFF_SQ_RING);
c3e2fc25 JA	444	printf("sq_ring ptr = 0x%p\n", ptr);
	445	sring->head = ptr + p.sq_off.head;
	446	sring->tail = ptr + p.sq_off.tail;
	447	sring->ring_mask = ptr + p.sq_off.ring_mask;
	448	sring->ring_entries = ptr + p.sq_off.ring_entries;
ce1705de	449	sring->flags = ptr + p.sq_off.flags;
ac122fea	450	sring->array = ptr + p.sq_off.array;
c3e2fc25 JA	451	sq_ring_mask = *sring->ring_mask;
c3e2fc25 JA	452
f0403f94	453	s->sqes = mmap(0, p.sq_entries * sizeof(struct io_uring_sqe),
f310970e	454	PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd,
f0403f94 JA	455	IORING_OFF_SQES);
f0403f94 JA	456	printf("sqes ptr = 0x%p\n", s->sqes);
c3e2fc25	457
f0403f94	458	ptr = mmap(0, p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe),
f310970e JA	459	PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd,
f310970e JA	460	IORING_OFF_CQ_RING);
c3e2fc25 JA	461	printf("cq_ring ptr = 0x%p\n", ptr);
	462	cring->head = ptr + p.cq_off.head;
	463	cring->tail = ptr + p.cq_off.tail;
	464	cring->ring_mask = ptr + p.cq_off.ring_mask;
	465	cring->ring_entries = ptr + p.cq_off.ring_entries;
f0403f94	466	cring->cqes = ptr + p.cq_off.cqes;
c3e2fc25 JA	467	cq_ring_mask = *cring->ring_mask;
	468	return 0;
	469	}
	470
2e7888ef JA	471	static void file_depths(char *buf)
	472	{
	473	struct submitter *s = &submitters[0];
	474	char *p;
	475	int i;
	476
ac21bfab	477	buf[0] = '\0';
2e7888ef JA	478	p = buf;
	479	for (i = 0; i < s->nr_files; i++) {
	480	struct file *f = &s->files[i];
	481
	482	if (i + 1 == s->nr_files)
	483	p += sprintf(p, "%d", f->pending_ios);
	484	else
	485	p += sprintf(p, "%d, ", f->pending_ios);
	486	}
	487	}
	488
c9fb4c5b JA	489	int main(int argc, char *argv[])
	490	{
	491	struct submitter *s = &submitters[0];
305c06ce	492	unsigned long done, calls, reap, cache_hit, cache_miss;
a7086591	493	int err, i, flags, fd;
2e7888ef	494	char *fdepths;
c3e2fc25	495	void *ret;
c9fb4c5b	496
8025517d	497	if (!do_nop && argc < 2) {
c9fb4c5b JA	498	printf("%s: filename\n", argv[0]);
	499	return 1;
	500	}
	501
701d1277	502	flags = O_RDONLY \| O_NOATIME;
a7086591 JA	503	if (!buffered)
	504	flags \|= O_DIRECT;
	505
	506	i = 1;
8025517d	507	while (!do_nop && i < argc) {
a7086591 JA	508	struct file *f = &s->files[s->nr_files];
	509
	510	fd = open(argv[i], flags);
	511	if (fd < 0) {
	512	perror("open");
	513	return 1;
	514	}
48e698fa	515	f->real_fd = fd;
a7086591 JA	516	if (get_file_size(f)) {
	517	printf("failed getting size of device/file\n");
	518	return 1;
	519	}
	520	if (f->max_blocks <= 1) {
	521	printf("Zero file/device size?\n");
	522	return 1;
	523	}
	524	f->max_blocks--;
	525
	526	printf("Added file %s\n", argv[i]);
	527	s->nr_files++;
	528	i++;
	529	}
	530
b3ce355d JA	531	if (fixedbufs) {
	532	struct rlimit rlim;
	533
	534	rlim.rlim_cur = RLIM_INFINITY;
	535	rlim.rlim_max = RLIM_INFINITY;
	536	if (setrlimit(RLIMIT_MEMLOCK, &rlim) < 0) {
	537	perror("setrlimit");
	538	return 1;
	539	}
c9fb4c5b JA	540	}
	541
	542	arm_sig_int();
	543
c3e2fc25 JA	544	for (i = 0; i < DEPTH; i++) {
c3e2fc25 JA	545	void *buf;
c9fb4c5b	546
c3e2fc25	547	if (posix_memalign(&buf, BS, BS)) {
c9fb4c5b JA	548	printf("failed alloc\n");
	549	return 1;
	550	}
c3e2fc25 JA	551	s->iovecs[i].iov_base = buf;
c3e2fc25 JA	552	s->iovecs[i].iov_len = BS;
c9fb4c5b JA	553	}
c9fb4c5b JA	554
c3e2fc25	555	err = setup_ring(s);
c9fb4c5b	556	if (err) {
c3e2fc25	557	printf("ring setup failed: %s, %d\n", strerror(errno), err);
c9fb4c5b JA	558	return 1;
c9fb4c5b JA	559	}
c3e2fc25 JA	560	printf("polled=%d, fixedbufs=%d, buffered=%d", polled, fixedbufs, buffered);
c3e2fc25 JA	561	printf(" QD=%d, sq_ring=%d, cq_ring=%d\n", DEPTH, s->sq_ring.ring_entries, s->cq_ring.ring_entries);
c9fb4c5b JA	562
	563	pthread_create(&s->thread, NULL, submitter_fn, s);
	564
2e7888ef	565	fdepths = malloc(8 * s->nr_files);
305c06ce	566	cache_hit = cache_miss = reap = calls = done = 0;
c9fb4c5b JA	567	do {
	568	unsigned long this_done = 0;
	569	unsigned long this_reap = 0;
	570	unsigned long this_call = 0;
305c06ce JA	571	unsigned long this_cache_hit = 0;
305c06ce JA	572	unsigned long this_cache_miss = 0;
c9fb4c5b	573	unsigned long rpc = 0, ipc = 0;
305c06ce	574	double hit = 0.0;
c9fb4c5b JA	575
	576	sleep(1);
	577	this_done += s->done;
	578	this_call += s->calls;
	579	this_reap += s->reaps;
305c06ce JA	580	this_cache_hit += s->cachehit;
	581	this_cache_miss += s->cachemiss;
	582	if (this_cache_hit && this_cache_miss) {
	583	unsigned long hits, total;
	584
	585	hits = this_cache_hit - cache_hit;
	586	total = hits + this_cache_miss - cache_miss;
	587	hit = (double) hits / (double) total;
	588	hit *= 100.0;
	589	}
c9fb4c5b JA	590	if (this_call - calls) {
	591	rpc = (this_done - done) / (this_call - calls);
	592	ipc = (this_reap - reap) / (this_call - calls);
191561c1 JA	593	} else
191561c1 JA	594	rpc = ipc = -1;
2e7888ef JA	595	file_depths(fdepths);
2e7888ef JA	596	printf("IOPS=%lu, IOS/call=%ld/%ld, inflight=%u (%s), Cachehit=%0.2f%%\n",
c9fb4c5b	597	this_done - done, rpc, ipc, s->inflight,
2e7888ef	598	fdepths, hit);
c9fb4c5b JA	599	done = this_done;
	600	calls = this_call;
	601	reap = this_reap;
305c06ce JA	602	cache_hit = s->cachehit;
305c06ce JA	603	cache_miss = s->cachemiss;
c9fb4c5b JA	604	} while (!finish);
	605
	606	pthread_join(s->thread, &ret);
e31b8288	607	close(s->ring_fd);
2e7888ef	608	free(fdepths);
c9fb4c5b JA	609	return 0;
c9fb4c5b JA	610	}