[fio.git] / t / io_uring.c

#include <stdio.h>
#include <errno.h>
#include <assert.h>
#include <stdlib.h>
#include <stddef.h>
#include <signal.h>
#include <inttypes.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <sys/syscall.h>
#include <sys/resource.h>
#include <sys/mman.h>
#include <sys/uio.h>
#include <linux/fs.h>
#include <fcntl.h>
#include <unistd.h>
#include <string.h>
#include <pthread.h>
#include <sched.h>

#include "../arch/arch.h"
#include "../lib/types.h"
#include "../os/io_uring.h"

#define barrier()	__asm__ __volatile__("": : :"memory")

#define min(a, b)		((a < b) ? (a) : (b))

struct io_sq_ring {
	unsigned *head;
	unsigned *tail;
	unsigned *ring_mask;
	unsigned *ring_entries;
	unsigned *array;
};

struct io_cq_ring {
	unsigned *head;
	unsigned *tail;
	unsigned *ring_mask;
	unsigned *ring_entries;
	struct io_uring_event *events;
};

#define DEPTH			32

#define BATCH_SUBMIT		8
#define BATCH_COMPLETE		8

#define BS			4096

static unsigned sq_ring_mask, cq_ring_mask;

struct submitter {
	pthread_t thread;
	unsigned long max_blocks;
	int ring_fd;
	struct drand48_data rand;
	struct io_sq_ring sq_ring;
	struct io_uring_iocb *iocbs;
	struct iovec iovecs[DEPTH];
	struct io_cq_ring cq_ring;
	int inflight;
	unsigned long reaps;
	unsigned long done;
	unsigned long calls;
	unsigned long cachehit, cachemiss;
	volatile int finish;
	char filename[128];
};

static struct submitter submitters[1];
static volatile int finish;

static int polled = 0;		/* use IO polling */
static int fixedbufs = 0;	/* use fixed user buffers */
static int buffered = 1;	/* use buffered IO, not O_DIRECT */
static int sq_thread = 0;	/* use kernel submission thread */
static int sq_thread_cpu = 0;	/* pin above thread to this CPU */

static int io_uring_setup(unsigned entries, struct iovec *iovecs,
			  struct io_uring_params *p)
{
	return syscall(__NR_sys_io_uring_setup, entries, iovecs, p);
}

static int io_uring_enter(struct submitter *s, unsigned int to_submit,
			  unsigned int min_complete, unsigned int flags)
{
	return syscall(__NR_sys_io_uring_enter, s->ring_fd, to_submit,
			min_complete, flags);
}

static int gettid(void)
{
	return syscall(__NR_gettid);
}

static void init_io(struct submitter *s, int fd, unsigned index)
{
	struct io_uring_iocb *iocb = &s->iocbs[index];
	unsigned long offset;
	long r;

	lrand48_r(&s->rand, &r);
	offset = (r % (s->max_blocks - 1)) * BS;

	if (fixedbufs)
		iocb->opcode = IORING_OP_READ_FIXED;
	else
		iocb->opcode = IORING_OP_READ;
	iocb->flags = 0;
	iocb->ioprio = 0;
	iocb->fd = fd;
	iocb->off = offset;
	iocb->addr = s->iovecs[index].iov_base;
	iocb->len = BS;
}

static int prep_more_ios(struct submitter *s, int fd, int max_ios)
{
	struct io_sq_ring *ring = &s->sq_ring;
	unsigned index, tail, next_tail, prepped = 0;

	next_tail = tail = *ring->tail;
	do {
		next_tail++;
		barrier();
		if (next_tail == *ring->head)
			break;

		index = tail & sq_ring_mask;
		init_io(s, fd, index);
		ring->array[index] = index;
		prepped++;
		tail = next_tail;
	} while (prepped < max_ios);

	if (*ring->tail != tail) {
		/* order tail store with writes to iocbs above */
		barrier();
		*ring->tail = tail;
		barrier();
	}
	return prepped;
}

static int get_file_size(int fd, unsigned long *blocks)
{
	struct stat st;

	if (fstat(fd, &st) < 0)
		return -1;
	if (S_ISBLK(st.st_mode)) {
		unsigned long long bytes;

		if (ioctl(fd, BLKGETSIZE64, &bytes) != 0)
			return -1;

		*blocks = bytes / BS;
		return 0;
	} else if (S_ISREG(st.st_mode)) {
		*blocks = st.st_size / BS;
		return 0;
	}

	return -1;
}

static int reap_events(struct submitter *s)
{
	struct io_cq_ring *ring = &s->cq_ring;
	struct io_uring_event *ev;
	unsigned head, reaped = 0;

	head = *ring->head;
	do {
		barrier();
		if (head == *ring->tail)
			break;
		ev = &ring->events[head & cq_ring_mask];
		if (ev->res != BS) {
			struct io_uring_iocb *iocb = &s->iocbs[ev->index];

			printf("io: unexpected ret=%d\n", ev->res);
			printf("offset=%lu, size=%lu\n",
					(unsigned long) iocb->off,
					(unsigned long) iocb->len);
			return -1;
		}
		if (ev->flags & IOEV_FLAG_CACHEHIT)
			s->cachehit++;
		else
			s->cachemiss++;
		reaped++;
		head++;
	} while (1);

	s->inflight -= reaped;
	*ring->head = head;
	barrier();
	return reaped;
}

static void *submitter_fn(void *data)
{
	struct submitter *s = data;
	int fd, ret, prepped, flags;

	printf("submitter=%d\n", gettid());

	flags = O_RDONLY;
	if (!buffered)
		flags |= O_DIRECT;
	fd = open(s->filename, flags);
	if (fd < 0) {
		perror("open");
		goto done;
	}

	if (get_file_size(fd, &s->max_blocks)) {
		printf("failed getting size of device/file\n");
		goto err;
	}
	if (s->max_blocks <= 1) {
		printf("Zero file/device size?\n");
		goto err;
	}
	s->max_blocks--;

	srand48_r(pthread_self(), &s->rand);

	prepped = 0;
	do {
		int to_wait, to_submit, this_reap, to_prep;

		if (!prepped && s->inflight < DEPTH) {
			to_prep = min(DEPTH - s->inflight, BATCH_SUBMIT);
			prepped = prep_more_ios(s, fd, to_prep);
		}
		s->inflight += prepped;
submit_more:
		to_submit = prepped;
submit:
		if (s->inflight + BATCH_SUBMIT < DEPTH)
			to_wait = 0;
		else
			to_wait = min(s->inflight + to_submit, BATCH_COMPLETE);

		ret = io_uring_enter(s, to_submit, to_wait,
					IORING_ENTER_GETEVENTS);
		s->calls++;

		this_reap = reap_events(s);
		if (this_reap == -1)
			break;
		s->reaps += this_reap;

		if (ret >= 0) {
			if (!ret) {
				to_submit = 0;
				if (s->inflight)
					goto submit;
				continue;
			} else if (ret < to_submit) {
				int diff = to_submit - ret;

				s->done += ret;
				prepped -= diff;
				goto submit_more;
			}
			s->done += ret;
			prepped = 0;
			continue;
		} else if (ret < 0) {
			if (errno == EAGAIN) {
				if (s->finish)
					break;
				if (this_reap)
					goto submit;
				to_submit = 0;
				goto submit;
			}
			printf("io_submit: %s\n", strerror(errno));
			break;
		}
	} while (!s->finish);
err:
	close(fd);
done:
	finish = 1;
	return NULL;
}

static void sig_int(int sig)
{
	printf("Exiting on signal %d\n", sig);
	submitters[0].finish = 1;
	finish = 1;
}

static void arm_sig_int(void)
{
	struct sigaction act;

	memset(&act, 0, sizeof(act));
	act.sa_handler = sig_int;
	act.sa_flags = SA_RESTART;
	sigaction(SIGINT, &act, NULL);
}

static int setup_ring(struct submitter *s)
{
	struct io_sq_ring *sring = &s->sq_ring;
	struct io_cq_ring *cring = &s->cq_ring;
	struct io_uring_params p;
	void *ptr;
	int fd;

	memset(&p, 0, sizeof(p));

	if (polled)
		p.flags |= IORING_SETUP_IOPOLL;
	if (fixedbufs)
		p.flags |= IORING_SETUP_FIXEDBUFS;
	if (buffered)
		p.flags |= IORING_SETUP_SQWQ;
	else if (sq_thread) {
		p.flags |= IORING_SETUP_SQTHREAD;
		p.sq_thread_cpu = sq_thread_cpu;
	}

	if (fixedbufs)
		fd = io_uring_setup(DEPTH, s->iovecs, &p);
	else
		fd = io_uring_setup(DEPTH, NULL, &p);
	if (fd < 0) {
		perror("io_uring_setup");
		return 1;
	}

	s->ring_fd = fd;
	ptr = mmap(0, p.sq_off.array + p.sq_entries * sizeof(__u32),
			PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
			IORING_OFF_SQ_RING);
	printf("sq_ring ptr = 0x%p\n", ptr);
	sring->head = ptr + p.sq_off.head;
	sring->tail = ptr + p.sq_off.tail;
	sring->ring_mask = ptr + p.sq_off.ring_mask;
	sring->ring_entries = ptr + p.sq_off.ring_entries;
	sring->array = ptr + p.sq_off.array;
	sq_ring_mask = *sring->ring_mask;

	s->iocbs = mmap(0, p.sq_entries * sizeof(struct io_uring_iocb),
			PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
			IORING_OFF_IOCB);
	printf("iocbs ptr   = 0x%p\n", s->iocbs);

	ptr = mmap(0, p.cq_off.events + p.cq_entries * sizeof(struct io_uring_event),
			PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
			IORING_OFF_CQ_RING);
	printf("cq_ring ptr = 0x%p\n", ptr);
	cring->head = ptr + p.cq_off.head;
	cring->tail = ptr + p.cq_off.tail;
	cring->ring_mask = ptr + p.cq_off.ring_mask;
	cring->ring_entries = ptr + p.cq_off.ring_entries;
	cring->events = ptr + p.cq_off.events;
	cq_ring_mask = *cring->ring_mask;
	return 0;
}

int main(int argc, char *argv[])
{
	struct submitter *s = &submitters[0];
	unsigned long done, calls, reap, cache_hit, cache_miss;
	int err, i;
	struct rlimit rlim;
	void *ret;

	if (argc < 2) {
		printf("%s: filename\n", argv[0]);
		return 1;
	}

	rlim.rlim_cur = RLIM_INFINITY;
	rlim.rlim_max = RLIM_INFINITY;
	if (setrlimit(RLIMIT_MEMLOCK, &rlim) < 0) {
		perror("setrlimit");
		return 1;
	}

	arm_sig_int();

	for (i = 0; i < DEPTH; i++) {
		void *buf;

		if (posix_memalign(&buf, BS, BS)) {
			printf("failed alloc\n");
			return 1;
		}
		s->iovecs[i].iov_base = buf;
		s->iovecs[i].iov_len = BS;
	}

	err = setup_ring(s);
	if (err) {
		printf("ring setup failed: %s, %d\n", strerror(errno), err);
		return 1;
	}
	printf("polled=%d, fixedbufs=%d, buffered=%d", polled, fixedbufs, buffered);
	printf(" QD=%d, sq_ring=%d, cq_ring=%d\n", DEPTH, *s->sq_ring.ring_entries, *s->cq_ring.ring_entries);
	strcpy(s->filename, argv[1]);

	pthread_create(&s->thread, NULL, submitter_fn, s);

	cache_hit = cache_miss = reap = calls = done = 0;
	do {
		unsigned long this_done = 0;
		unsigned long this_reap = 0;
		unsigned long this_call = 0;
		unsigned long this_cache_hit = 0;
		unsigned long this_cache_miss = 0;
		unsigned long rpc = 0, ipc = 0;
		double hit = 0.0;

		sleep(1);
		this_done += s->done;
		this_call += s->calls;
		this_reap += s->reaps;
		this_cache_hit += s->cachehit;
		this_cache_miss += s->cachemiss;
		if (this_cache_hit && this_cache_miss) {
			unsigned long hits, total;

			hits = this_cache_hit - cache_hit;
			total = hits + this_cache_miss - cache_miss;
			hit = (double) hits / (double) total;
			hit *= 100.0;
		}
		if (this_call - calls) {
			rpc = (this_done - done) / (this_call - calls);
			ipc = (this_reap - reap) / (this_call - calls);
		}
		printf("IOPS=%lu, IOS/call=%lu/%lu, inflight=%u (head=%u tail=%u), Cachehit=%0.2f%%\n",
				this_done - done, rpc, ipc, s->inflight,
				*s->cq_ring.head, *s->cq_ring.tail, hit);
		done = this_done;
		calls = this_call;
		reap = this_reap;
		cache_hit = s->cachehit;
		cache_miss = s->cachemiss;
	} while (!finish);

	pthread_join(s->thread, &ret);
	close(s->ring_fd);
	return 0;
}
Commit	Line	Data
c9fb4c5b JA	1	#include <stdio.h>
	2	#include <errno.h>
	3	#include <assert.h>
	4	#include <stdlib.h>
	5	#include <stddef.h>
	6	#include <signal.h>
	7	#include <inttypes.h>
	8
	9	#include <sys/types.h>
	10	#include <sys/stat.h>
	11	#include <sys/ioctl.h>
	12	#include <sys/syscall.h>
	13	#include <sys/resource.h>
c3e2fc25	14	#include <sys/mman.h>
e31b8288	15	#include <sys/uio.h>
c9fb4c5b JA	16	#include <linux/fs.h>
	17	#include <fcntl.h>
	18	#include <unistd.h>
c9fb4c5b JA	19	#include <string.h>
	20	#include <pthread.h>
	21	#include <sched.h>
	22
74efb029	23	#include "../arch/arch.h"
57fa61f0	24	#include "../lib/types.h"
e31b8288	25	#include "../os/io_uring.h"
ac122fea	26
e31b8288	27	#define barrier() __asm__ __volatile__("": : :"memory")
c3e2fc25	28
e31b8288	29	#define min(a, b) ((a < b) ? (a) : (b))
c3e2fc25	30
e31b8288	31	struct io_sq_ring {
e2239016 JA	32	unsigned *head;
	33	unsigned *tail;
	34	unsigned *ring_mask;
	35	unsigned *ring_entries;
	36	unsigned *array;
c9fb4c5b JA	37	};
c9fb4c5b JA	38
e31b8288	39	struct io_cq_ring {
e2239016 JA	40	unsigned *head;
	41	unsigned *tail;
	42	unsigned *ring_mask;
	43	unsigned *ring_entries;
e31b8288	44	struct io_uring_event *events;
c9fb4c5b JA	45	};
c9fb4c5b JA	46
c9fb4c5b	47	#define DEPTH 32
c9fb4c5b JA	48
	49	#define BATCH_SUBMIT 8
	50	#define BATCH_COMPLETE 8
	51
	52	#define BS 4096
	53
c3e2fc25	54	static unsigned sq_ring_mask, cq_ring_mask;
e39c34dc	55
c9fb4c5b JA	56	struct submitter {
	57	pthread_t thread;
	58	unsigned long max_blocks;
f310970e	59	int ring_fd;
c9fb4c5b	60	struct drand48_data rand;
e31b8288 JA	61	struct io_sq_ring sq_ring;
e31b8288 JA	62	struct io_uring_iocb *iocbs;
c3e2fc25	63	struct iovec iovecs[DEPTH];
e31b8288	64	struct io_cq_ring cq_ring;
c9fb4c5b JA	65	int inflight;
	66	unsigned long reaps;
	67	unsigned long done;
	68	unsigned long calls;
305c06ce	69	unsigned long cachehit, cachemiss;
c9fb4c5b JA	70	volatile int finish;
	71	char filename[128];
	72	};
	73
	74	static struct submitter submitters[1];
	75	static volatile int finish;
	76
c3e2fc25 JA	77	static int polled = 0; /* use IO polling */
	78	static int fixedbufs = 0; /* use fixed user buffers */
	79	static int buffered = 1; /* use buffered IO, not O_DIRECT */
0527b23f JA	80	static int sq_thread = 0; /* use kernel submission thread */
0527b23f JA	81	static int sq_thread_cpu = 0; /* pin above thread to this CPU */
c9fb4c5b	82
c3e2fc25	83	static int io_uring_setup(unsigned entries, struct iovec *iovecs,
e31b8288	84	struct io_uring_params *p)
c9fb4c5b	85	{
74efb029	86	return syscall(__NR_sys_io_uring_setup, entries, iovecs, p);
c9fb4c5b JA	87	}
c9fb4c5b JA	88
c3e2fc25 JA	89	static int io_uring_enter(struct submitter *s, unsigned int to_submit,
c3e2fc25 JA	90	unsigned int min_complete, unsigned int flags)
c9fb4c5b	91	{
f310970e JA	92	return syscall(__NR_sys_io_uring_enter, s->ring_fd, to_submit,
f310970e JA	93	min_complete, flags);
c9fb4c5b JA	94	}
	95
	96	static int gettid(void)
	97	{
	98	return syscall(__NR_gettid);
	99	}
	100
c3e2fc25	101	static void init_io(struct submitter *s, int fd, unsigned index)
c9fb4c5b	102	{
e31b8288	103	struct io_uring_iocb *iocb = &s->iocbs[index];
c9fb4c5b JA	104	unsigned long offset;
	105	long r;
	106
	107	lrand48_r(&s->rand, &r);
	108	offset = (r % (s->max_blocks - 1)) * BS;
	109
fb191295 JA	110	if (fixedbufs)
	111	iocb->opcode = IORING_OP_READ_FIXED;
	112	else
	113	iocb->opcode = IORING_OP_READ;
e31b8288 JA	114	iocb->flags = 0;
	115	iocb->ioprio = 0;
	116	iocb->fd = fd;
	117	iocb->off = offset;
	118	iocb->addr = s->iovecs[index].iov_base;
	119	iocb->len = BS;
c9fb4c5b JA	120	}
	121
	122	static int prep_more_ios(struct submitter *s, int fd, int max_ios)
	123	{
e31b8288	124	struct io_sq_ring *ring = &s->sq_ring;
e2239016	125	unsigned index, tail, next_tail, prepped = 0;
c9fb4c5b	126
c3e2fc25	127	next_tail = tail = *ring->tail;
c9fb4c5b JA	128	do {
c9fb4c5b JA	129	next_tail++;
c9fb4c5b	130	barrier();
c3e2fc25	131	if (next_tail == *ring->head)
c9fb4c5b JA	132	break;
c9fb4c5b JA	133
e39c34dc	134	index = tail & sq_ring_mask;
c3e2fc25 JA	135	init_io(s, fd, index);
c3e2fc25 JA	136	ring->array[index] = index;
c9fb4c5b JA	137	prepped++;
	138	tail = next_tail;
	139	} while (prepped < max_ios);
	140
c3e2fc25	141	if (*ring->tail != tail) {
c9fb4c5b JA	142	/* order tail store with writes to iocbs above */
c9fb4c5b JA	143	barrier();
c3e2fc25	144	*ring->tail = tail;
c9fb4c5b JA	145	barrier();
	146	}
	147	return prepped;
	148	}
	149
	150	static int get_file_size(int fd, unsigned long *blocks)
	151	{
	152	struct stat st;
	153
	154	if (fstat(fd, &st) < 0)
	155	return -1;
	156	if (S_ISBLK(st.st_mode)) {
	157	unsigned long long bytes;
	158
	159	if (ioctl(fd, BLKGETSIZE64, &bytes) != 0)
	160	return -1;
	161
	162	*blocks = bytes / BS;
	163	return 0;
	164	} else if (S_ISREG(st.st_mode)) {
	165	*blocks = st.st_size / BS;
	166	return 0;
	167	}
	168
	169	return -1;
	170	}
	171
	172	static int reap_events(struct submitter *s)
	173	{
e31b8288 JA	174	struct io_cq_ring *ring = &s->cq_ring;
e31b8288 JA	175	struct io_uring_event *ev;
e2239016	176	unsigned head, reaped = 0;
c9fb4c5b	177
c3e2fc25	178	head = *ring->head;
c9fb4c5b JA	179	do {
c9fb4c5b JA	180	barrier();
c3e2fc25	181	if (head == *ring->tail)
c9fb4c5b	182	break;
e39c34dc	183	ev = &ring->events[head & cq_ring_mask];
c9fb4c5b	184	if (ev->res != BS) {
e31b8288	185	struct io_uring_iocb *iocb = &s->iocbs[ev->index];
c9fb4c5b	186
e31b8288	187	printf("io: unexpected ret=%d\n", ev->res);
f310970e	188	printf("offset=%lu, size=%lu\n",
e31b8288 JA	189	(unsigned long) iocb->off,
e31b8288 JA	190	(unsigned long) iocb->len);
c9fb4c5b JA	191	return -1;
c9fb4c5b JA	192	}
e31b8288	193	if (ev->flags & IOEV_FLAG_CACHEHIT)
305c06ce JA	194	s->cachehit++;
	195	else
	196	s->cachemiss++;
c9fb4c5b JA	197	reaped++;
c9fb4c5b JA	198	head++;
c9fb4c5b JA	199	} while (1);
	200
	201	s->inflight -= reaped;
c3e2fc25	202	*ring->head = head;
c9fb4c5b JA	203	barrier();
	204	return reaped;
	205	}
	206
	207	static void submitter_fn(void data)
	208	{
	209	struct submitter *s = data;
	210	int fd, ret, prepped, flags;
	211
	212	printf("submitter=%d\n", gettid());
	213
	214	flags = O_RDONLY;
	215	if (!buffered)
	216	flags \|= O_DIRECT;
	217	fd = open(s->filename, flags);
	218	if (fd < 0) {
	219	perror("open");
	220	goto done;
	221	}
	222
	223	if (get_file_size(fd, &s->max_blocks)) {
	224	printf("failed getting size of device/file\n");
	225	goto err;
	226	}
f310970e	227	if (s->max_blocks <= 1) {
c9fb4c5b JA	228	printf("Zero file/device size?\n");
	229	goto err;
	230	}
c9fb4c5b JA	231	s->max_blocks--;
	232
	233	srand48_r(pthread_self(), &s->rand);
	234
	235	prepped = 0;
	236	do {
f310970e	237	int to_wait, to_submit, this_reap, to_prep;
c9fb4c5b	238
f310970e JA	239	if (!prepped && s->inflight < DEPTH) {
	240	to_prep = min(DEPTH - s->inflight, BATCH_SUBMIT);
	241	prepped = prep_more_ios(s, fd, to_prep);
	242	}
c9fb4c5b JA	243	s->inflight += prepped;
	244	submit_more:
	245	to_submit = prepped;
	246	submit:
	247	if (s->inflight + BATCH_SUBMIT < DEPTH)
	248	to_wait = 0;
	249	else
	250	to_wait = min(s->inflight + to_submit, BATCH_COMPLETE);
	251
f310970e JA	252	ret = io_uring_enter(s, to_submit, to_wait,
f310970e JA	253	IORING_ENTER_GETEVENTS);
c9fb4c5b JA	254	s->calls++;
	255
	256	this_reap = reap_events(s);
	257	if (this_reap == -1)
	258	break;
	259	s->reaps += this_reap;
	260
	261	if (ret >= 0) {
	262	if (!ret) {
	263	to_submit = 0;
	264	if (s->inflight)
	265	goto submit;
	266	continue;
	267	} else if (ret < to_submit) {
	268	int diff = to_submit - ret;
	269
	270	s->done += ret;
	271	prepped -= diff;
	272	goto submit_more;
	273	}
	274	s->done += ret;
	275	prepped = 0;
	276	continue;
	277	} else if (ret < 0) {
ac122fea	278	if (errno == EAGAIN) {
c9fb4c5b JA	279	if (s->finish)
	280	break;
	281	if (this_reap)
	282	goto submit;
c9fb4c5b JA	283	to_submit = 0;
	284	goto submit;
	285	}
ac122fea	286	printf("io_submit: %s\n", strerror(errno));
c9fb4c5b JA	287	break;
	288	}
	289	} while (!s->finish);
	290	err:
	291	close(fd);
	292	done:
	293	finish = 1;
	294	return NULL;
	295	}
	296
	297	static void sig_int(int sig)
	298	{
	299	printf("Exiting on signal %d\n", sig);
	300	submitters[0].finish = 1;
	301	finish = 1;
	302	}
	303
	304	static void arm_sig_int(void)
	305	{
	306	struct sigaction act;
	307
	308	memset(&act, 0, sizeof(act));
	309	act.sa_handler = sig_int;
	310	act.sa_flags = SA_RESTART;
	311	sigaction(SIGINT, &act, NULL);
	312	}
	313
c3e2fc25 JA	314	static int setup_ring(struct submitter *s)
c3e2fc25 JA	315	{
e31b8288 JA	316	struct io_sq_ring *sring = &s->sq_ring;
	317	struct io_cq_ring *cring = &s->cq_ring;
	318	struct io_uring_params p;
c3e2fc25 JA	319	void *ptr;
	320	int fd;
	321
	322	memset(&p, 0, sizeof(p));
	323
c3e2fc25	324	if (polled)
e31b8288	325	p.flags \|= IORING_SETUP_IOPOLL;
c3e2fc25	326	if (fixedbufs)
e31b8288	327	p.flags \|= IORING_SETUP_FIXEDBUFS;
c3e2fc25	328	if (buffered)
e31b8288	329	p.flags \|= IORING_SETUP_SQWQ;
c3e2fc25	330	else if (sq_thread) {
e31b8288	331	p.flags \|= IORING_SETUP_SQTHREAD;
c3e2fc25 JA	332	p.sq_thread_cpu = sq_thread_cpu;
	333	}
	334
	335	if (fixedbufs)
	336	fd = io_uring_setup(DEPTH, s->iovecs, &p);
	337	else
	338	fd = io_uring_setup(DEPTH, NULL, &p);
	339	if (fd < 0) {
	340	perror("io_uring_setup");
	341	return 1;
	342	}
	343
f310970e	344	s->ring_fd = fd;
e2239016	345	ptr = mmap(0, p.sq_off.array + p.sq_entries * sizeof(__u32),
f310970e JA	346	PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd,
f310970e JA	347	IORING_OFF_SQ_RING);
c3e2fc25 JA	348	printf("sq_ring ptr = 0x%p\n", ptr);
	349	sring->head = ptr + p.sq_off.head;
	350	sring->tail = ptr + p.sq_off.tail;
	351	sring->ring_mask = ptr + p.sq_off.ring_mask;
	352	sring->ring_entries = ptr + p.sq_off.ring_entries;
ac122fea	353	sring->array = ptr + p.sq_off.array;
c3e2fc25 JA	354	sq_ring_mask = *sring->ring_mask;
c3e2fc25 JA	355
e31b8288	356	s->iocbs = mmap(0, p.sq_entries * sizeof(struct io_uring_iocb),
f310970e JA	357	PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd,
f310970e JA	358	IORING_OFF_IOCB);
c3e2fc25 JA	359	printf("iocbs ptr = 0x%p\n", s->iocbs);
c3e2fc25 JA	360
e31b8288	361	ptr = mmap(0, p.cq_off.events + p.cq_entries * sizeof(struct io_uring_event),
f310970e JA	362	PROT_READ \| PROT_WRITE, MAP_SHARED \| MAP_POPULATE, fd,
f310970e JA	363	IORING_OFF_CQ_RING);
c3e2fc25 JA	364	printf("cq_ring ptr = 0x%p\n", ptr);
	365	cring->head = ptr + p.cq_off.head;
	366	cring->tail = ptr + p.cq_off.tail;
	367	cring->ring_mask = ptr + p.cq_off.ring_mask;
	368	cring->ring_entries = ptr + p.cq_off.ring_entries;
ac122fea	369	cring->events = ptr + p.cq_off.events;
c3e2fc25 JA	370	cq_ring_mask = *cring->ring_mask;
	371	return 0;
	372	}
	373
c9fb4c5b JA	374	int main(int argc, char *argv[])
	375	{
	376	struct submitter *s = &submitters[0];
305c06ce	377	unsigned long done, calls, reap, cache_hit, cache_miss;
c3e2fc25	378	int err, i;
c9fb4c5b	379	struct rlimit rlim;
c3e2fc25	380	void *ret;
c9fb4c5b JA	381
	382	if (argc < 2) {
	383	printf("%s: filename\n", argv[0]);
	384	return 1;
	385	}
	386
	387	rlim.rlim_cur = RLIM_INFINITY;
	388	rlim.rlim_max = RLIM_INFINITY;
	389	if (setrlimit(RLIMIT_MEMLOCK, &rlim) < 0) {
	390	perror("setrlimit");
	391	return 1;
	392	}
	393
	394	arm_sig_int();
	395
c3e2fc25 JA	396	for (i = 0; i < DEPTH; i++) {
c3e2fc25 JA	397	void *buf;
c9fb4c5b	398
c3e2fc25	399	if (posix_memalign(&buf, BS, BS)) {
c9fb4c5b JA	400	printf("failed alloc\n");
	401	return 1;
	402	}
c3e2fc25 JA	403	s->iovecs[i].iov_base = buf;
c3e2fc25 JA	404	s->iovecs[i].iov_len = BS;
c9fb4c5b JA	405	}
c9fb4c5b JA	406
c3e2fc25	407	err = setup_ring(s);
c9fb4c5b	408	if (err) {
c3e2fc25	409	printf("ring setup failed: %s, %d\n", strerror(errno), err);
c9fb4c5b JA	410	return 1;
c9fb4c5b JA	411	}
c3e2fc25 JA	412	printf("polled=%d, fixedbufs=%d, buffered=%d", polled, fixedbufs, buffered);
c3e2fc25 JA	413	printf(" QD=%d, sq_ring=%d, cq_ring=%d\n", DEPTH, s->sq_ring.ring_entries, s->cq_ring.ring_entries);
c9fb4c5b JA	414	strcpy(s->filename, argv[1]);
	415
	416	pthread_create(&s->thread, NULL, submitter_fn, s);
	417
305c06ce	418	cache_hit = cache_miss = reap = calls = done = 0;
c9fb4c5b JA	419	do {
	420	unsigned long this_done = 0;
	421	unsigned long this_reap = 0;
	422	unsigned long this_call = 0;
305c06ce JA	423	unsigned long this_cache_hit = 0;
305c06ce JA	424	unsigned long this_cache_miss = 0;
c9fb4c5b	425	unsigned long rpc = 0, ipc = 0;
305c06ce	426	double hit = 0.0;
c9fb4c5b JA	427
	428	sleep(1);
	429	this_done += s->done;
	430	this_call += s->calls;
	431	this_reap += s->reaps;
305c06ce JA	432	this_cache_hit += s->cachehit;
	433	this_cache_miss += s->cachemiss;
	434	if (this_cache_hit && this_cache_miss) {
	435	unsigned long hits, total;
	436
	437	hits = this_cache_hit - cache_hit;
	438	total = hits + this_cache_miss - cache_miss;
	439	hit = (double) hits / (double) total;
	440	hit *= 100.0;
	441	}
c9fb4c5b JA	442	if (this_call - calls) {
	443	rpc = (this_done - done) / (this_call - calls);
	444	ipc = (this_reap - reap) / (this_call - calls);
	445	}
10c4d131	446	printf("IOPS=%lu, IOS/call=%lu/%lu, inflight=%u (head=%u tail=%u), Cachehit=%0.2f%%\n",
c9fb4c5b	447	this_done - done, rpc, ipc, s->inflight,
c3e2fc25	448	s->cq_ring.head, s->cq_ring.tail, hit);
c9fb4c5b JA	449	done = this_done;
	450	calls = this_call;
	451	reap = this_reap;
305c06ce JA	452	cache_hit = s->cachehit;
305c06ce JA	453	cache_miss = s->cachemiss;
c9fb4c5b JA	454	} while (!finish);
	455
	456	pthread_join(s->thread, &ret);
e31b8288	457	close(s->ring_fd);
c9fb4c5b JA	458	return 0;
c9fb4c5b JA	459	}