X-Git-Url: https://git.kernel.dk/?a=blobdiff_plain;f=engines%2Fio_uring.c;h=9bcfec1726b05402bbc0875b2a53bf3edfc13e52;hb=576d4cf9b040e49ca467cd94db0567d392510e9d;hp=56af8d7181321c3ff0768687e43766c134d48e61;hpb=e3466352e66f2ab9f351b544a569158d0d6cfa10;p=fio.git

diff --git a/engines/io_uring.c b/engines/io_uring.c
index 56af8d71..9bcfec17 100644
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -62,9 +62,6 @@ struct ioring_data {
 	int cq_ring_off;
 	unsigned iodepth;
 
-	uint64_t cachehit;
-	uint64_t cachemiss;
-
 	struct ioring_mmap mmap[3];
 };
 
@@ -132,7 +129,7 @@ static int io_uring_enter(struct ioring_data *ld, unsigned int to_submit,
 			 unsigned int min_complete, unsigned int flags)
 {
 	return syscall(__NR_sys_io_uring_enter, ld->ring_fd, to_submit,
-			min_complete, flags);
+			min_complete, flags, NULL, 0);
 }
 
 static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u)
@@ -154,7 +151,7 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u)
 				sqe->opcode = IORING_OP_READ_FIXED;
 			else
 				sqe->opcode = IORING_OP_WRITE_FIXED;
-			sqe->addr = io_u->xfer_buf;
+			sqe->addr = (unsigned long) io_u->xfer_buf;
 			sqe->len = io_u->xfer_buflen;
 			sqe->buf_index = io_u->index;
 		} else {
@@ -162,7 +159,7 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u)
 				sqe->opcode = IORING_OP_READV;
 			else
 				sqe->opcode = IORING_OP_WRITEV;
-			sqe->addr = &ld->iovecs[io_u->index];
+			sqe->addr = (unsigned long) &ld->iovecs[io_u->index];
 			sqe->len = 1;
 		}
 		sqe->off = io_u->offset;
@@ -197,13 +194,6 @@ static struct io_u *fio_ioring_event(struct thread_data *td, int event)
 	} else
 		io_u->error = 0;
 
-	if (io_u->ddir == DDIR_READ) {
-		if (cqe->flags & IOCQE_FLAG_CACHEHIT)
-			ld->cachehit++;
-		else
-			ld->cachemiss++;
-	}
-
 	return io_u;
 }
 
@@ -243,6 +233,8 @@ static int fio_ioring_getevents(struct thread_data *td, unsigned int min,
 		r = fio_ioring_cqring_reap(td, events, max);
 		if (r) {
 			events += r;
+			if (actual_min != 0)
+				actual_min -= r;
 			continue;
 		}
 
@@ -289,6 +281,8 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td,
 	if (next_tail == *ring->head)
 		return FIO_Q_BUSY;
 
+	/* ensure sqe stores are ordered with tail update */
+	write_barrier();
 	ring->array[tail & ld->sq_ring_mask] = io_u->index;
 	*ring->tail = next_tail;
 	write_barrier();
@@ -338,7 +332,8 @@ static int fio_ioring_commit(struct thread_data *td)
 
 		read_barrier();
 		if (*ring->flags & IORING_SQ_NEED_WAKEUP)
-			io_uring_enter(ld, ld->queued, 0, 0);
+			io_uring_enter(ld, ld->queued, 0,
+					IORING_ENTER_SQ_WAKEUP);
 		ld->queued = 0;
 		return 0;
 	}
@@ -388,9 +383,6 @@ static void fio_ioring_cleanup(struct thread_data *td)
 	struct ioring_data *ld = td->io_ops_data;
 
 	if (ld) {
-		td->ts.cachehit += ld->cachehit;
-		td->ts.cachemiss += ld->cachemiss;
-
 		if (!(td->flags & TD_F_CHILD))
 			fio_ioring_unmap(ld);
 
@@ -460,15 +452,6 @@ static int fio_ioring_queue_init(struct thread_data *td)
 		}
 	}
 
-	if (o->fixedbufs) {
-		struct rlimit rlim = {
-			.rlim_cur = RLIM_INFINITY,
-			.rlim_max = RLIM_INFINITY,
-		};
-
-		setrlimit(RLIMIT_MEMLOCK, &rlim);
-	}
-
 	ret = syscall(__NR_sys_io_uring_setup, depth, &p);
 	if (ret < 0)
 		return ret;
@@ -476,13 +459,16 @@ static int fio_ioring_queue_init(struct thread_data *td)
 	ld->ring_fd = ret;
 
 	if (o->fixedbufs) {
-		struct io_uring_register_buffers reg = {
-			.iovecs = ld->iovecs,
-			.nr_iovecs = depth
+		struct rlimit rlim = {
+			.rlim_cur = RLIM_INFINITY,
+			.rlim_max = RLIM_INFINITY,
 		};
 
+		if (setrlimit(RLIMIT_MEMLOCK, &rlim) < 0)
+			return -1;
+
 		ret = syscall(__NR_sys_io_uring_register, ld->ring_fd,
-				IORING_REGISTER_BUFFERS, &reg);
+				IORING_REGISTER_BUFFERS, ld->iovecs, depth);
 		if (ret < 0)
 			return ret;
 	}
@@ -547,6 +533,7 @@ static int fio_ioring_io_u_init(struct thread_data *td, struct io_u *io_u)
 static struct ioengine_ops ioengine = {
 	.name			= "io_uring",
 	.version		= FIO_IOOPS_VERSION,
+	.flags			= FIO_ASYNCIO_SYNC_TRIM,
 	.init			= fio_ioring_init,
 	.post_init		= fio_ioring_post_init,
 	.io_u_init		= fio_ioring_io_u_init,