summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJens Axboe <axboe@kernel.dk>2022-02-20 11:34:46 -0700
committerJens Axboe <axboe@kernel.dk>2022-05-16 09:30:43 -0600
commit1efee07cc554c0d59ffbc323b577e28bc96fac24 (patch)
treef2eb27a7162e4d2605ed3a3f17f34676238285c2
parentd6927ea5786d3056dabf696c6861cc10a864de79 (diff)
downloadliburing-1efee07cc554c0d59ffbc323b577e28bc96fac24.tar.gz
liburing-1efee07cc554c0d59ffbc323b577e28bc96fac24.tar.bz2
setup: add basic support for SQE128
Adjust the helpers for getting ring size and knowing how much memory we need for a ring of a given size, and basic support for knowing how much to increment a ring when an SQE is allocated. Signed-off-by: Jens Axboe <axboe@kernel.dk>
-rw-r--r--src/include/liburing.h13
-rw-r--r--src/setup.c28
2 files changed, 30 insertions, 11 deletions
diff --git a/src/include/liburing.h b/src/include/liburing.h
index 89b2e5b..eebb23a 100644
--- a/src/include/liburing.h
+++ b/src/include/liburing.h
@@ -969,13 +969,20 @@ static inline struct io_uring_sqe *_io_uring_get_sqe(struct io_uring *ring)
struct io_uring_sq *sq = &ring->sq;
unsigned int head = io_uring_smp_load_acquire(sq->khead);
unsigned int next = sq->sqe_tail + 1;
- struct io_uring_sqe *sqe = NULL;
+ int shift = 0;
+
+ if (ring->flags & IORING_SETUP_SQE128)
+ shift = 1;
if (next - head <= *sq->kring_entries) {
- sqe = &sq->sqes[sq->sqe_tail & *sq->kring_mask];
+ struct io_uring_sqe *sqe;
+
+ sqe = &sq->sqes[(sq->sqe_tail & *sq->kring_mask) << shift];
sq->sqe_tail = next;
+ return sqe;
}
- return sqe;
+
+ return NULL;
}
#ifndef LIBURING_INTERNAL
diff --git a/src/setup.c b/src/setup.c
index 35981da..aec8b33 100644
--- a/src/setup.c
+++ b/src/setup.c
@@ -56,8 +56,10 @@ static int io_uring_mmap(int fd, struct io_uring_params *p,
sq->kdropped = sq->ring_ptr + p->sq_off.dropped;
sq->array = sq->ring_ptr + p->sq_off.array;
- size = p->sq_entries * sizeof(struct io_uring_sqe);
- sq->sqes = __sys_mmap(0, size, PROT_READ | PROT_WRITE,
+ size = sizeof(struct io_uring_sqe);
+ if (p->flags & IORING_SETUP_SQE128)
+ size += 64;
+ sq->sqes = __sys_mmap(0, size * p->sq_entries, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQES);
if (IS_ERR(sq->sqes)) {
ret = PTR_ERR(sq->sqes);
@@ -109,7 +111,10 @@ int io_uring_ring_dontfork(struct io_uring *ring)
if (!ring->sq.ring_ptr || !ring->sq.sqes || !ring->cq.ring_ptr)
return -EINVAL;
- len = *ring->sq.kring_entries * sizeof(struct io_uring_sqe);
+ len = sizeof(struct io_uring_sqe);
+ if (ring->flags & IORING_SETUP_SQE128)
+ len += 64;
+ len *= *ring->sq.kring_entries;
ret = __sys_madvise(ring->sq.sqes, len, MADV_DONTFORK);
if (ret < 0)
return ret;
@@ -166,8 +171,12 @@ void io_uring_queue_exit(struct io_uring *ring)
{
struct io_uring_sq *sq = &ring->sq;
struct io_uring_cq *cq = &ring->cq;
+ size_t sqe_size;
- __sys_munmap(sq->sqes, *sq->kring_entries * sizeof(struct io_uring_sqe));
+ sqe_size = sizeof(struct io_uring_sqe);
+ if (ring->flags & IORING_SETUP_SQE128)
+ sqe_size += 64;
+ __sys_munmap(sq->sqes, sqe_size * *sq->kring_entries);
io_uring_unmap_rings(sq, cq);
/*
* Not strictly required, but frees up the slot we used now rather
@@ -239,8 +248,8 @@ static size_t npages(size_t size, unsigned page_size)
#define KRING_SIZE 320
-static size_t rings_size(unsigned entries, unsigned cq_entries,
- unsigned page_size)
+static size_t rings_size(struct io_uring_params *p, unsigned entries,
+ unsigned cq_entries, unsigned page_size)
{
size_t pages, sq_size, cq_size;
@@ -249,7 +258,10 @@ static size_t rings_size(unsigned entries, unsigned cq_entries,
cq_size = (cq_size + 63) & ~63UL;
pages = (size_t) 1 << npages(cq_size, page_size);
- sq_size = sizeof(struct io_uring_sqe) * entries;
+ sq_size = sizeof(struct io_uring_sqe);
+ if (p->flags & IORING_SETUP_SQE128)
+ sq_size += 64;
+ sq_size *= entries;
pages += (size_t) 1 << npages(sq_size, page_size);
return pages * page_size;
}
@@ -317,7 +329,7 @@ ssize_t io_uring_mlock_size_params(unsigned entries, struct io_uring_params *p)
}
page_size = get_page_size();
- return rings_size(entries, cq_entries, page_size);
+ return rings_size(p, entries, cq_entries, page_size);
}
/*