io_uring/chan: cache consumer head loads

Posting a message on the channel currently requires reading the
destination to know how far along it is. But in practice, this only
needs to be done every time the tail has caught up.

Initialize a cached_head to be that of the ring size, and use the cached
head when posting an event. If the cached entries are used up, do a
proper c->head read and update the cached_head again.

This greatly reduces the cross traffic on the posting side, by avoiding
pulling in the consumer ring head entry until it's required.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/chan.c b/io_uring/chan.c
index ebda7179544fe49509b76af653cfd5df7e26100c..3d1454e8814b3b2339b57247f73b85bfcc2d96a2 100644 (file)
--- a/io_uring/chan.c
+++ b/io_uring/chan.c
@@ -94,6 +94,8 @@ static struct io_queue_chan *__io_register_queue_chan(struct io_ring_ctx *ctx,
        atomic_set(&c->refs, 2);
        c->nentries = chan->nentries;
        c->mask = chan->nentries - 1;
+       c->cached_head = 0;
+       c->cached_nr = chan->nentries;
        c->req.ctx = dst;
 
        ret = xa_alloc(&ctx->xa_src_chan, &ids->src_id, c, lim, GFP_KERNEL_ACCOUNT);
@@ -358,8 +360,20 @@ is_dead:
                goto err;
        }
 
-       head = smp_load_acquire(&c->head);
+       /*
+        * If we have cached head entries, there's no need to pull in the
+        * cacheline that the consumer dirties all the time. If no more
+        * cached entries exists, acquire the actual head and update the
+        * cached count.
+        */
        tail = c->tail;
+       if (c->cached_nr) {
+               head = c->cached_head++;
+               c->cached_nr--;
+       } else {
+               head = c->cached_head = smp_load_acquire(&c->head);
+               c->cached_nr = (tail - head) & c->mask;
+       }
        if (tail - head >= c->nentries) {
                ret = -EXFULL;
                goto err;

diff --git a/io_uring/chan.h b/io_uring/chan.h
index d09b449e6d46e3d462bd1e73f14123c44eb49410..d7c4ca4b69e82ec3070dba4335fd800245ef0012 100644 (file)
--- a/io_uring/chan.h
+++ b/io_uring/chan.h
@@ -19,6 +19,8 @@ struct io_queue_chan {
        __u32                           nentries;
        __u32                           mask;
        __u32                           tail;
+       __u32                           cached_head;
+       __u32                           cached_nr;
        __u32                           resp_id;
        atomic_t                        flags;
        struct io_ring_ctx __rcu        *dst_ring;