io_uring/chan: grab reference to destination ring

Grab a reference to the destination ring, and store it in the
io_queue_chan struct. When either end of the channel gets closed, eg
when either of the rings go away, then the reference is dropped
unconditionally. Hence the stored ctx must only be used within a RCU
read lock region.

Note that channels only support DEFER_TASKRUN targets, and such rings
will call synchronize_rcu() before the ring (and task) can go away. This
makes it safe to use ->dst_ring inside rcu_read_lock(), even though the
reference to the ring is put during unregistration.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/chan.c b/io_uring/chan.c
index b4f16aec2ffaef7a8ad4a4e509ae5318900710ea..e205bfd1f2861705d7ec583d59b0fc2ffdacb340 100644 (file)
--- a/io_uring/chan.c
+++ b/io_uring/chan.c
@@ -33,14 +33,20 @@ void io_unregister_queue_chans(struct io_ring_ctx *ctx)
 
        lockdep_assert_held(&ctx->uring_lock);
 
+       rcu_read_lock();
        xa_for_each(&ctx->xa_src_chan, index, c) {
                if (atomic_dec_and_test(&c->refs))
                        kfree_rcu(c, rcu_head);
        }
        xa_for_each(&ctx->xa_dst_chan, index, c) {
+               if (rcu_dereference(c->dst_ring) == ctx) {
+                       percpu_ref_put(&ctx->refs);
+                       rcu_assign_pointer(c->dst_ring, NULL);
+               }
                if (atomic_dec_and_test(&c->refs))
                        kfree_rcu(c, rcu_head);
        }
+       rcu_read_unlock();
        xa_destroy(&ctx->xa_src_chan);
        xa_destroy(&ctx->xa_dst_chan);
 }
@@ -93,6 +99,8 @@ static struct io_queue_chan *__io_register_queue_chan(struct io_ring_ctx *ctx,
                return ERR_PTR(ret);
        }
 
+       percpu_ref_get(&dst->refs);
+       rcu_assign_pointer(c->dst_ring, dst);
        return c;
 }
 

diff --git a/io_uring/chan.h b/io_uring/chan.h
index cea16b5d2f4e36813c413a7ee064f20710946d98..a8dc962fc61f4363643335acd81c4497d0e1833c 100644 (file)
--- a/io_uring/chan.h
+++ b/io_uring/chan.h
@@ -12,6 +12,7 @@ struct io_queue_chan {
        __u32                           mask;
        __u32                           tail;
        __u32                           resp_id;
+       struct io_ring_ctx __rcu        *dst_ring;
        struct rcu_head                 rcu_head;
        struct io_queue_chan_entry      data[];
 };