net: extend ubuf_info callback to ops structure

We'll need to associate additional callbacks with ubuf_info, introduce
a structure holding ubuf_info callbacks. Apart from a more smarter
io_uring notification management introduced in next patches, it can be
used to generalise msg_zerocopy_put_abort() and also store
->sg_from_iter, which is currently passed in struct msghdr.

Reviewed-by: Jens Axboe <axboe@kernel.dk>
Reviewed-by: David Ahern <dsahern@kernel.org>
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Link: https://lore.kernel.org/all/a62015541de49c0e2a8a0377a1d5d0a5aeb07016.1713369317.git.asml.silence@gmail.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/drivers/net/tap.c b/drivers/net/tap.c
index 9f0495e8df4d0b34535c58c03aca56008c901ea4..bfdd3875fe86535d991d958f08b3fd1531af1045 100644 (file)
--- a/drivers/net/tap.c
+++ b/drivers/net/tap.c
@@ -754,7 +754,7 @@ static ssize_t tap_get_user(struct tap_queue *q, void *msg_control,
                skb_zcopy_init(skb, msg_control);
        } else if (msg_control) {
                struct ubuf_info *uarg = msg_control;
-               uarg->callback(NULL, uarg, false);
+               uarg->ops->complete(NULL, uarg, false);
        }
 
        dev_queue_xmit(skb);

diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index 0b3f21cba552f27221a2c7fbe8147feb34e69724..b7401d990680453919f83f3fdec0ad1c3c90af35 100644 (file)
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -1906,7 +1906,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
                skb_zcopy_init(skb, msg_control);
        } else if (msg_control) {
                struct ubuf_info *uarg = msg_control;
-               uarg->callback(NULL, uarg, false);
+               uarg->ops->complete(NULL, uarg, false);
        }
 
        skb_reset_network_header(skb);

diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h
index 1fcbd83f7ff2e3e46e0c230da0ea60f0665e7e78..17421da139f2b7f6ab97cb299dc0d605293569d9 100644 (file)
--- a/drivers/net/xen-netback/common.h
+++ b/drivers/net/xen-netback/common.h
@@ -390,9 +390,8 @@ bool xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb);
 
 void xenvif_carrier_on(struct xenvif *vif);
 
-/* Callback from stack when TX packet can be released */
-void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf,
-                             bool zerocopy_success);
+/* Callbacks from stack when TX packet can be released */
+extern const struct ubuf_info_ops xenvif_ubuf_ops;
 
 static inline pending_ring_idx_t nr_pending_reqs(struct xenvif_queue *queue)
 {

diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c
index 7cff90aa8d24c280cdd36afad282fb08877f8d86..65db5f14465f3e1526c42ec0dffbf64d0160d46c 100644 (file)
--- a/drivers/net/xen-netback/interface.c
+++ b/drivers/net/xen-netback/interface.c
@@ -593,7 +593,7 @@ int xenvif_init_queue(struct xenvif_queue *queue)
 
        for (i = 0; i < MAX_PENDING_REQS; i++) {
                queue->pending_tx_info[i].callback_struct = (struct ubuf_info_msgzc)
-                       { { .callback = xenvif_zerocopy_callback },
+                       { { .ops = &xenvif_ubuf_ops },
                          { { .ctx = NULL,
                              .desc = i } } };
                queue->grant_tx_handle[i] = NETBACK_INVALID_HANDLE;

diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c
index ef76850d9bcd232e84f00c4576c5a98ace51458f..bab7e43ea05b5a123aef18bb531cdf41dee1bfff 100644 (file)
--- a/drivers/net/xen-netback/netback.c
+++ b/drivers/net/xen-netback/netback.c
@@ -1156,7 +1156,7 @@ static int xenvif_handle_frag_list(struct xenvif_queue *queue, struct sk_buff *s
        uarg = skb_shinfo(skb)->destructor_arg;
        /* increase inflight counter to offset decrement in callback */
        atomic_inc(&queue->inflight_packets);
-       uarg->callback(NULL, uarg, true);
+       uarg->ops->complete(NULL, uarg, true);
        skb_shinfo(skb)->destructor_arg = NULL;
 
        /* Fill the skb with the new (local) frags. */
@@ -1278,8 +1278,9 @@ static int xenvif_tx_submit(struct xenvif_queue *queue)
        return work_done;
 }
 
-void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base,
-                             bool zerocopy_success)
+static void xenvif_zerocopy_callback(struct sk_buff *skb,
+                                    struct ubuf_info *ubuf_base,
+                                    bool zerocopy_success)
 {
        unsigned long flags;
        pending_ring_idx_t index;
@@ -1312,6 +1313,10 @@ void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base,
        xenvif_skb_zerocopy_complete(queue);
 }
 
+const struct ubuf_info_ops xenvif_ubuf_ops = {
+       .complete = xenvif_zerocopy_callback,
+};
+
 static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue)
 {
        struct gnttab_unmap_grant_ref *gop;

diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index c64ded183f8dd8eb13442185f5780de737b0edda..f16279351db56e8a55db21c12c27742877ae0e71 100644 (file)
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -380,7 +380,7 @@ static void vhost_zerocopy_signal_used(struct vhost_net *net,
        }
 }
 
-static void vhost_zerocopy_callback(struct sk_buff *skb,
+static void vhost_zerocopy_complete(struct sk_buff *skb,
                                    struct ubuf_info *ubuf_base, bool success)
 {
        struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base);
@@ -408,6 +408,10 @@ static void vhost_zerocopy_callback(struct sk_buff *skb,
        rcu_read_unlock_bh();
 }
 
+static const struct ubuf_info_ops vhost_ubuf_ops = {
+       .complete = vhost_zerocopy_complete,
+};
+
 static inline unsigned long busy_clock(void)
 {
        return local_clock() >> 10;
@@ -879,7 +883,7 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
                        vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
                        ubuf->ctx = nvq->ubufs;
                        ubuf->desc = nvq->upend_idx;
-                       ubuf->ubuf.callback = vhost_zerocopy_callback;
+                       ubuf->ubuf.ops = &vhost_ubuf_ops;
                        ubuf->ubuf.flags = SKBFL_ZEROCOPY_FRAG;
                        refcount_set(&ubuf->ubuf.refcnt, 1);
                        msg.msg_control = &ctl;

diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 9d24aec064e888316b5a5cf450cf25ffdd391183..a110e97e074a0c9eafaa90ddb6fc165675a95711 100644 (file)
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -527,6 +527,11 @@ enum {
 #define SKBFL_ALL_ZEROCOPY     (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \
                                 SKBFL_DONT_ORPHAN | SKBFL_MANAGED_FRAG_REFS)
 
+struct ubuf_info_ops {
+       void (*complete)(struct sk_buff *, struct ubuf_info *,
+                        bool zerocopy_success);
+};
+
 /*
  * The callback notifies userspace to release buffers when skb DMA is done in
  * lower device, the skb last reference should be 0 when calling this.
@@ -536,8 +541,7 @@ enum {
  * The desc field is used to track userspace buffer index.
  */
 struct ubuf_info {
-       void (*callback)(struct sk_buff *, struct ubuf_info *,
-                        bool zerocopy_success);
+       const struct ubuf_info_ops *ops;
        refcount_t refcnt;
        u8 flags;
 };
@@ -1662,14 +1666,13 @@ static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
 }
 #endif
 
+extern const struct ubuf_info_ops msg_zerocopy_ubuf_ops;
+
 struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
                                       struct ubuf_info *uarg);
 
 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref);
 
-void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
-                          bool success);
-
 int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
                            struct sk_buff *skb, struct iov_iter *from,
                            size_t length);
@@ -1757,13 +1760,13 @@ static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb)
 static inline void net_zcopy_put(struct ubuf_info *uarg)
 {
        if (uarg)
-               uarg->callback(NULL, uarg, true);
+               uarg->ops->complete(NULL, uarg, true);
 }
 
 static inline void net_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref)
 {
        if (uarg) {
-               if (uarg->callback == msg_zerocopy_callback)
+               if (uarg->ops == &msg_zerocopy_ubuf_ops)
                        msg_zerocopy_put_abort(uarg, have_uref);
                else if (have_uref)
                        net_zcopy_put(uarg);
@@ -1777,7 +1780,7 @@ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success)
 
        if (uarg) {
                if (!skb_zcopy_is_nouarg(skb))
-                       uarg->callback(skb, uarg, zerocopy_success);
+                       uarg->ops->complete(skb, uarg, zerocopy_success);
 
                skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY;
        }

diff --git a/io_uring/notif.c b/io_uring/notif.c
index d3e703c37aba83535a4f199329aae2b482d81e04..204f0fc7d96634427846477f77593b6784593361 100644 (file)
--- a/io_uring/notif.c
+++ b/io_uring/notif.c
@@ -24,7 +24,7 @@ static void io_notif_complete_tw_ext(struct io_kiocb *notif, struct io_tw_state
        io_req_task_complete(notif, ts);
 }
 
-static void io_tx_ubuf_callback(struct sk_buff *skb, struct ubuf_info *uarg,
+static void io_tx_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg,
                                bool success)
 {
        struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);
@@ -45,19 +45,27 @@ static void io_tx_ubuf_callback_ext(struct sk_buff *skb, struct ubuf_info *uarg,
                else if (!success && !nd->zc_copied)
                        WRITE_ONCE(nd->zc_copied, true);
        }
-       io_tx_ubuf_callback(skb, uarg, success);
+       io_tx_ubuf_complete(skb, uarg, success);
 }
 
+static const struct ubuf_info_ops io_ubuf_ops = {
+       .complete = io_tx_ubuf_complete,
+};
+
+static const struct ubuf_info_ops io_ubuf_ops_ext = {
+       .complete = io_tx_ubuf_callback_ext,
+};
+
 void io_notif_set_extended(struct io_kiocb *notif)
 {
        struct io_notif_data *nd = io_notif_to_data(notif);
 
-       if (nd->uarg.callback != io_tx_ubuf_callback_ext) {
+       if (nd->uarg.ops != &io_ubuf_ops_ext) {
                nd->account_pages = 0;
                nd->zc_report = false;
                nd->zc_used = false;
                nd->zc_copied = false;
-               nd->uarg.callback = io_tx_ubuf_callback_ext;
+               nd->uarg.ops = &io_ubuf_ops_ext;
                notif->io_task_work.func = io_notif_complete_tw_ext;
        }
 }
@@ -80,7 +88,7 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
 
        nd = io_notif_to_data(notif);
        nd->uarg.flags = IO_NOTIF_UBUF_FLAGS;
-       nd->uarg.callback = io_tx_ubuf_callback;
+       nd->uarg.ops = &io_ubuf_ops;
        refcount_set(&nd->uarg.refcnt, 1);
        return notif;
 }

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index b99127712e6704dda41636014db46096da171bfd..9d36750459f394750cd29061adae8de5db1d5aaa 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -1708,7 +1708,7 @@ static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size)
                return NULL;
        }
 
-       uarg->ubuf.callback = msg_zerocopy_callback;
+       uarg->ubuf.ops = &msg_zerocopy_ubuf_ops;
        uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
        uarg->len = 1;
        uarg->bytelen = size;
@@ -1734,7 +1734,7 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
                u32 bytelen, next;
 
                /* there might be non MSG_ZEROCOPY users */
-               if (uarg->callback != msg_zerocopy_callback)
+               if (uarg->ops != &msg_zerocopy_ubuf_ops)
                        return NULL;
 
                /* realloc only when socket is locked (TCP, UDP cork),
@@ -1845,8 +1845,8 @@ release:
        sock_put(sk);
 }
 
-void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
-                          bool success)
+static void msg_zerocopy_complete(struct sk_buff *skb, struct ubuf_info *uarg,
+                                 bool success)
 {
        struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg);
 
@@ -1855,7 +1855,6 @@ void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
        if (refcount_dec_and_test(&uarg->refcnt))
                __msg_zerocopy_callback(uarg_zc);
 }
-EXPORT_SYMBOL_GPL(msg_zerocopy_callback);
 
 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
 {
@@ -1865,10 +1864,15 @@ void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
        uarg_to_msgzc(uarg)->len--;
 
        if (have_uref)
-               msg_zerocopy_callback(NULL, uarg, true);
+               msg_zerocopy_complete(NULL, uarg, true);
 }
 EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort);
 
+const struct ubuf_info_ops msg_zerocopy_ubuf_ops = {
+       .complete = msg_zerocopy_complete,
+};
+EXPORT_SYMBOL_GPL(msg_zerocopy_ubuf_ops);
+
 int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
                             struct msghdr *msg, int len,
                             struct ubuf_info *uarg)