io_uring: switch network send/recv to ITER_UBUF

This is more efficient than iter_iov.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
[merged to 6.2]
Signed-off-by: Keith Busch <kbusch@kernel.org>
Reviewed-by: Christoph Hellwig <hch@lst.de>

diff --git a/io_uring/net.c b/io_uring/net.c
index fbc34a7c2743946c7fbabd91513489c8e25a0b25..0fb9dcab20b74bd0a963081dbca92a910de23542 100644 (file)
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -181,7 +181,7 @@ static int io_setup_async_msg(struct io_kiocb *req,
        if (async_msg->msg.msg_name)
                async_msg->msg.msg_name = &async_msg->addr;
        /* if were using fast_iov, set it to the new one */
-       if (!kmsg->free_iov) {
+       if (iter_is_iovec(&kmsg->msg.msg_iter) && !kmsg->free_iov) {
                size_t fast_idx = kmsg->msg.msg_iter.iov - kmsg->fast_iov;
                async_msg->msg.msg_iter.iov = &async_msg->fast_iov[fast_idx];
        }
@@ -344,7 +344,6 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags)
        struct sockaddr_storage __address;
        struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
        struct msghdr msg;
-       struct iovec iov;
        struct socket *sock;
        unsigned flags;
        int min_ret = 0;
@@ -378,7 +377,7 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags)
        if (unlikely(!sock))
                return -ENOTSOCK;
 
-       ret = import_single_range(ITER_SOURCE, sr->buf, sr->len, &iov, &msg.msg_iter);
+       ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len, &msg.msg_iter);
        if (unlikely(ret))
                return ret;
 
@@ -764,10 +763,7 @@ retry_multishot:
                        }
                }
 
-               kmsg->fast_iov[0].iov_base = buf;
-               kmsg->fast_iov[0].iov_len = len;
-               iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, kmsg->fast_iov, 1,
-                               len);
+               iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, buf, len);
        }
 
        flags = sr->msg_flags;
@@ -835,7 +831,6 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)
        struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
        struct msghdr msg;
        struct socket *sock;
-       struct iovec iov;
        unsigned int cflags;
        unsigned flags;
        int ret, min_ret = 0;
@@ -863,7 +858,7 @@ retry_multishot:
                sr->buf = buf;
        }
 
-       ret = import_single_range(ITER_DEST, sr->buf, len, &iov, &msg.msg_iter);
+       ret = import_ubuf(ITER_DEST, sr->buf, len, &msg.msg_iter);
        if (unlikely(ret))
                goto out_free;
 
@@ -1074,7 +1069,6 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)
        struct sockaddr_storage __address;
        struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
        struct msghdr msg;
-       struct iovec iov;
        struct socket *sock;
        unsigned msg_flags;
        int ret, min_ret = 0;
@@ -1116,8 +1110,7 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)
                msg.sg_from_iter = io_sg_from_iter;
        } else {
                io_notif_set_extended(zc->notif);
-               ret = import_single_range(ITER_SOURCE, zc->buf, zc->len, &iov,
-                                         &msg.msg_iter);
+               ret = import_ubuf(ITER_SOURCE, zc->buf, zc->len, &msg.msg_iter);
                if (unlikely(ret))
                        return ret;
                ret = io_notif_account_mem(zc->notif, zc->len);