io_uring/net: add IORING_SEND_IGNORE_INLINE support to zerocopy send

If IORING_SEND_IGNORE_INLINE is set for a send zerocopy request, then a
successful inline completion of such a request will be ignored for a
submit_and_wait() type of submissions. In other words, if an application
submits a send for socketA with a recv for socketB, it can now do:

io_uring_submit_and_wait(ring, 1);

and have the inline send completion be ignored from the number of items
to wait for. Note that this only applies to the direct zerocopy send
completion, it does not include the notification when it's safe to reuse
the buffer. Those happen out-of-line anyway.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/net.c b/io_uring/net.c
index dbeda354fb02d8181c1a73df752e8199acd8023d..0219980e40f3d44dbf37e9cffaed9d563a42a1ca 100644 (file)
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -1226,7 +1226,8 @@ void io_send_zc_cleanup(struct io_kiocb *req)
 }
 
 #define IO_ZC_FLAGS_COMMON (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_FIXED_BUF)
-#define IO_ZC_FLAGS_VALID  (IO_ZC_FLAGS_COMMON | IORING_SEND_ZC_REPORT_USAGE)
+#define IO_ZC_FLAGS_VALID  (IO_ZC_FLAGS_COMMON | IORING_SEND_ZC_REPORT_USAGE | \
+                               IORING_SEND_IGNORE_INLINE)
 
 int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 {
@@ -1262,6 +1263,11 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
                        nd->zc_used = false;
                        nd->zc_copied = false;
                }
+               if (zc->flags & IORING_SEND_IGNORE_INLINE) {
+                       if (req->flags & REQ_F_CQE_SKIP)
+                               return -EINVAL;
+                       req->flags |= REQ_F_IGNORE_INLINE;
+               }
        }
 
        if (zc->flags & IORING_RECVSEND_FIXED_BUF) {
@@ -1409,6 +1415,8 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)
        ret = sock_sendmsg(sock, &kmsg->msg);
 
        if (unlikely(ret < min_ret)) {
+               req->flags &= ~REQ_F_IGNORE_INLINE;
+
                if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
                        return -EAGAIN;