io_uring: add IO_URING_F_INLINE issue flag

Set when the execution of the request is done inline from the system
call itself. Any deferred issue will never have this flag set.

Reviewed-by: Caleb Sander Mateos <csander@purestorage.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 2922635986f5210cb7d98060ddf771e26a19054b..054c43c02c960598c1c7966c8147305929474376 100644 (file)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -26,6 +26,8 @@ enum io_uring_cmd_flags {
        IO_URING_F_MULTISHOT            = 4,
        /* executed by io-wq */
        IO_URING_F_IOWQ                 = 8,
+       /* executed inline from syscall */
+       IO_URING_F_INLINE               = 16,
        /* int's last bit, sign checks are usually faster than a bit test */
        IO_URING_F_NONBLOCK             = INT_MIN,
 

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index cf759c172083c543207a24496042be5564a460f0..0f9f6a173e6621b684b21949ad0647279e980940 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -147,7 +147,7 @@ static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
                                         bool cancel_all,
                                         bool is_sqpoll_thread);
 
-static void io_queue_sqe(struct io_kiocb *req);
+static void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags);
 static void __io_req_caches_free(struct io_ring_ctx *ctx);
 
 static __read_mostly DEFINE_STATIC_KEY_FALSE(io_key_has_sqarray);
@@ -1377,7 +1377,7 @@ void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw)
        else if (req->flags & REQ_F_FORCE_ASYNC)
                io_queue_iowq(req);
        else
-               io_queue_sqe(req);
+               io_queue_sqe(req, 0);
 }
 
 void io_req_task_queue_fail(struct io_kiocb *req, int ret)
@@ -1957,12 +1957,14 @@ static void io_queue_async(struct io_kiocb *req, int ret)
        }
 }
 
-static inline void io_queue_sqe(struct io_kiocb *req)
+static inline void io_queue_sqe(struct io_kiocb *req, unsigned int extra_flags)
        __must_hold(&req->ctx->uring_lock)
 {
+       unsigned int issue_flags = IO_URING_F_NONBLOCK |
+                                  IO_URING_F_COMPLETE_DEFER | extra_flags;
        int ret;
 
-       ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
+       ret = io_issue_sqe(req, issue_flags);
 
        /*
         * We async punt it if the file wasn't marked NOWAIT, or if the file
@@ -2218,7 +2220,7 @@ fallback:
                return 0;
        }
 
-       io_queue_sqe(req);
+       io_queue_sqe(req, IO_URING_F_INLINE);
        return 0;
 }