summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJens Axboe <axboe@kernel.dk>2021-08-10 15:15:25 -0600
committerJens Axboe <axboe@kernel.dk>2021-08-12 11:23:02 -0600
commit46989efacb1c21c60537790244947703814be641 (patch)
treebaa673db7cc1220415f5d037cb33d8afa6cf45b1
parentcfee77f8ca0d8e70e59dae0b22db93c4c65b4680 (diff)
io_uring: run regular file completions from task_work
This is in preparation to making the completion lock work outside of hard/soft IRQ context. Signed-off-by: Jens Axboe <axboe@kernel.dk>
-rw-r--r--fs/io_uring.c31
1 files changed, 24 insertions, 7 deletions
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 91863f9362f2..042a81f46acc 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2480,31 +2480,48 @@ static bool io_rw_should_reissue(struct io_kiocb *req)
}
#endif
-static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
- unsigned int issue_flags)
+static bool __io_complete_rw_common(struct io_kiocb *req, long res)
{
- int cflags = 0;
-
if (req->rw.kiocb.ki_flags & IOCB_WRITE)
kiocb_end_write(req);
if (res != req->result) {
if ((res == -EAGAIN || res == -EOPNOTSUPP) &&
io_rw_should_reissue(req)) {
req->flags |= REQ_F_REISSUE;
- return;
+ return true;
}
req_set_fail(req);
+ req->result = res;
}
+ return false;
+}
+
+static void io_req_task_complete(struct io_kiocb *req)
+{
+ int cflags = 0;
+
if (req->flags & REQ_F_BUFFER_SELECTED)
cflags = io_put_rw_kbuf(req);
- __io_req_complete(req, issue_flags, res, cflags);
+ __io_req_complete(req, 0, req->result, cflags);
+}
+
+static void __io_complete_rw(struct io_kiocb *req, long res, long res2,
+ unsigned int issue_flags)
+{
+ if (__io_complete_rw_common(req, res))
+ return;
+ io_req_task_complete(req);
}
static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
{
struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
- __io_complete_rw(req, res, res2, 0);
+ if (__io_complete_rw_common(req, res))
+ return;
+ req->result = res;
+ req->io_task_work.func = io_req_task_complete;
+ io_req_task_work_add(req);
}
static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)