Merge tag 'perf_urgent_for_v6.1_rc3' of git://git.kernel.org/pub/scm/linux/kernel...

[linux-2.6-block.git] / io_uring / rw.c

diff --git a/io_uring/rw.c b/io_uring/rw.c
index a25cd44cd41531bbf38bcd263f4f2874c61ec0cf..bb47cc4da713c3000c39ac755df3e27b6560d9d5 100644 (file)
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -234,11 +234,32 @@ static void kiocb_end_write(struct io_kiocb *req)
        }
 }
 
+/*
+ * Trigger the notifications after having done some IO, and finish the write
+ * accounting, if any.
+ */
+static void io_req_io_end(struct io_kiocb *req)
+{
+       struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
+
+       if (rw->kiocb.ki_flags & IOCB_WRITE) {
+               kiocb_end_write(req);
+               fsnotify_modify(req->file);
+       } else {
+               fsnotify_access(req->file);
+       }
+}
+
 static bool __io_complete_rw_common(struct io_kiocb *req, long res)
 {
        if (unlikely(res != req->cqe.res)) {
                if ((res == -EAGAIN || res == -EOPNOTSUPP) &&
                    io_rw_should_reissue(req)) {
+                       /*
+                        * Reissue will start accounting again, finish the
+                        * current cycle.
+                        */
+                       io_req_io_end(req);
                        req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO;
                        return true;
                }
@@ -264,15 +285,7 @@ static inline int io_fixup_rw_res(struct io_kiocb *req, long res)
 
 static void io_req_rw_complete(struct io_kiocb *req, bool *locked)
 {
-       struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
-
-       if (rw->kiocb.ki_flags & IOCB_WRITE) {
-               kiocb_end_write(req);
-               fsnotify_modify(req->file);
-       } else {
-               fsnotify_access(req->file);
-       }
-
+       io_req_io_end(req);
        io_req_task_complete(req, locked);
 }
 
@@ -317,6 +330,11 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret,
                req->file->f_pos = rw->kiocb.ki_pos;
        if (ret >= 0 && (rw->kiocb.ki_complete == io_complete_rw)) {
                if (!__io_complete_rw_common(req, ret)) {
+                       /*
+                        * Safe to call io_end from here as we're inline
+                        * from the submission path.
+                        */
+                       io_req_io_end(req);
                        io_req_set_res(req, final_ret,
                                       io_put_kbuf(req, issue_flags));
                        return IOU_OK;
@@ -916,7 +934,7 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags)
                        goto copy_iov;
 
                if (ret2 != req->cqe.res && ret2 >= 0 && need_complete_io(req)) {
-                       struct io_async_rw *rw;
+                       struct io_async_rw *io;
 
                        trace_io_uring_short_write(req->ctx, kiocb->ki_pos - ret2,
                                                req->cqe.res, ret2);
@@ -929,9 +947,9 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags)
                        iov_iter_save_state(&s->iter, &s->iter_state);
                        ret = io_setup_async_rw(req, iovec, s, true);
 
-                       rw = req->async_data;
-                       if (rw)
-                               rw->bytes_done += ret2;
+                       io = req->async_data;
+                       if (io)
+                               io->bytes_done += ret2;
 
                        if (kiocb->ki_flags & IOCB_WRITE)
                                kiocb_end_write(req);