io_uring/io-wq: eliminate redundant io_work_get_acct() calls

Instead of calling io_work_get_acct() again, pass acct to
io_wq_insert_work() and io_wq_remove_pending().

This atomic access in io_work_get_acct() was done under the
`acct->lock`, and optimizing it away reduces lock contention a bit.

Signed-off-by: Max Kellermann <max.kellermann@ionos.com>
Link: https://lore.kernel.org/r/20250128133927.3989681-2-max.kellermann@ionos.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c
index 5d0928f37471e5334731f75fac3e8ca56802c289..6d26f6f068af85e3d2de3693c1d036e4430f169a 100644 (file)
--- a/io_uring/io-wq.c
+++ b/io_uring/io-wq.c
@@ -903,9 +903,8 @@ static void io_run_cancel(struct io_wq_work *work, struct io_wq *wq)
        } while (work);
 }
 
-static void io_wq_insert_work(struct io_wq *wq, struct io_wq_work *work)
+static void io_wq_insert_work(struct io_wq *wq, struct io_wq_acct *acct, struct io_wq_work *work)
 {
-       struct io_wq_acct *acct = io_work_get_acct(wq, work);
        unsigned int hash;
        struct io_wq_work *tail;
 
@@ -951,7 +950,7 @@ void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work)
        }
 
        raw_spin_lock(&acct->lock);
-       io_wq_insert_work(wq, work);
+       io_wq_insert_work(wq, acct, work);
        clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
        raw_spin_unlock(&acct->lock);
 
@@ -1021,10 +1020,10 @@ static bool io_wq_worker_cancel(struct io_worker *worker, void *data)
 }
 
 static inline void io_wq_remove_pending(struct io_wq *wq,
+                                       struct io_wq_acct *acct,
                                         struct io_wq_work *work,
                                         struct io_wq_work_node *prev)
 {
-       struct io_wq_acct *acct = io_work_get_acct(wq, work);
        unsigned int hash = io_get_work_hash(work);
        struct io_wq_work *prev_work = NULL;
 
@@ -1051,7 +1050,7 @@ static bool io_acct_cancel_pending_work(struct io_wq *wq,
                work = container_of(node, struct io_wq_work, list);
                if (!match->fn(work, match->data))
                        continue;
-               io_wq_remove_pending(wq, work, prev);
+               io_wq_remove_pending(wq, acct, work, prev);
                raw_spin_unlock(&acct->lock);
                io_run_cancel(work, wq);
                match->nr_pending++;