summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPavel Begunkov <asml.silence@gmail.com>2021-08-10 17:53:55 +0100
committerJens Axboe <axboe@kernel.dk>2021-08-11 16:35:42 -0600
commit8de254e05555fab96226be5bd062f131df14d9e0 (patch)
tree92dfb727e51a97328e34caa0acda7f8c502d522c
parent3b8fa628e7e55a855871a4e43666fd5f4f710d9d (diff)
io_uring: clean up tctx_task_work()
After recent fixes, tctx_task_work() always does proper spinlocking before looking into ->task_list, so now we don't need atomics for ->task_state, replace it with non-atomic task_running using the critical section. Tide it up, combine two separate block with spinlocking, and always try to splice in there, so we do less locking when new requests are arriving during the function execution. Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> [axboe: fix missing ->task_running reset on task_work_add() failure] Signed-off-by: Jens Axboe <axboe@kernel.dk>
-rw-r--r--fs/io_uring.c32
1 files changed, 14 insertions, 18 deletions
diff --git a/fs/io_uring.c b/fs/io_uring.c
index b877338cf946..592b9d7a0dda 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -476,8 +476,8 @@ struct io_uring_task {
spinlock_t task_lock;
struct io_wq_work_list task_list;
- unsigned long task_state;
struct callback_head task_work;
+ bool task_running;
};
/*
@@ -1958,9 +1958,13 @@ static void tctx_task_work(struct callback_head *cb)
spin_lock_irq(&tctx->task_lock);
node = tctx->task_list.first;
INIT_WQ_LIST(&tctx->task_list);
+ if (!node)
+ tctx->task_running = false;
spin_unlock_irq(&tctx->task_lock);
+ if (!node)
+ break;
- while (node) {
+ do {
struct io_wq_work_node *next = node->next;
struct io_kiocb *req = container_of(node, struct io_kiocb,
io_task_work.node);
@@ -1972,19 +1976,8 @@ static void tctx_task_work(struct callback_head *cb)
}
req->io_task_work.func(req);
node = next;
- }
- if (wq_list_empty(&tctx->task_list)) {
- spin_lock_irq(&tctx->task_lock);
- clear_bit(0, &tctx->task_state);
- if (wq_list_empty(&tctx->task_list)) {
- spin_unlock_irq(&tctx->task_lock);
- break;
- }
- spin_unlock_irq(&tctx->task_lock);
- /* another tctx_task_work() is enqueued, yield */
- if (test_and_set_bit(0, &tctx->task_state))
- break;
- }
+ } while (node);
+
cond_resched();
}
@@ -1998,16 +1991,19 @@ static void io_req_task_work_add(struct io_kiocb *req)
enum task_work_notify_mode notify;
struct io_wq_work_node *node;
unsigned long flags;
+ bool running;
WARN_ON_ONCE(!tctx);
spin_lock_irqsave(&tctx->task_lock, flags);
wq_list_add_tail(&req->io_task_work.node, &tctx->task_list);
+ running = tctx->task_running;
+ if (!running)
+ tctx->task_running = true;
spin_unlock_irqrestore(&tctx->task_lock, flags);
/* task_work already pending, we're done */
- if (test_bit(0, &tctx->task_state) ||
- test_and_set_bit(0, &tctx->task_state))
+ if (running)
return;
/*
@@ -2022,8 +2018,8 @@ static void io_req_task_work_add(struct io_kiocb *req)
return;
}
- clear_bit(0, &tctx->task_state);
spin_lock_irqsave(&tctx->task_lock, flags);
+ tctx->task_running = false;
node = tctx->task_list.first;
INIT_WQ_LIST(&tctx->task_list);
spin_unlock_irqrestore(&tctx->task_lock, flags);