summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPavel Begunkov <asml.silence@gmail.com>2022-06-20 15:27:35 +0100
committerJens Axboe <axboe@kernel.dk>2022-06-22 11:32:34 -0600
commit9c7cbab2778399186538f0aac56936354aae4c12 (patch)
tree75345477bb3972f89334b73362414afe481f1992
parent85f0b5fabbf88357b288cff16d95904e95719eac (diff)
io_uring: optimize io_uring_task layout
task_work bits of io_uring_task are split into two cache lines causing extra cache bouncing, place them into a separate cache line. Also move the most used submission path fields closer together, so there are hot. Cc: stable@vger.kernel.org # 5.15+ Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
-rw-r--r--io_uring/tctx.h34
1 files changed, 18 insertions, 16 deletions
diff --git a/io_uring/tctx.h b/io_uring/tctx.h
index dde82ce4d8e2..dead0ed00429 100644
--- a/io_uring/tctx.h
+++ b/io_uring/tctx.h
@@ -7,22 +7,24 @@
struct io_uring_task {
/* submission side */
- int cached_refs;
- struct xarray xa;
- struct wait_queue_head wait;
- const struct io_ring_ctx *last;
- struct io_wq *io_wq;
- struct percpu_counter inflight;
- atomic_t inflight_tracked;
- atomic_t in_idle;
-
- spinlock_t task_lock;
- struct io_wq_work_list task_list;
- struct io_wq_work_list prio_task_list;
- struct callback_head task_work;
- bool task_running;
-
- struct file *registered_rings[IO_RINGFD_REG_MAX];
+ int cached_refs;
+ const struct io_ring_ctx *last;
+ struct io_wq *io_wq;
+ struct file *registered_rings[IO_RINGFD_REG_MAX];
+
+ struct xarray xa;
+ struct wait_queue_head wait;
+ atomic_t in_idle;
+ atomic_t inflight_tracked;
+ struct percpu_counter inflight;
+
+ struct { /* task_work */
+ spinlock_t task_lock;
+ bool task_running;
+ struct io_wq_work_list task_list;
+ struct io_wq_work_list prio_task_list;
+ struct callback_head task_work;
+ } ____cacheline_aligned_in_smp;
};
struct io_tctx_node {