summaryrefslogtreecommitdiff
path: root/fs
diff options
context:
space:
mode:
Diffstat (limited to 'fs')
-rw-r--r--fs/io-wq.c10
-rw-r--r--fs/io-wq.h2
-rw-r--r--fs/io_uring.c4
3 files changed, 12 insertions, 4 deletions
diff --git a/fs/io-wq.c b/fs/io-wq.c
index b0d09f60200b..9d5ab6d744dc 100644
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -85,7 +85,6 @@ struct io_wqe {
struct {
raw_spinlock_t lock;
struct io_wq_work_list work_list;
- unsigned long hash_map;
unsigned flags;
} ____cacheline_aligned_in_smp;
@@ -111,6 +110,9 @@ struct io_wq {
struct task_struct *manager;
struct user_struct *user;
+
+ unsigned long *hash_map;
+
refcount_t refs;
struct completion done;
@@ -353,8 +355,7 @@ static struct io_wq_work *io_get_next_work(struct io_wqe *wqe)
/* hashed, can run if not already running */
hash = io_get_work_hash(work);
- if (!(wqe->hash_map & BIT(hash))) {
- wqe->hash_map |= BIT(hash);
+ if (!test_and_set_bit(hash, wqe->wq->hash_map)) {
/* all items with this hash lie in [work, tail] */
tail = wqe->hash_tail[hash];
wqe->hash_tail[hash] = NULL;
@@ -452,7 +453,7 @@ get_next:
if (hash != -1U && !next_hashed) {
raw_spin_lock_irq(&wqe->lock);
- wqe->hash_map &= ~BIT_ULL(hash);
+ clear_bit(hash, wq->hash_map);
wqe->flags &= ~IO_WQE_FLAG_STALLED;
/* skip unnecessary unlock-lock wqe->lock */
if (!work)
@@ -975,6 +976,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
if (ret)
goto err_wqes;
+ wq->hash_map = data->hash_map;
wq->free_work = data->free_work;
wq->do_work = data->do_work;
diff --git a/fs/io-wq.h b/fs/io-wq.h
index 3c63a99d1629..d5f4e1ae2d5f 100644
--- a/fs/io-wq.h
+++ b/fs/io-wq.h
@@ -96,6 +96,8 @@ typedef void (io_wq_work_fn)(struct io_wq_work *);
struct io_wq_data {
struct user_struct *user;
+ unsigned long *hash_map;
+
io_wq_work_fn *do_work;
free_work_fn *free_work;
};
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 239eacec3f3a..e71bc4e3bf08 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -359,6 +359,9 @@ struct io_ring_ctx {
unsigned cached_cq_overflow;
unsigned long sq_check_overflow;
+ /* hashed buffered write serialization */
+ unsigned long hash_map;
+
struct list_head defer_list;
struct list_head timeout_list;
struct list_head cq_overflow_list;
@@ -7796,6 +7799,7 @@ static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx)
unsigned int concurrency;
data.user = ctx->user;
+ data.hash_map = &ctx->hash_map;
data.free_work = io_free_work;
data.do_work = io_wq_submit_work;