From 63e8595c060a1fef421e3eecfc05ad882dafb8ac Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 16 Apr 2025 18:29:15 +0200 Subject: [PATCH] futex: Allow to make the private hash immutable My initial testing showed that: perf bench futex hash reported less operations/sec with private hash. After using the same amount of buckets in the private hash as used by the global hash then the operations/sec were about the same. This changed once the private hash became resizable. This feature added an RCU section and reference counting via atomic inc+dec operation into the hot path. The reference counting can be avoided if the private hash is made immutable. Extend PR_FUTEX_HASH_SET_SLOTS by a fourth argument which denotes if the private should be made immutable. Once set (to true) the a further resize is not allowed (same if set to global hash). Add PR_FUTEX_HASH_GET_IMMUTABLE which returns true if the hash can not be changed. Update "perf bench" suite. For comparison, results of "perf bench futex hash -s": - Xeon CPU E5-2650, 2 NUMA nodes, total 32 CPUs: - Before the introducing task local hash shared Averaged 1.487.148 operations/sec (+- 0,53%), total secs = 10 private Averaged 2.192.405 operations/sec (+- 0,07%), total secs = 10 - With the series shared Averaged 1.326.342 operations/sec (+- 0,41%), total secs = 10 -b128 Averaged 141.394 operations/sec (+- 1,15%), total secs = 10 -Ib128 Averaged 851.490 operations/sec (+- 0,67%), total secs = 10 -b8192 Averaged 131.321 operations/sec (+- 2,13%), total secs = 10 -Ib8192 Averaged 1.923.077 operations/sec (+- 0,61%), total secs = 10 128 is the default allocation of hash buckets. 8192 was the previous amount of allocated hash buckets. - Xeon(R) CPU E7-8890 v3, 4 NUMA nodes, total 144 CPUs: - Before the introducing task local hash shared Averaged 1.810.936 operations/sec (+- 0,26%), total secs = 20 private Averaged 2.505.801 operations/sec (+- 0,05%), total secs = 20 - With the series shared Averaged 1.589.002 operations/sec (+- 0,25%), total secs = 20 -b1024 Averaged 42.410 operations/sec (+- 0,20%), total secs = 20 -Ib1024 Averaged 740.638 operations/sec (+- 1,51%), total secs = 20 -b65536 Averaged 48.811 operations/sec (+- 1,35%), total secs = 20 -Ib65536 Averaged 1.963.165 operations/sec (+- 0,18%), total secs = 20 1024 is the default allocation of hash buckets. 65536 was the previous amount of allocated hash buckets. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Peter Zijlstra (Intel) Acked-by: Shrikanth Hegde Link: https://lore.kernel.org/r/20250416162921.513656-16-bigeasy@linutronix.de --- include/uapi/linux/prctl.h | 2 ++ kernel/futex/core.c | 49 +++++++++++++++++++++++++++++++++----- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index 3b93fb906e3c..43dec6eed559 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -367,6 +367,8 @@ struct prctl_mm_map { /* FUTEX hash management */ #define PR_FUTEX_HASH 78 # define PR_FUTEX_HASH_SET_SLOTS 1 +# define FH_FLAG_IMMUTABLE (1ULL << 0) # define PR_FUTEX_HASH_GET_SLOTS 2 +# define PR_FUTEX_HASH_GET_IMMUTABLE 3 #endif /* _LINUX_PRCTL_H */ diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 9e7dad52abea..8054fda94719 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -63,6 +63,7 @@ struct futex_private_hash { struct rcu_head rcu; void *mm; bool custom; + bool immutable; struct futex_hash_bucket queues[]; }; @@ -132,12 +133,16 @@ static inline bool futex_key_is_private(union futex_key *key) bool futex_private_hash_get(struct futex_private_hash *fph) { + if (fph->immutable) + return true; return rcuref_get(&fph->users); } void futex_private_hash_put(struct futex_private_hash *fph) { /* Ignore return value, last put is verified via rcuref_is_dead() */ + if (fph->immutable) + return; if (rcuref_put(&fph->users)) wake_up_var(fph->mm); } @@ -277,6 +282,8 @@ again: if (!fph) return NULL; + if (fph->immutable) + return fph; if (rcuref_get(&fph->users)) return fph; } @@ -1383,6 +1390,9 @@ static void futex_hash_bucket_init(struct futex_hash_bucket *fhb, spin_lock_init(&fhb->lock); } +#define FH_CUSTOM 0x01 +#define FH_IMMUTABLE 0x02 + #ifdef CONFIG_FUTEX_PRIVATE_HASH void futex_hash_free(struct mm_struct *mm) { @@ -1433,10 +1443,11 @@ static bool futex_hash_less(struct futex_private_hash *a, return false; /* equal */ } -static int futex_hash_allocate(unsigned int hash_slots, bool custom) +static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) { struct mm_struct *mm = current->mm; struct futex_private_hash *fph; + bool custom = flags & FH_CUSTOM; int i; if (hash_slots && (hash_slots == 1 || !is_power_of_2(hash_slots))) @@ -1447,7 +1458,7 @@ static int futex_hash_allocate(unsigned int hash_slots, bool custom) */ scoped_guard(rcu) { fph = rcu_dereference(mm->futex_phash); - if (fph && !fph->hash_mask) { + if (fph && (!fph->hash_mask || fph->immutable)) { if (custom) return -EBUSY; return 0; @@ -1461,6 +1472,7 @@ static int futex_hash_allocate(unsigned int hash_slots, bool custom) rcuref_init(&fph->users, 1); fph->hash_mask = hash_slots ? hash_slots - 1 : 0; fph->custom = custom; + fph->immutable = !!(flags & FH_IMMUTABLE); fph->mm = mm; for (i = 0; i < hash_slots; i++) @@ -1553,7 +1565,7 @@ int futex_hash_allocate_default(void) if (current_buckets >= buckets) return 0; - return futex_hash_allocate(buckets, false); + return futex_hash_allocate(buckets, 0); } static int futex_hash_get_slots(void) @@ -1567,9 +1579,22 @@ static int futex_hash_get_slots(void) return 0; } +static int futex_hash_get_immutable(void) +{ + struct futex_private_hash *fph; + + guard(rcu)(); + fph = rcu_dereference(current->mm->futex_phash); + if (fph && fph->immutable) + return 1; + if (fph && !fph->hash_mask) + return 1; + return 0; +} + #else -static int futex_hash_allocate(unsigned int hash_slots, bool custom) +static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) { return -EINVAL; } @@ -1578,23 +1603,35 @@ static int futex_hash_get_slots(void) { return 0; } + +static int futex_hash_get_immutable(void) +{ + return 0; +} #endif int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4) { + unsigned int flags = FH_CUSTOM; int ret; switch (arg2) { case PR_FUTEX_HASH_SET_SLOTS: - if (arg4 != 0) + if (arg4 & ~FH_FLAG_IMMUTABLE) return -EINVAL; - ret = futex_hash_allocate(arg3, true); + if (arg4 & FH_FLAG_IMMUTABLE) + flags |= FH_IMMUTABLE; + ret = futex_hash_allocate(arg3, flags); break; case PR_FUTEX_HASH_GET_SLOTS: ret = futex_hash_get_slots(); break; + case PR_FUTEX_HASH_GET_IMMUTABLE: + ret = futex_hash_get_immutable(); + break; + default: ret = -EINVAL; break; -- 2.25.1