From 40f61ec7193682a332587095f76ff8f1293efb2a Mon Sep 17 00:00:00 2001 From: mayuanpeng Date: Fri, 21 Oct 2022 20:18:13 +0800 Subject: [PATCH] cpus_allowed: use __NRPROCESSORS_CONF instead of __SC_NPROCESSORS_ONLN for non-sequential CPU ids When disabling SMT on some systems, the ID of some available CPU may be larger than the value of sysconf(_SC_NPROCESSORS_ONLN). Without this patch, fio complains that the expected CPU ID is invalid. Here's an example from my server: $ ./fio --cpus_allowed=14 --ioengine=libaio --direct=1 --name=test --numjobs=1 --blocksize=128k --iodepth=1 --rw=read --filename=/dev/nvme0n1 fio: CPU 14 too large (max=11) fio: failed parsing cpus_allowed=14 System information: $ lscpu Architecture: x86_64 CPU op-mode(s): 32-bit, 64-bit Byte Order: Little Endian CPU(s): 20 On-line CPU(s) list: 0,2,4,6,8,10,12,14,16-19 Off-line CPU(s) list: 1,3,5,7,9,11,13,15 ... Model name: 12th Gen Intel(R) Core(TM) i7-12700 BIOS Model name: 12th Gen Intel(R) Core(TM) i7-12700 ... $ uname -a Linux localhost.localdomain 4.18.0-348.el8.x86_64 #1 SMP Tue Oct 19 15:14:17 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux $ cat /etc/redhat-release CentOS Linux release 8.5.2111 $ cat /proc/cmdline BOOT_IMAGE=(hd0,gpt2)/vmlinuz-4.18.0-348.el8.x86_64 root=/dev/mapper/cl-root ro nosmt isolcpus=0,2,4,6,8,10,12,14 crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet $ cat test.c #include #include int main(int argc, char *argv[]) { printf("_SC_NPROCESSORS_ONLN=%ld _SC_NPROCESSORS_CONF=%ld\n", sysconf(_SC_NPROCESSORS_ONLN), sysconf(_SC_NPROCESSORS_CONF)); } $ gcc test.c $ ./a.out _SC_NPROCESSORS_ONLN=12 _SC_NPROCESSORS_CONF=20 Signed-off-by: mayuanpeng --- gettime.c | 2 +- idletime.c | 2 +- options.c | 8 ++++---- os/os-hpux.h | 4 ++-- os/os-linux.h | 8 -------- os/os-solaris.h | 2 +- os/os-windows.h | 5 +---- os/os.h | 8 ++++---- os/windows/cpu-affinity.c | 6 ------ os/windows/posix.c | 16 ++++++++++++---- server.c | 2 +- t/dedupe.c | 2 +- 12 files changed, 28 insertions(+), 37 deletions(-) diff --git a/gettime.c b/gettime.c index 8993be16..bc66a3ac 100644 --- a/gettime.c +++ b/gettime.c @@ -671,7 +671,7 @@ static int clock_cmp(const void *p1, const void *p2) int fio_monotonic_clocktest(int debug) { struct clock_thread *cthreads; - unsigned int seen_cpus, nr_cpus = cpus_online(); + unsigned int seen_cpus, nr_cpus = cpus_configured(); struct clock_entry *entries; unsigned long nr_entries, tentries, failed = 0; struct clock_entry *prev, *this; diff --git a/idletime.c b/idletime.c index fc1df8e9..90ed77ea 100644 --- a/idletime.c +++ b/idletime.c @@ -189,7 +189,7 @@ void fio_idle_prof_init(void) pthread_condattr_t cattr; struct idle_prof_thread *ipt; - ipc.nr_cpus = cpus_online(); + ipc.nr_cpus = cpus_configured(); ipc.status = IDLE_PROF_STATUS_OK; if (ipc.opt == IDLE_PROF_OPT_NONE) diff --git a/options.c b/options.c index a668b0e4..9e4d8cd1 100644 --- a/options.c +++ b/options.c @@ -627,7 +627,7 @@ static int str_exitall_cb(void) int fio_cpus_split(os_cpu_mask_t *mask, unsigned int cpu_index) { unsigned int i, index, cpus_in_mask; - const long max_cpu = cpus_online(); + const long max_cpu = cpus_configured(); cpus_in_mask = fio_cpu_count(mask); if (!cpus_in_mask) @@ -666,7 +666,7 @@ static int str_cpumask_cb(void *data, unsigned long long *val) return 1; } - max_cpu = cpus_online(); + max_cpu = cpus_configured(); for (i = 0; i < sizeof(int) * 8; i++) { if ((1 << i) & *val) { @@ -702,7 +702,7 @@ static int set_cpus_allowed(struct thread_data *td, os_cpu_mask_t *mask, strip_blank_front(&str); strip_blank_end(str); - max_cpu = cpus_online(); + max_cpu = cpus_configured(); while ((cpu = strsep(&str, ",")) != NULL) { char *str2, *cpu2; @@ -5305,7 +5305,7 @@ void fio_keywords_init(void) sprintf(buf, "%llu", mb_memory); fio_keywords[1].replace = strdup(buf); - l = cpus_online(); + l = cpus_configured(); sprintf(buf, "%lu", l); fio_keywords[2].replace = strdup(buf); } diff --git a/os/os-hpux.h b/os/os-hpux.h index a80cb2bc..9f3d76f5 100644 --- a/os/os-hpux.h +++ b/os/os-hpux.h @@ -88,9 +88,9 @@ static inline unsigned long long os_phys_mem(void) return ret; } -#define FIO_HAVE_CPU_ONLINE_SYSCONF +#define FIO_HAVE_CPU_CONF_SYSCONF -static inline unsigned int cpus_online(void) +static inline unsigned int cpus_configured(void) { return mpctl(MPC_GETNUMSPUS, 0, NULL); } diff --git a/os/os-linux.h b/os/os-linux.h index 831f0ad0..bbb1f27c 100644 --- a/os/os-linux.h +++ b/os/os-linux.h @@ -251,14 +251,6 @@ static inline int arch_cache_line_size(void) return atoi(size); } -#ifdef __powerpc64__ -#define FIO_HAVE_CPU_ONLINE_SYSCONF -static inline unsigned int cpus_online(void) -{ - return sysconf(_SC_NPROCESSORS_CONF); -} -#endif - static inline unsigned long long get_fs_free_size(const char *path) { unsigned long long ret; diff --git a/os/os-solaris.h b/os/os-solaris.h index ea1f081c..60d4c1ec 100644 --- a/os/os-solaris.h +++ b/os/os-solaris.h @@ -119,7 +119,7 @@ static inline int fio_set_odirect(struct fio_file *f) static inline bool fio_cpu_isset(os_cpu_mask_t *mask, int cpu) { - const unsigned int max_cpus = sysconf(_SC_NPROCESSORS_ONLN); + const unsigned int max_cpus = sysconf(_SC_NPROCESSORS_CONF); unsigned int num_cpus; processorid_t *cpus; bool ret; diff --git a/os/os-windows.h b/os/os-windows.h index 510b8143..12f33486 100644 --- a/os/os-windows.h +++ b/os/os-windows.h @@ -44,7 +44,7 @@ #define fio_swap64(x) _byteswap_uint64(x) #define _SC_PAGESIZE 0x1 -#define _SC_NPROCESSORS_ONLN 0x2 +#define _SC_NPROCESSORS_CONF 0x2 #define _SC_PHYS_PAGES 0x4 #define SA_RESTART 0 @@ -219,9 +219,6 @@ static inline int fio_mkdir(const char *path, mode_t mode) { return 0; } -#define FIO_HAVE_CPU_ONLINE_SYSCONF -unsigned int cpus_online(void); - int first_set_cpu(os_cpu_mask_t *cpumask); int fio_setaffinity(int pid, os_cpu_mask_t cpumask); int fio_cpuset_init(os_cpu_mask_t *mask); diff --git a/os/os.h b/os/os.h index aba6813f..a6fde1fd 100644 --- a/os/os.h +++ b/os/os.h @@ -352,10 +352,10 @@ static inline unsigned long long get_fs_free_size(const char *path) } #endif -#ifndef FIO_HAVE_CPU_ONLINE_SYSCONF -static inline unsigned int cpus_online(void) +#ifndef FIO_HAVE_CPU_CONF_SYSCONF +static inline unsigned int cpus_configured(void) { - return sysconf(_SC_NPROCESSORS_ONLN); + return sysconf(_SC_NPROCESSORS_CONF); } #endif @@ -363,7 +363,7 @@ static inline unsigned int cpus_online(void) #ifdef FIO_HAVE_CPU_AFFINITY static inline int CPU_COUNT(os_cpu_mask_t *mask) { - int max_cpus = cpus_online(); + int max_cpus = cpus_configured(); int nr_cpus, i; for (i = 0, nr_cpus = 0; i < max_cpus; i++) diff --git a/os/windows/cpu-affinity.c b/os/windows/cpu-affinity.c index 7601970f..8f3d6a76 100644 --- a/os/windows/cpu-affinity.c +++ b/os/windows/cpu-affinity.c @@ -2,12 +2,6 @@ #include -/* Return all processors regardless of processor group */ -unsigned int cpus_online(void) -{ - return GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); -} - static void print_mask(os_cpu_mask_t *cpumask) { for (int i = 0; i < FIO_CPU_MASK_ROWS; i++) diff --git a/os/windows/posix.c b/os/windows/posix.c index a3a6c89f..a47223da 100644 --- a/os/windows/posix.c +++ b/os/windows/posix.c @@ -216,10 +216,18 @@ long sysconf(int name) MEMORYSTATUSEX status; switch (name) { - case _SC_NPROCESSORS_ONLN: - val = GetNumLogicalProcessors(); + case _SC_NPROCESSORS_CONF: + /* + * Using GetMaximumProcessorCount introduces a problem in + * gettime.c because Windows does not have + * fio_get_thread_affinity. Log sample (see #1479): + * + * CPU mask contains processor beyond last active processor index (2) + * clock setaffinity failed: No error + */ + val = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); if (val == -1) - log_err("sysconf(_SC_NPROCESSORS_ONLN) failed\n"); + log_err("sysconf(_SC_NPROCESSORS_CONF) failed\n"); break; @@ -1201,4 +1209,4 @@ cleanup: DisconnectNamedPipe(hpipe); CloseHandle(hpipe); return ret; -} \ No newline at end of file +} diff --git a/server.c b/server.c index b453be5f..b869d387 100644 --- a/server.c +++ b/server.c @@ -999,7 +999,7 @@ static int handle_probe_cmd(struct fio_net_cmd *cmd) .os = FIO_OS, .arch = FIO_ARCH, .bpp = sizeof(void *), - .cpus = __cpu_to_le32(cpus_online()), + .cpus = __cpu_to_le32(cpus_configured()), }; dprint(FD_NET, "server: sending probe reply\n"); diff --git a/t/dedupe.c b/t/dedupe.c index d21e96f4..02e52b74 100644 --- a/t/dedupe.c +++ b/t/dedupe.c @@ -688,7 +688,7 @@ int main(int argc, char *argv[]) use_bloom = 0; if (!num_threads) - num_threads = cpus_online(); + num_threads = cpus_configured(); if (argc == optind) return usage(argv); -- 2.25.1