cpus_allowed: use __NRPROCESSORS_CONF instead of __SC_NPROCESSORS_ONLN for non-sequen...

When disabling SMT on some systems, the ID of some available CPU may
be larger than the value of sysconf(_SC_NPROCESSORS_ONLN). Without
this patch, fio complains that the expected CPU ID is invalid. Here's
an example from my server:

    $ ./fio --cpus_allowed=14 --ioengine=libaio --direct=1 --name=test --numjobs=1 --blocksize=128k --iodepth=1 --rw=read --filename=/dev/nvme0n1
    fio: CPU 14 too large (max=11)
    fio: failed parsing cpus_allowed=14

System information:

    $ lscpu
    Architecture:         x86_64
    CPU op-mode(s):       32-bit, 64-bit
    Byte Order:           Little Endian
    CPU(s):               20
    On-line CPU(s) list:  0,2,4,6,8,10,12,14,16-19
    Off-line CPU(s) list: 1,3,5,7,9,11,13,15
    ...
    Model name:           12th Gen Intel(R) Core(TM) i7-12700
    BIOS Model name:      12th Gen Intel(R) Core(TM) i7-12700
    ...

    $ uname -a
    Linux localhost.localdomain 4.18.0-348.el8.x86_64 #1 SMP Tue Oct 19 15:14:17 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux

    $ cat /etc/redhat-release
    CentOS Linux release 8.5.2111

    $ cat /proc/cmdline
    BOOT_IMAGE=(hd0,gpt2)/vmlinuz-4.18.0-348.el8.x86_64 root=/dev/mapper/cl-root ro nosmt isolcpus=0,2,4,6,8,10,12,14 crashkernel=auto resume=/dev/mapper/cl-swap rd.lvm.lv=cl/root rd.lvm.lv=cl/swap rhgb quiet

    $ cat test.c
    #include <stdio.h>
    #include <unistd.h>
    int main(int argc, char *argv[])
    {
        printf("_SC_NPROCESSORS_ONLN=%ld _SC_NPROCESSORS_CONF=%ld\n",
               sysconf(_SC_NPROCESSORS_ONLN), sysconf(_SC_NPROCESSORS_CONF));
    }

    $ gcc test.c

    $ ./a.out
    _SC_NPROCESSORS_ONLN=12 _SC_NPROCESSORS_CONF=20

Signed-off-by: mayuanpeng <mayuanpeng@dapustor.com>

diff --git a/gettime.c b/gettime.c
index 8993be1688c3073c01a16461c0d5b81fcb1df9bb..bc66a3ac9f908cb8512afdc73c9c240dd1ade8e3 100644 (file)
--- a/gettime.c
+++ b/gettime.c
@@ -671,7 +671,7 @@ static int clock_cmp(const void *p1, const void *p2)
 int fio_monotonic_clocktest(int debug)
 {
        struct clock_thread *cthreads;
-       unsigned int seen_cpus, nr_cpus = cpus_online();
+       unsigned int seen_cpus, nr_cpus = cpus_configured();
        struct clock_entry *entries;
        unsigned long nr_entries, tentries, failed = 0;
        struct clock_entry *prev, *this;

diff --git a/idletime.c b/idletime.c
index fc1df8e9d009a49059558c6ca576a6e553f304c7..90ed77ea6e7d671e7bc251f5ddca69f4fb074e78 100644 (file)
--- a/idletime.c
+++ b/idletime.c
@@ -189,7 +189,7 @@ void fio_idle_prof_init(void)
        pthread_condattr_t cattr;
        struct idle_prof_thread *ipt;
 
-       ipc.nr_cpus = cpus_online();
+       ipc.nr_cpus = cpus_configured();
        ipc.status = IDLE_PROF_STATUS_OK;
 
        if (ipc.opt == IDLE_PROF_OPT_NONE)

diff --git a/options.c b/options.c
index a668b0e475f9f013672a089b289bae410acf796e..9e4d8cd1a92d6a625d5a0a06cf4da861fe227413 100644 (file)
--- a/options.c
+++ b/options.c
@@ -627,7 +627,7 @@ static int str_exitall_cb(void)
 int fio_cpus_split(os_cpu_mask_t *mask, unsigned int cpu_index)
 {
        unsigned int i, index, cpus_in_mask;
-       const long max_cpu = cpus_online();
+       const long max_cpu = cpus_configured();
 
        cpus_in_mask = fio_cpu_count(mask);
        if (!cpus_in_mask)
@@ -666,7 +666,7 @@ static int str_cpumask_cb(void *data, unsigned long long *val)
                return 1;
        }
 
-       max_cpu = cpus_online();
+       max_cpu = cpus_configured();
 
        for (i = 0; i < sizeof(int) * 8; i++) {
                if ((1 << i) & *val) {
@@ -702,7 +702,7 @@ static int set_cpus_allowed(struct thread_data *td, os_cpu_mask_t *mask,
        strip_blank_front(&str);
        strip_blank_end(str);
 
-       max_cpu = cpus_online();
+       max_cpu = cpus_configured();
 
        while ((cpu = strsep(&str, ",")) != NULL) {
                char *str2, *cpu2;
@@ -5305,7 +5305,7 @@ void fio_keywords_init(void)
        sprintf(buf, "%llu", mb_memory);
        fio_keywords[1].replace = strdup(buf);
 
-       l = cpus_online();
+       l = cpus_configured();
        sprintf(buf, "%lu", l);
        fio_keywords[2].replace = strdup(buf);
 }

diff --git a/os/os-hpux.h b/os/os-hpux.h
index a80cb2bc474ecee0733ffac7c07662006762318d..9f3d76f50719736c998e76dbe4e30f1cfdd067e6 100644 (file)
--- a/os/os-hpux.h
+++ b/os/os-hpux.h
@@ -88,9 +88,9 @@ static inline unsigned long long os_phys_mem(void)
        return ret;
 }
 
-#define FIO_HAVE_CPU_ONLINE_SYSCONF
+#define FIO_HAVE_CPU_CONF_SYSCONF
 
-static inline unsigned int cpus_online(void)
+static inline unsigned int cpus_configured(void)
 {
        return mpctl(MPC_GETNUMSPUS, 0, NULL);
 }

diff --git a/os/os-linux.h b/os/os-linux.h
index 831f0ad0474d5c61c47990371575170472fe56c7..bbb1f27c8234465ab06e9aec1abeb77318defe8d 100644 (file)
--- a/os/os-linux.h
+++ b/os/os-linux.h
@@ -251,14 +251,6 @@ static inline int arch_cache_line_size(void)
                return atoi(size);
 }
 
-#ifdef __powerpc64__
-#define FIO_HAVE_CPU_ONLINE_SYSCONF
-static inline unsigned int cpus_online(void)
-{
-        return sysconf(_SC_NPROCESSORS_CONF);
-}
-#endif
-
 static inline unsigned long long get_fs_free_size(const char *path)
 {
        unsigned long long ret;

diff --git a/os/os-solaris.h b/os/os-solaris.h
index ea1f081c89e5d24bdc2c6854e29916ccdacc4b1a..60d4c1eca49e21e5f040e3ce810f80094f493935 100644 (file)
--- a/os/os-solaris.h
+++ b/os/os-solaris.h
@@ -119,7 +119,7 @@ static inline int fio_set_odirect(struct fio_file *f)
 
 static inline bool fio_cpu_isset(os_cpu_mask_t *mask, int cpu)
 {
-       const unsigned int max_cpus = sysconf(_SC_NPROCESSORS_ONLN);
+       const unsigned int max_cpus = sysconf(_SC_NPROCESSORS_CONF);
        unsigned int num_cpus;
        processorid_t *cpus;
        bool ret;

diff --git a/os/os-windows.h b/os/os-windows.h
index 510b8143db1d0b538a63a1b8c67d9b79cb39dea1..12f334861126678d73f92c92c91d0d041514ada4 100644 (file)
--- a/os/os-windows.h
+++ b/os/os-windows.h
@@ -44,7 +44,7 @@
 #define fio_swap64(x)  _byteswap_uint64(x)
 
 #define _SC_PAGESIZE                   0x1
-#define _SC_NPROCESSORS_ONLN   0x2
+#define _SC_NPROCESSORS_CONF   0x2
 #define _SC_PHYS_PAGES                 0x4
 
 #define SA_RESTART     0
@@ -219,9 +219,6 @@ static inline int fio_mkdir(const char *path, mode_t mode) {
        return 0;
 }
 
-#define FIO_HAVE_CPU_ONLINE_SYSCONF
-unsigned int cpus_online(void);
-
 int first_set_cpu(os_cpu_mask_t *cpumask);
 int fio_setaffinity(int pid, os_cpu_mask_t cpumask);
 int fio_cpuset_init(os_cpu_mask_t *mask);

diff --git a/os/os.h b/os/os.h
index aba6813f23e943bc0843ce3947b29548c34d897e..a6fde1fd27f6cab60020b4eb71af1ebe4363a7ff 100644 (file)
--- a/os/os.h
+++ b/os/os.h
@@ -352,10 +352,10 @@ static inline unsigned long long get_fs_free_size(const char *path)
 }
 #endif
 
-#ifndef FIO_HAVE_CPU_ONLINE_SYSCONF
-static inline unsigned int cpus_online(void)
+#ifndef FIO_HAVE_CPU_CONF_SYSCONF
+static inline unsigned int cpus_configured(void)
 {
-       return sysconf(_SC_NPROCESSORS_ONLN);
+       return sysconf(_SC_NPROCESSORS_CONF);
 }
 #endif
 
@@ -363,7 +363,7 @@ static inline unsigned int cpus_online(void)
 #ifdef FIO_HAVE_CPU_AFFINITY
 static inline int CPU_COUNT(os_cpu_mask_t *mask)
 {
-       int max_cpus = cpus_online();
+       int max_cpus = cpus_configured();
        int nr_cpus, i;
 
        for (i = 0, nr_cpus = 0; i < max_cpus; i++)

diff --git a/os/windows/cpu-affinity.c b/os/windows/cpu-affinity.c
index 7601970fc7c284bd120d0ed1b6637ed46882dae3..8f3d6a76b45242d148c637c7bc07a6335985a554 100644 (file)
--- a/os/windows/cpu-affinity.c
+++ b/os/windows/cpu-affinity.c
@@ -2,12 +2,6 @@
 
 #include <windows.h>
 
-/* Return all processors regardless of processor group */
-unsigned int cpus_online(void)
-{
-       return GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);
-}
-
 static void print_mask(os_cpu_mask_t *cpumask)
 {
        for (int i = 0; i < FIO_CPU_MASK_ROWS; i++)

diff --git a/os/windows/posix.c b/os/windows/posix.c
index a3a6c89fd0786240bbe50fffeec7777542d91cb5..a47223daaf761e0a5ae5f0843195ea93de62d011 100644 (file)
--- a/os/windows/posix.c
+++ b/os/windows/posix.c
@@ -216,10 +216,18 @@ long sysconf(int name)
        MEMORYSTATUSEX status;
 
        switch (name) {
-       case _SC_NPROCESSORS_ONLN:
-               val = GetNumLogicalProcessors();
+       case _SC_NPROCESSORS_CONF:
+               /*
+                * Using GetMaximumProcessorCount introduces a problem in
+                * gettime.c because Windows does not have
+                * fio_get_thread_affinity. Log sample (see #1479):
+                *
+                *   CPU mask contains processor beyond last active processor index (2)
+                *   clock setaffinity failed: No error
+                */
+               val = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);
                if (val == -1)
-                       log_err("sysconf(_SC_NPROCESSORS_ONLN) failed\n");
+                       log_err("sysconf(_SC_NPROCESSORS_CONF) failed\n");
 
                break;
 
@@ -1201,4 +1209,4 @@ cleanup:
        DisconnectNamedPipe(hpipe);
        CloseHandle(hpipe);
        return ret;
-}
\ No newline at end of file
+}

diff --git a/server.c b/server.c
index b453be5fc38b85c73627915eb78895cb1239b933..b869d3872734f11c4d79ba6a908eef33081f3298 100644 (file)
--- a/server.c
+++ b/server.c
@@ -999,7 +999,7 @@ static int handle_probe_cmd(struct fio_net_cmd *cmd)
                .os             = FIO_OS,
                .arch           = FIO_ARCH,
                .bpp            = sizeof(void *),
-               .cpus           = __cpu_to_le32(cpus_online()),
+               .cpus           = __cpu_to_le32(cpus_configured()),
        };
 
        dprint(FD_NET, "server: sending probe reply\n");

diff --git a/t/dedupe.c b/t/dedupe.c
index d21e96f4d991201189a3e0cb82f68b6326b25a19..02e52b742ee33730e94c42c4ca6741f9b17f5eaf 100644 (file)
--- a/t/dedupe.c
+++ b/t/dedupe.c
@@ -688,7 +688,7 @@ int main(int argc, char *argv[])
                use_bloom = 0;
 
        if (!num_threads)
-               num_threads = cpus_online();
+               num_threads = cpus_configured();
 
        if (argc == optind)
                return usage(argv);