X-Git-Url: https://git.kernel.dk/?a=blobdiff_plain;f=os%2Fos-linux.h;h=3001140ca486630d6bb60a2958c01d824fbd7fc8;hb=2b3d4a6a924e0aa82654d3b96fb134085af7a98a;hp=23c16b65b79c8a6bfcae13a2ebbbb0481b61a5cd;hpb=250e878ab5f26b32facbb6e134f3738aa1aa0120;p=fio.git diff --git a/os/os-linux.h b/os/os-linux.h index 23c16b65..3001140c 100644 --- a/os/os-linux.h +++ b/os/os-linux.h @@ -14,30 +14,44 @@ #include #include #include -#include #include -#include - -#include "binject.h" +#include +#include + +#ifdef ARCH_HAVE_CRC_CRYPTO +#include +#ifndef HWCAP_PMULL +#define HWCAP_PMULL (1 << 4) +#endif /* HWCAP_PMULL */ +#ifndef HWCAP_CRC32 +#define HWCAP_CRC32 (1 << 7) +#endif /* HWCAP_CRC32 */ +#endif /* ARCH_HAVE_CRC_CRYPTO */ + +#include "./os-linux-syscall.h" #include "../file.h" +#ifndef __has_builtin // Optional of course. + #define __has_builtin(x) 0 // Compatibility with non-clang compilers. +#endif + #define FIO_HAVE_CPU_AFFINITY #define FIO_HAVE_DISK_UTIL #define FIO_HAVE_SGIO #define FIO_HAVE_IOPRIO +#define FIO_HAVE_IOPRIO_CLASS #define FIO_HAVE_IOSCHED_SWITCH #define FIO_HAVE_ODIRECT #define FIO_HAVE_HUGETLB -#define FIO_HAVE_RAWBIND #define FIO_HAVE_BLKTRACE -#define FIO_HAVE_PSHARED_MUTEX #define FIO_HAVE_CL_SIZE #define FIO_HAVE_CGROUPS #define FIO_HAVE_FS_STAT #define FIO_HAVE_TRIM -#define FIO_HAVE_BINJECT #define FIO_HAVE_GETTID #define FIO_USE_GENERIC_INIT_RANDOM_STATE +#define FIO_HAVE_PWRITEV2 +#define FIO_HAVE_SHM_ATTACH_REMOVED #ifdef MAP_HUGETLB #define FIO_HAVE_MMAP_HUGE @@ -45,9 +59,9 @@ #define OS_MAP_ANON MAP_ANONYMOUS -typedef cpu_set_t os_cpu_mask_t; +#define FIO_EXT_ENG_DIR "/usr/local/lib/fio" -typedef struct drand48_data os_random_state_t; +typedef cpu_set_t os_cpu_mask_t; #ifdef CONFIG_3ARG_AFFINITY #define fio_setaffinity(pid, cpumask) \ @@ -61,9 +75,15 @@ typedef struct drand48_data os_random_state_t; sched_getaffinity((pid), (ptr)) #endif +#ifdef CONFIG_PTHREAD_GETAFFINITY +#define FIO_HAVE_GET_THREAD_AFFINITY +#define fio_get_thread_affinity(mask) \ + pthread_getaffinity_np(pthread_self(), sizeof(mask), &(mask)) +#endif + #define fio_cpu_clear(mask, cpu) (void) CPU_CLR((cpu), (mask)) #define fio_cpu_set(mask, cpu) (void) CPU_SET((cpu), (mask)) -#define fio_cpu_isset(mask, cpu) CPU_ISSET((cpu), (mask)) +#define fio_cpu_isset(mask, cpu) (CPU_ISSET((cpu), (mask)) != 0) #define fio_cpu_count(mask) CPU_COUNT((mask)) static inline int fio_cpuset_init(os_cpu_mask_t *mask) @@ -95,22 +115,40 @@ enum { #define IOPRIO_BITS 16 #define IOPRIO_CLASS_SHIFT 13 -static inline int ioprio_set(int which, int who, int ioprio_class, int ioprio) +#define IOPRIO_MIN_PRIO 0 /* highest priority */ +#define IOPRIO_MAX_PRIO 7 /* lowest priority */ + +#define IOPRIO_MIN_PRIO_CLASS 0 +#define IOPRIO_MAX_PRIO_CLASS 3 + +static inline int ioprio_value(int ioprio_class, int ioprio) { /* * If no class is set, assume BE */ - if (!ioprio_class) - ioprio_class = IOPRIO_CLASS_BE; + if (!ioprio_class) + ioprio_class = IOPRIO_CLASS_BE; - ioprio |= ioprio_class << IOPRIO_CLASS_SHIFT; - return syscall(__NR_ioprio_set, which, who, ioprio); + return (ioprio_class << IOPRIO_CLASS_SHIFT) | ioprio; } +static inline bool ioprio_value_is_class_rt(unsigned int priority) +{ + return (priority >> IOPRIO_CLASS_SHIFT) == IOPRIO_CLASS_RT; +} + +static inline int ioprio_set(int which, int who, int ioprio_class, int ioprio) +{ + return syscall(__NR_ioprio_set, which, who, + ioprio_value(ioprio_class, ioprio)); +} + +#ifndef CONFIG_HAVE_GETTID static inline int gettid(void) { return syscall(__NR_gettid); } +#endif #define SPLICE_DEF_SIZE (64*1024) @@ -151,49 +189,6 @@ static inline unsigned long long os_phys_mem(void) return (unsigned long long) pages * (unsigned long long) pagesize; } -static inline void os_random_seed(unsigned long seed, os_random_state_t *rs) -{ - srand48_r(seed, rs); -} - -static inline long os_random_long(os_random_state_t *rs) -{ - long val; - - lrand48_r(rs, &val); - return val; -} - -static inline int fio_lookup_raw(dev_t dev, int *majdev, int *mindev) -{ - struct raw_config_request rq; - int fd; - - if (major(dev) != RAW_MAJOR) - return 1; - - /* - * we should be able to find /dev/rawctl or /dev/raw/rawctl - */ - fd = open("/dev/rawctl", O_RDONLY); - if (fd < 0) { - fd = open("/dev/raw/rawctl", O_RDONLY); - if (fd < 0) - return 1; - } - - rq.raw_minor = minor(dev); - if (ioctl(fd, RAW_GETBIND, &rq) < 0) { - close(fd); - return 1; - } - - close(fd); - *majdev = rq.block_major; - *mindev = rq.block_minor; - return 0; -} - #ifdef O_NOATIME #define FIO_O_NOATIME O_NOATIME #else @@ -210,21 +205,19 @@ static inline int fio_lookup_raw(dev_t dev, int *majdev, int *mindev) #define FIO_MADV_FREE MADV_REMOVE #endif -#if defined(__builtin_bswap16) +/* Check for GCC or Clang byte swap intrinsics */ +#if (__has_builtin(__builtin_bswap16) && __has_builtin(__builtin_bswap32) \ + && __has_builtin(__builtin_bswap64)) || (__GNUC__ > 4 \ + || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) /* fio_swapN */ #define fio_swap16(x) __builtin_bswap16(x) -#else -#define fio_swap16(x) __bswap_16(x) -#endif -#if defined(__builtin_bswap32) #define fio_swap32(x) __builtin_bswap32(x) -#else -#define fio_swap32(x) __bswap_32(x) -#endif -#if defined(__builtin_bswap64) #define fio_swap64(x) __builtin_bswap64(x) #else -#define fio_swap64(x) __bswap_64(x) -#endif +#include +#define fio_swap16(x) bswap_16(x) +#define fio_swap32(x) bswap_32(x) +#define fio_swap64(x) bswap_64(x) +#endif /* fio_swapN */ #define CACHE_LINE_FILE \ "/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size" @@ -248,6 +241,14 @@ static inline int arch_cache_line_size(void) return atoi(size); } +#ifdef __powerpc64__ +#define FIO_HAVE_CPU_ONLINE_SYSCONF +static inline unsigned int cpus_online(void) +{ + return sysconf(_SC_NPROCESSORS_CONF); +} +#endif + static inline unsigned long long get_fs_free_size(const char *path) { unsigned long long ret; @@ -261,7 +262,7 @@ static inline unsigned long long get_fs_free_size(const char *path) return ret; } -static inline int os_trim(int fd, unsigned long long start, +static inline int os_trim(struct fio_file *f, unsigned long long start, unsigned long long len) { uint64_t range[2]; @@ -269,7 +270,7 @@ static inline int os_trim(int fd, unsigned long long start, range[0] = start; range[1] = len; - if (!ioctl(fd, BLKDISCARD, range)) + if (!ioctl(f->fd, BLKDISCARD, range)) return 0; return errno; @@ -283,10 +284,139 @@ static inline int fio_set_sched_idle(void) } #endif -#ifndef POSIX_FADV_STREAMID -#define POSIX_FADV_STREAMID 8 +#ifndef F_GET_RW_HINT +#ifndef F_LINUX_SPECIFIC_BASE +#define F_LINUX_SPECIFIC_BASE 1024 +#endif +#define F_GET_RW_HINT (F_LINUX_SPECIFIC_BASE + 11) +#define F_SET_RW_HINT (F_LINUX_SPECIFIC_BASE + 12) +#define F_GET_FILE_RW_HINT (F_LINUX_SPECIFIC_BASE + 13) +#define F_SET_FILE_RW_HINT (F_LINUX_SPECIFIC_BASE + 14) +#endif + +#ifndef RWH_WRITE_LIFE_NONE +#define RWH_WRITE_LIFE_NOT_SET 0 +#define RWH_WRITE_LIFE_NONE 1 +#define RWH_WRITE_LIFE_SHORT 2 +#define RWH_WRITE_LIFE_MEDIUM 3 +#define RWH_WRITE_LIFE_LONG 4 +#define RWH_WRITE_LIFE_EXTREME 5 +#endif + +#define FIO_HAVE_WRITE_HINT + +#ifndef RWF_HIPRI +#define RWF_HIPRI 0x00000001 +#endif +#ifndef RWF_DSYNC +#define RWF_DSYNC 0x00000002 +#endif +#ifndef RWF_SYNC +#define RWF_SYNC 0x00000004 +#endif +#ifndef RWF_NOWAIT +#define RWF_NOWAIT 0x00000008 +#endif + +#ifndef RWF_UNCACHED +#define RWF_UNCACHED 0x00000040 +#endif + +#ifndef RWF_WRITE_LIFE_SHIFT +#define RWF_WRITE_LIFE_SHIFT 4 +#define RWF_WRITE_LIFE_SHORT (1 << RWF_WRITE_LIFE_SHIFT) +#define RWF_WRITE_LIFE_MEDIUM (2 << RWF_WRITE_LIFE_SHIFT) +#define RWF_WRITE_LIFE_LONG (3 << RWF_WRITE_LIFE_SHIFT) +#define RWF_WRITE_LIFE_EXTREME (4 << RWF_WRITE_LIFE_SHIFT) +#endif + +#ifndef CONFIG_PWRITEV2 +#ifdef __NR_preadv2 +static inline void make_pos_h_l(unsigned long *pos_h, unsigned long *pos_l, + off_t offset) +{ +#if BITS_PER_LONG == 64 + *pos_l = offset; + *pos_h = 0; +#else + *pos_l = offset & 0xffffffff; + *pos_h = ((uint64_t) offset) >> 32; +#endif +} +static inline ssize_t preadv2(int fd, const struct iovec *iov, int iovcnt, + off_t offset, unsigned int flags) +{ + unsigned long pos_l, pos_h; + + make_pos_h_l(&pos_h, &pos_l, offset); + return syscall(__NR_preadv2, fd, iov, iovcnt, pos_l, pos_h, flags); +} +static inline ssize_t pwritev2(int fd, const struct iovec *iov, int iovcnt, + off_t offset, unsigned int flags) +{ + unsigned long pos_l, pos_h; + + make_pos_h_l(&pos_h, &pos_l, offset); + return syscall(__NR_pwritev2, fd, iov, iovcnt, pos_l, pos_h, flags); +} +#else +static inline ssize_t preadv2(int fd, const struct iovec *iov, int iovcnt, + off_t offset, unsigned int flags) +{ + errno = ENOSYS; + return -1; +} +static inline ssize_t pwritev2(int fd, const struct iovec *iov, int iovcnt, + off_t offset, unsigned int flags) +{ + errno = ENOSYS; + return -1; +} +#endif /* __NR_preadv2 */ +#endif /* CONFIG_PWRITEV2 */ + +static inline int shm_attach_to_open_removed(void) +{ + return 1; +} + +#ifdef CONFIG_LINUX_FALLOCATE +#define FIO_HAVE_NATIVE_FALLOCATE +static inline bool fio_fallocate(struct fio_file *f, uint64_t offset, + uint64_t len) +{ + int ret; + ret = fallocate(f->fd, 0, offset, len); + if (ret == 0) + return true; + + /* Work around buggy old glibc versions... */ + if (ret > 0) + errno = ret; + + return false; +} +#endif + +#define FIO_HAVE_CPU_HAS +static inline bool os_cpu_has(cpu_features feature) +{ + bool have_feature; + unsigned long fio_unused hwcap; + + switch (feature) { +#ifdef ARCH_HAVE_CRC_CRYPTO + case CPU_ARM64_CRC32C: + hwcap = getauxval(AT_HWCAP); + have_feature = (hwcap & (HWCAP_PMULL | HWCAP_CRC32)) == + (HWCAP_PMULL | HWCAP_CRC32); + break; #endif + default: + have_feature = false; + } -#define FIO_HAVE_STREAMID + return have_feature; +} #endif