X-Git-Url: https://git.kernel.dk/?a=blobdiff_plain;f=os%2Fos-linux.h;h=a36552ba68ba93caf6ca58e5fc9761fc53c0b88d;hb=757be0e0a61bb20dc1b76f0b222ec700ee7e22c8;hp=90349016ffe1004ae84904ec45922125f8e96b4a;hpb=8cc7afa91ddc5a0980b9d2dbfdd66e1511a0d4b1;p=fio.git diff --git a/os/os-linux.h b/os/os-linux.h index 90349016..a550bba6 100644 --- a/os/os-linux.h +++ b/os/os-linux.h @@ -1,121 +1,137 @@ #ifndef FIO_OS_LINUX_H #define FIO_OS_LINUX_H +#define FIO_OS os_linux + #include #include #include +#include +#include +#include #include #include +#include +#include #include #include #include +#include +#include + +#ifdef ARCH_HAVE_CRC_CRYPTO +#include +#ifndef HWCAP_CRC32 +#define HWCAP_CRC32 (1 << 7) +#endif /* HWCAP_CRC32 */ +#endif /* ARCH_HAVE_CRC_CRYPTO */ + +#include "./os-linux-syscall.h" +#include "../file.h" + +#ifndef __has_builtin // Optional of course. + #define __has_builtin(x) 0 // Compatibility with non-clang compilers. +#endif -#define FIO_HAVE_LIBAIO -#define FIO_HAVE_POSIXAIO -#define FIO_HAVE_FADVISE #define FIO_HAVE_CPU_AFFINITY #define FIO_HAVE_DISK_UTIL #define FIO_HAVE_SGIO #define FIO_HAVE_IOPRIO -#define FIO_HAVE_SPLICE +#define FIO_HAVE_IOPRIO_CLASS #define FIO_HAVE_IOSCHED_SWITCH #define FIO_HAVE_ODIRECT #define FIO_HAVE_HUGETLB #define FIO_HAVE_RAWBIND +#define FIO_HAVE_BLKTRACE +#define FIO_HAVE_CL_SIZE +#define FIO_HAVE_CGROUPS +#define FIO_HAVE_FS_STAT +#define FIO_HAVE_TRIM +#define FIO_HAVE_GETTID +#define FIO_USE_GENERIC_INIT_RANDOM_STATE +#define FIO_HAVE_PWRITEV2 +#define FIO_HAVE_SHM_ATTACH_REMOVED + +#ifdef MAP_HUGETLB +#define FIO_HAVE_MMAP_HUGE +#endif -#define OS_MAP_ANON (MAP_ANONYMOUS) +#define OS_MAP_ANON MAP_ANONYMOUS typedef cpu_set_t os_cpu_mask_t; -typedef struct drand48_data os_random_state_t; -/* - * we want fadvise64 really, but it's so tangled... later - */ -#define fadvise(fd, off, len, advice) \ - posix_fadvise((fd), (off_t)(off), (len), (advice)) +typedef struct drand48_data os_random_state_t; -#define fio_setaffinity(td) \ - sched_setaffinity((td)->pid, sizeof((td)->o.cpumask), &(td)->o.cpumask) +#ifdef CONFIG_3ARG_AFFINITY +#define fio_setaffinity(pid, cpumask) \ + sched_setaffinity((pid), sizeof(cpumask), &(cpumask)) #define fio_getaffinity(pid, ptr) \ sched_getaffinity((pid), sizeof(cpu_set_t), (ptr)) +#elif defined(CONFIG_2ARG_AFFINITY) +#define fio_setaffinity(pid, cpumask) \ + sched_setaffinity((pid), &(cpumask)) +#define fio_getaffinity(pid, ptr) \ + sched_getaffinity((pid), (ptr)) +#endif -static inline int ioprio_set(int which, int who, int ioprio) -{ - return syscall(__NR_ioprio_set, which, who, ioprio); -} - -/* - * Just check for SPLICE_F_MOVE, if that isn't there, assume the others - * aren't either. - */ -#ifndef SPLICE_F_MOVE -#define SPLICE_F_MOVE (0x01) /* move pages instead of copying */ -#define SPLICE_F_NONBLOCK (0x02) /* don't block on the pipe splicing (but */ - /* we may still block on the fd we splice */ - /* from/to, of course */ -#define SPLICE_F_MORE (0x04) /* expect more data */ -#define SPLICE_F_GIFT (0x08) /* pages passed in are a gift */ +#define fio_cpu_clear(mask, cpu) (void) CPU_CLR((cpu), (mask)) +#define fio_cpu_set(mask, cpu) (void) CPU_SET((cpu), (mask)) +#define fio_cpu_isset(mask, cpu) (CPU_ISSET((cpu), (mask)) != 0) +#define fio_cpu_count(mask) CPU_COUNT((mask)) -static inline int splice(int fdin, loff_t *off_in, int fdout, loff_t *off_out, - size_t len, unsigned long flags) +static inline int fio_cpuset_init(os_cpu_mask_t *mask) { - return syscall(__NR_sys_splice, fdin, off_in, fdout, off_out, len, flags); + CPU_ZERO(mask); + return 0; } -static inline int tee(int fdin, int fdout, size_t len, unsigned int flags) +static inline int fio_cpuset_exit(os_cpu_mask_t *mask) { - return syscall(__NR_sys_tee, fdin, fdout, len, flags); + return 0; } -static inline int vmsplice(int fd, const struct iovec *iov, - unsigned long nr_segs, unsigned int flags) -{ - return syscall(__NR_sys_vmsplice, fd, iov, nr_segs, flags); -} -#endif +#define FIO_MAX_CPUS CPU_SETSIZE -#define SPLICE_DEF_SIZE (64*1024) +enum { + IOPRIO_CLASS_NONE, + IOPRIO_CLASS_RT, + IOPRIO_CLASS_BE, + IOPRIO_CLASS_IDLE, +}; -#ifdef FIO_HAVE_SYSLET +enum { + IOPRIO_WHO_PROCESS = 1, + IOPRIO_WHO_PGRP, + IOPRIO_WHO_USER, +}; -struct syslet_uatom; -struct async_head_user; +#define IOPRIO_BITS 16 +#define IOPRIO_CLASS_SHIFT 13 -/* - * syslet stuff - */ -static inline struct syslet_uatom * -async_exec(struct syslet_uatom *atom, struct async_head_user *ahu) -{ - return (void *) syscall(__NR_async_exec, atom, ahu); -} +#define IOPRIO_MIN_PRIO 0 /* highest priority */ +#define IOPRIO_MAX_PRIO 7 /* lowest priority */ -static inline long -async_wait(unsigned long min_wait_events, unsigned long user_ring_idx, - struct async_head_user *ahu) -{ - return syscall(__NR_async_wait, min_wait_events, - user_ring_idx, ahu); -} +#define IOPRIO_MIN_PRIO_CLASS 0 +#define IOPRIO_MAX_PRIO_CLASS 3 -static inline long async_thread(void *event, struct async_head_user *ahu) +static inline int ioprio_set(int which, int who, int ioprio_class, int ioprio) { - return syscall(__NR_async_thread, event, ahu); + /* + * If no class is set, assume BE + */ + if (!ioprio_class) + ioprio_class = IOPRIO_CLASS_BE; + + ioprio |= ioprio_class << IOPRIO_CLASS_SHIFT; + return syscall(__NR_ioprio_set, which, who, ioprio); } -static inline long umem_add(unsigned long *uptr, unsigned long inc) +static inline int gettid(void) { - return syscall(__NR_umem_add, uptr, inc); + return syscall(__NR_gettid); } -#endif /* FIO_HAVE_SYSLET */ - -enum { - IOPRIO_WHO_PROCESS = 1, - IOPRIO_WHO_PGRP, - IOPRIO_WHO_USER, -}; -#define IOPRIO_CLASS_SHIFT 13 +#define SPLICE_DEF_SIZE (64*1024) #ifndef BLKGETSIZE64 #define BLKGETSIZE64 _IOR(0x12,114,size_t) @@ -125,14 +141,18 @@ enum { #define BLKFLSBUF _IO(0x12,97) #endif -static inline int blockdev_invalidate_cache(int fd) +#ifndef BLKDISCARD +#define BLKDISCARD _IO(0x12,119) +#endif + +static inline int blockdev_invalidate_cache(struct fio_file *f) { - return ioctl(fd, BLKFLSBUF); + return ioctl(f->fd, BLKFLSBUF); } -static inline int blockdev_size(int fd, unsigned long long *bytes) +static inline int blockdev_size(struct fio_file *f, unsigned long long *bytes) { - if (!ioctl(fd, BLKGETSIZE64, bytes)) + if (!ioctl(f->fd, BLKGETSIZE64, bytes)) return 0; return errno; @@ -163,14 +183,6 @@ static inline long os_random_long(os_random_state_t *rs) return val; } -static inline double os_random_double(os_random_state_t *rs) -{ - double val; - - drand48_r(rs, &val); - return val; -} - static inline int fio_lookup_raw(dev_t dev, int *majdev, int *mindev) { struct raw_config_request rq; @@ -201,4 +213,226 @@ static inline int fio_lookup_raw(dev_t dev, int *majdev, int *mindev) return 0; } +#ifdef O_NOATIME +#define FIO_O_NOATIME O_NOATIME +#else +#define FIO_O_NOATIME 0 +#endif + +#ifdef O_ATOMIC +#define OS_O_ATOMIC O_ATOMIC +#else +#define OS_O_ATOMIC 040000000 +#endif + +#ifdef MADV_REMOVE +#define FIO_MADV_FREE MADV_REMOVE +#endif + +/* Check for GCC or Clang byte swap intrinsics */ +#if (__has_builtin(__builtin_bswap16) && __has_builtin(__builtin_bswap32) \ + && __has_builtin(__builtin_bswap64)) || (__GNUC__ > 4 \ + || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) /* fio_swapN */ +#define fio_swap16(x) __builtin_bswap16(x) +#define fio_swap32(x) __builtin_bswap32(x) +#define fio_swap64(x) __builtin_bswap64(x) +#else +#include +#define fio_swap16(x) bswap_16(x) +#define fio_swap32(x) bswap_32(x) +#define fio_swap64(x) bswap_64(x) +#endif /* fio_swapN */ + +#define CACHE_LINE_FILE \ + "/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size" + +static inline int arch_cache_line_size(void) +{ + char size[32]; + int fd, ret; + + fd = open(CACHE_LINE_FILE, O_RDONLY); + if (fd < 0) + return -1; + + ret = read(fd, size, sizeof(size)); + + close(fd); + + if (ret <= 0) + return -1; + else + return atoi(size); +} + +#ifdef __powerpc64__ +#define FIO_HAVE_CPU_ONLINE_SYSCONF +static inline unsigned int cpus_online(void) +{ + return sysconf(_SC_NPROCESSORS_CONF); +} +#endif + +static inline unsigned long long get_fs_free_size(const char *path) +{ + unsigned long long ret; + struct statfs s; + + if (statfs(path, &s) < 0) + return -1ULL; + + ret = s.f_bsize; + ret *= (unsigned long long) s.f_bfree; + return ret; +} + +static inline int os_trim(struct fio_file *f, unsigned long long start, + unsigned long long len) +{ + uint64_t range[2]; + + range[0] = start; + range[1] = len; + + if (!ioctl(f->fd, BLKDISCARD, range)) + return 0; + + return errno; +} + +#ifdef CONFIG_SCHED_IDLE +static inline int fio_set_sched_idle(void) +{ + struct sched_param p = { .sched_priority = 0, }; + return sched_setscheduler(gettid(), SCHED_IDLE, &p); +} +#endif + +#ifndef F_GET_RW_HINT +#ifndef F_LINUX_SPECIFIC_BASE +#define F_LINUX_SPECIFIC_BASE 1024 +#endif +#define F_GET_RW_HINT (F_LINUX_SPECIFIC_BASE + 11) +#define F_SET_RW_HINT (F_LINUX_SPECIFIC_BASE + 12) +#define F_GET_FILE_RW_HINT (F_LINUX_SPECIFIC_BASE + 13) +#define F_SET_FILE_RW_HINT (F_LINUX_SPECIFIC_BASE + 14) +#endif + +#ifndef RWH_WRITE_LIFE_NONE +#define RWH_WRITE_LIFE_NOT_SET 0 +#define RWH_WRITE_LIFE_NONE 1 +#define RWH_WRITE_LIFE_SHORT 2 +#define RWH_WRITE_LIFE_MEDIUM 3 +#define RWH_WRITE_LIFE_LONG 4 +#define RWH_WRITE_LIFE_EXTREME 5 +#endif + +#define FIO_HAVE_WRITE_HINT + +#ifndef RWF_HIPRI +#define RWF_HIPRI 0x00000001 +#endif +#ifndef RWF_DSYNC +#define RWF_DSYNC 0x00000002 +#endif +#ifndef RWF_SYNC +#define RWF_SYNC 0x00000004 +#endif + +#ifndef RWF_WRITE_LIFE_SHIFT +#define RWF_WRITE_LIFE_SHIFT 4 +#define RWF_WRITE_LIFE_SHORT (1 << RWF_WRITE_LIFE_SHIFT) +#define RWF_WRITE_LIFE_MEDIUM (2 << RWF_WRITE_LIFE_SHIFT) +#define RWF_WRITE_LIFE_LONG (3 << RWF_WRITE_LIFE_SHIFT) +#define RWF_WRITE_LIFE_EXTREME (4 << RWF_WRITE_LIFE_SHIFT) +#endif + +#ifndef CONFIG_PWRITEV2 +#ifdef __NR_preadv2 +static inline void make_pos_h_l(unsigned long *pos_h, unsigned long *pos_l, + off_t offset) +{ +#if BITS_PER_LONG == 64 + *pos_l = offset; + *pos_h = 0; +#else + *pos_l = offset & 0xffffffff; + *pos_h = ((uint64_t) offset) >> 32; +#endif +} +static inline ssize_t preadv2(int fd, const struct iovec *iov, int iovcnt, + off_t offset, unsigned int flags) +{ + unsigned long pos_l, pos_h; + + make_pos_h_l(&pos_h, &pos_l, offset); + return syscall(__NR_preadv2, fd, iov, iovcnt, pos_l, pos_h, flags); +} +static inline ssize_t pwritev2(int fd, const struct iovec *iov, int iovcnt, + off_t offset, unsigned int flags) +{ + unsigned long pos_l, pos_h; + + make_pos_h_l(&pos_h, &pos_l, offset); + return syscall(__NR_pwritev2, fd, iov, iovcnt, pos_l, pos_h, flags); +} +#else +static inline ssize_t preadv2(int fd, const struct iovec *iov, int iovcnt, + off_t offset, unsigned int flags) +{ + errno = ENOSYS; + return -1; +} +static inline ssize_t pwritev2(int fd, const struct iovec *iov, int iovcnt, + off_t offset, unsigned int flags) +{ + errno = ENOSYS; + return -1; +} +#endif /* __NR_preadv2 */ +#endif /* CONFIG_PWRITEV2 */ + +static inline int shm_attach_to_open_removed(void) +{ + return 1; +} + +#ifdef CONFIG_LINUX_FALLOCATE +#define FIO_HAVE_NATIVE_FALLOCATE +static inline bool fio_fallocate(struct fio_file *f, uint64_t offset, + uint64_t len) +{ + int ret; + ret = fallocate(f->fd, 0, 0, len); + if (ret == 0) + return true; + + /* Work around buggy old glibc versions... */ + if (ret > 0) + errno = ret; + + return false; +} +#endif + +#define FIO_HAVE_CPU_HAS +static inline bool os_cpu_has(cpu_features feature) +{ + bool have_feature; + unsigned long fio_unused hwcap; + + switch (feature) { +#ifdef ARCH_HAVE_CRC_CRYPTO + case CPU_ARM64_CRC32C: + hwcap = getauxval(AT_HWCAP); + have_feature = (hwcap & HWCAP_CRC32) != 0; + break; +#endif + default: + have_feature = false; + } + + return have_feature; +} + #endif