X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=os%2Fos-linux.h;h=ba53590d01a22611469c3733a3fbd25b1d449684;hp=9b7ff29e50af7f07d656cf3e763e7fafe409a669;hb=dd3805d49995e59fdf61e2560c3fec5b7f5c71b6;hpb=eebd2f2d8a022a248acac5685f73665aa0c6ee99 diff --git a/os/os-linux.h b/os/os-linux.h index 9b7ff29e..ba53590d 100644 --- a/os/os-linux.h +++ b/os/os-linux.h @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -14,86 +16,51 @@ #include #include #include -#include -#include "indirect.h" +#include "./os-linux-syscall.h" #include "binject.h" #include "../file.h" -#define FIO_HAVE_LIBAIO -#define FIO_HAVE_POSIXAIO -#define FIO_HAVE_FADVISE +#ifndef __has_builtin // Optional of course. + #define __has_builtin(x) 0 // Compatibility with non-clang compilers. +#endif + #define FIO_HAVE_CPU_AFFINITY #define FIO_HAVE_DISK_UTIL #define FIO_HAVE_SGIO #define FIO_HAVE_IOPRIO -#define FIO_HAVE_SPLICE +#define FIO_HAVE_IOPRIO_CLASS #define FIO_HAVE_IOSCHED_SWITCH #define FIO_HAVE_ODIRECT #define FIO_HAVE_HUGETLB #define FIO_HAVE_RAWBIND #define FIO_HAVE_BLKTRACE -#define FIO_HAVE_STRSEP -#define FIO_HAVE_POSIXAIO_FSYNC -#define FIO_HAVE_PSHARED_MUTEX #define FIO_HAVE_CL_SIZE #define FIO_HAVE_CGROUPS -#define FIO_HAVE_FDATASYNC #define FIO_HAVE_FS_STAT #define FIO_HAVE_TRIM #define FIO_HAVE_BINJECT -#define FIO_HAVE_CLOCK_MONOTONIC #define FIO_HAVE_GETTID #define FIO_USE_GENERIC_INIT_RANDOM_STATE -#define FIO_HAVE_E4_ENG - -/* - * Can only enable this for newer glibcs, or the header and defines are - * missing - */ -#if __GLIBC__ >= 2 && __GLIBC_MINOR__ >= 6 -#define FIO_HAVE_FALLOCATE -#endif -#if __GLIBC__ >= 2 && __GLIBC_MINOR__ >= 8 -#define FIO_HAVE_LINUX_FALLOCATE -#endif +#define FIO_HAVE_PWRITEV2 +#define FIO_HAVE_SHM_ATTACH_REMOVED -#ifdef FIO_HAVE_LINUX_FALLOCATE -#define FIO_HAVE_FALLOC_ENG -#endif - -#ifdef SYNC_FILE_RANGE_WAIT_BEFORE -#define FIO_HAVE_SYNC_FILE_RANGE +#ifdef MAP_HUGETLB +#define FIO_HAVE_MMAP_HUGE #endif #define OS_MAP_ANON MAP_ANONYMOUS -#ifndef CLOCK_MONOTONIC -#define CLOCK_MONOTONIC 1 -#endif - typedef cpu_set_t os_cpu_mask_t; typedef struct drand48_data os_random_state_t; -/* - * we want fadvise64 really, but it's so tangled... later - */ -#ifdef FIO_HAVE_FADVISE -#define fadvise(fd, off, len, advice) \ - posix_fadvise((fd), (off_t)(off), (len), (advice)) -#endif - -/* - * If you are on an ancient glibc (2.3.2), then define GLIBC_2_3_2 if you want - * the affinity helpers to work. - */ -#ifndef GLIBC_2_3_2 +#ifdef CONFIG_3ARG_AFFINITY #define fio_setaffinity(pid, cpumask) \ sched_setaffinity((pid), sizeof(cpumask), &(cpumask)) #define fio_getaffinity(pid, ptr) \ sched_getaffinity((pid), sizeof(cpu_set_t), (ptr)) -#else +#elif defined(CONFIG_2ARG_AFFINITY) #define fio_setaffinity(pid, cpumask) \ sched_setaffinity((pid), &(cpumask)) #define fio_getaffinity(pid, ptr) \ @@ -102,6 +69,8 @@ typedef struct drand48_data os_random_state_t; #define fio_cpu_clear(mask, cpu) (void) CPU_CLR((cpu), (mask)) #define fio_cpu_set(mask, cpu) (void) CPU_SET((cpu), (mask)) +#define fio_cpu_isset(mask, cpu) CPU_ISSET((cpu), (mask)) +#define fio_cpu_count(mask) CPU_COUNT((mask)) static inline int fio_cpuset_init(os_cpu_mask_t *mask) { @@ -116,81 +85,6 @@ static inline int fio_cpuset_exit(os_cpu_mask_t *mask) #define FIO_MAX_CPUS CPU_SETSIZE -static inline int ioprio_set(int which, int who, int ioprio) -{ - return syscall(__NR_ioprio_set, which, who, ioprio); -} - -static inline int gettid(void) -{ - return syscall(__NR_gettid); -} - -/* - * Just check for SPLICE_F_MOVE, if that isn't there, assume the others - * aren't either. - */ -#ifndef SPLICE_F_MOVE -#define SPLICE_F_MOVE (0x01) /* move pages instead of copying */ -#define SPLICE_F_NONBLOCK (0x02) /* don't block on the pipe splicing (but */ - /* we may still block on the fd we splice */ - /* from/to, of course */ -#define SPLICE_F_MORE (0x04) /* expect more data */ -#define SPLICE_F_GIFT (0x08) /* pages passed in are a gift */ - -static inline int splice(int fdin, loff_t *off_in, int fdout, loff_t *off_out, - size_t len, unsigned int flags) -{ - return syscall(__NR_sys_splice, fdin, off_in, fdout, off_out, len, flags); -} - -static inline int tee(int fdin, int fdout, size_t len, unsigned int flags) -{ - return syscall(__NR_sys_tee, fdin, fdout, len, flags); -} - -static inline int vmsplice(int fd, const struct iovec *iov, - unsigned long nr_segs, unsigned int flags) -{ - return syscall(__NR_sys_vmsplice, fd, iov, nr_segs, flags); -} -#endif - -#define SPLICE_DEF_SIZE (64*1024) - -#ifdef FIO_HAVE_SYSLET - -struct syslet_uatom; -struct async_head_user; - -/* - * syslet stuff - */ -static inline struct syslet_uatom * -async_exec(struct syslet_uatom *atom, struct async_head_user *ahu) -{ - return (struct syslet_uatom *) syscall(__NR_async_exec, atom, ahu); -} - -static inline long -async_wait(unsigned long min_wait_events, unsigned long user_ring_idx, - struct async_head_user *ahu) -{ - return syscall(__NR_async_wait, min_wait_events, - user_ring_idx, ahu); -} - -static inline long async_thread(void *event, struct async_head_user *ahu) -{ - return syscall(__NR_async_thread, event, ahu); -} - -static inline long umem_add(unsigned long *uptr, unsigned long inc) -{ - return syscall(__NR_umem_add, uptr, inc); -} -#endif /* FIO_HAVE_SYSLET */ - enum { IOPRIO_CLASS_NONE, IOPRIO_CLASS_RT, @@ -207,6 +101,31 @@ enum { #define IOPRIO_BITS 16 #define IOPRIO_CLASS_SHIFT 13 +#define IOPRIO_MIN_PRIO 0 /* highest priority */ +#define IOPRIO_MAX_PRIO 7 /* lowest priority */ + +#define IOPRIO_MIN_PRIO_CLASS 0 +#define IOPRIO_MAX_PRIO_CLASS 3 + +static inline int ioprio_set(int which, int who, int ioprio_class, int ioprio) +{ + /* + * If no class is set, assume BE + */ + if (!ioprio_class) + ioprio_class = IOPRIO_CLASS_BE; + + ioprio |= ioprio_class << IOPRIO_CLASS_SHIFT; + return syscall(__NR_ioprio_set, which, who, ioprio); +} + +static inline int gettid(void) +{ + return syscall(__NR_gettid); +} + +#define SPLICE_DEF_SIZE (64*1024) + #ifndef BLKGETSIZE64 #define BLKGETSIZE64 _IOR(0x12,114,size_t) #endif @@ -293,21 +212,29 @@ static inline int fio_lookup_raw(dev_t dev, int *majdev, int *mindev) #define FIO_O_NOATIME 0 #endif +#ifdef O_ATOMIC +#define OS_O_ATOMIC O_ATOMIC +#else +#define OS_O_ATOMIC 040000000 +#endif + #ifdef MADV_REMOVE #define FIO_MADV_FREE MADV_REMOVE #endif -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define FIO_LITTLE_ENDIAN -#elif __BYTE_ORDER == __BIG_ENDIAN -#define FIO_BIG_ENDIAN +/* Check for GCC or Clang byte swap intrinsics */ +#if (__has_builtin(__builtin_bswap16) && __has_builtin(__builtin_bswap32) \ + && __has_builtin(__builtin_bswap64)) || (__GNUC__ > 4 \ + || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) /* fio_swapN */ +#define fio_swap16(x) __builtin_bswap16(x) +#define fio_swap32(x) __builtin_bswap32(x) +#define fio_swap64(x) __builtin_bswap64(x) #else -#error "Unknown endianness" -#endif - -#define fio_swap16(x) __bswap_16(x) -#define fio_swap32(x) __bswap_32(x) -#define fio_swap64(x) __bswap_64(x) +#include +#define fio_swap16(x) bswap_16(x) +#define fio_swap32(x) bswap_32(x) +#define fio_swap64(x) bswap_64(x) +#endif /* fio_swapN */ #define CACHE_LINE_FILE \ "/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size" @@ -331,7 +258,7 @@ static inline int arch_cache_line_size(void) return atoi(size); } -static inline unsigned long long get_fs_size(const char *path) +static inline unsigned long long get_fs_free_size(const char *path) { unsigned long long ret; struct statfs s; @@ -358,4 +285,78 @@ static inline int os_trim(int fd, unsigned long long start, return errno; } +#ifdef CONFIG_SCHED_IDLE +static inline int fio_set_sched_idle(void) +{ + struct sched_param p = { .sched_priority = 0, }; + return sched_setscheduler(gettid(), SCHED_IDLE, &p); +} +#endif + +#ifndef POSIX_FADV_STREAMID +#define POSIX_FADV_STREAMID 8 +#endif + +#define FIO_HAVE_STREAMID + +#ifndef RWF_HIPRI +#define RWF_HIPRI 0x00000001 +#endif +#ifndef RWF_DSYNC +#define RWF_DSYNC 0x00000002 +#endif +#ifndef RWF_SYNC +#define RWF_SYNC 0x00000004 +#endif + +#ifndef CONFIG_PWRITEV2 +#ifdef __NR_preadv2 +static inline void make_pos_h_l(unsigned long *pos_h, unsigned long *pos_l, + off_t offset) +{ +#if BITS_PER_LONG == 64 + *pos_l = offset; + *pos_h = 0; +#else + *pos_l = offset & 0xffffffff; + *pos_h = ((uint64_t) offset) >> 32; +#endif +} +static inline ssize_t preadv2(int fd, const struct iovec *iov, int iovcnt, + off_t offset, unsigned int flags) +{ + unsigned long pos_l, pos_h; + + make_pos_h_l(&pos_h, &pos_l, offset); + return syscall(__NR_preadv2, fd, iov, iovcnt, pos_l, pos_h, flags); +} +static inline ssize_t pwritev2(int fd, const struct iovec *iov, int iovcnt, + off_t offset, unsigned int flags) +{ + unsigned long pos_l, pos_h; + + make_pos_h_l(&pos_h, &pos_l, offset); + return syscall(__NR_pwritev2, fd, iov, iovcnt, pos_l, pos_h, flags); +} +#else +static inline ssize_t preadv2(int fd, const struct iovec *iov, int iovcnt, + off_t offset, unsigned int flags) +{ + errno = ENOSYS; + return -1; +} +static inline ssize_t pwritev2(int fd, const struct iovec *iov, int iovcnt, + off_t offset, unsigned int flags) +{ + errno = ENOSYS; + return -1; +} +#endif /* __NR_preadv2 */ +#endif /* CONFIG_PWRITEV2 */ + +static inline int shm_attach_to_open_removed(void) +{ + return 1; +} + #endif