cpuio engine cpuload bug fix
[fio.git] / os / os-linux.h
CommitLineData
ebac4655
JA
1#ifndef FIO_OS_LINUX_H
2#define FIO_OS_LINUX_H
3
cca84643
JA
4#define FIO_OS os_linux
5
ebac4655 6#include <sys/ioctl.h>
8756e4d4 7#include <sys/uio.h>
3c39a379 8#include <sys/syscall.h>
2e3bd4c2 9#include <sys/vfs.h>
3c39a379 10#include <unistd.h>
97fbdfa1 11#include <fcntl.h>
3c2d93ed 12#include <errno.h>
d09913c2 13#include <sched.h>
ea421790 14#include <linux/unistd.h>
07e5b264
JA
15#include <linux/raw.h>
16#include <linux/major.h>
610a730c 17#include <endian.h>
ebac4655 18
f3de88a7 19#include "indirect.h"
79a43187 20#include "binject.h"
e2e58886 21#include "../file.h"
f3de88a7 22
ebac4655
JA
23#define FIO_HAVE_LIBAIO
24#define FIO_HAVE_POSIXAIO
25#define FIO_HAVE_FADVISE
26#define FIO_HAVE_CPU_AFFINITY
27#define FIO_HAVE_DISK_UTIL
28#define FIO_HAVE_SGIO
ba4f8923 29#define FIO_HAVE_IOPRIO
8756e4d4 30#define FIO_HAVE_SPLICE
22f78b32 31#define FIO_HAVE_IOSCHED_SWITCH
2c0ecd28 32#define FIO_HAVE_ODIRECT
74b025b0 33#define FIO_HAVE_HUGETLB
07e5b264 34#define FIO_HAVE_RAWBIND
5e62c22a 35#define FIO_HAVE_BLKTRACE
5921e80c 36#define FIO_HAVE_STRSEP
207cb0f0 37#define FIO_HAVE_POSIXAIO_FSYNC
f356d01d 38#define FIO_HAVE_PSHARED_MUTEX
eb7ccf38 39#define FIO_HAVE_CL_SIZE
a696fa2a 40#define FIO_HAVE_CGROUPS
c36d16f5 41#define FIO_HAVE_FDATASYNC
2e3bd4c2 42#define FIO_HAVE_FS_STAT
a5f3027c 43#define FIO_HAVE_TRIM
79a43187 44#define FIO_HAVE_BINJECT
ecc314ba 45#define FIO_HAVE_CLOCK_MONOTONIC
47f767c1 46#define FIO_HAVE_GETTID
93bcfd20 47#define FIO_USE_GENERIC_INIT_RANDOM_STATE
1ecc95ca 48#define FIO_HAVE_E4_ENG
b6959b55 49
804bb9a1
JA
50/*
51 * Can only enable this for newer glibcs, or the header and defines are
52 * missing
53 */
52ad4b94
JA
54#if __GLIBC__ >= 2 && __GLIBC_MINOR__ >= 6
55#define FIO_HAVE_FALLOCATE
56#endif
78b269e3 57#if __GLIBC__ >= 2 && __GLIBC_MINOR__ >= 8
804bb9a1
JA
58#define FIO_HAVE_LINUX_FALLOCATE
59#endif
60
1ecc95ca
JA
61#ifdef FIO_HAVE_LINUX_FALLOCATE
62#define FIO_HAVE_FALLOC_ENG
63#endif
64
093729f7
JA
65#ifdef SYNC_FILE_RANGE_WAIT_BEFORE
66#define FIO_HAVE_SYNC_FILE_RANGE
67#endif
68
dc873b6f 69#define OS_MAP_ANON MAP_ANONYMOUS
ebac4655 70
4d8947de
SL
71#ifndef CLOCK_MONOTONIC
72#define CLOCK_MONOTONIC 1
73#endif
74
ebac4655 75typedef cpu_set_t os_cpu_mask_t;
4e78e405 76
6dfd46b9 77typedef struct drand48_data os_random_state_t;
ebac4655
JA
78
79/*
80 * we want fadvise64 really, but it's so tangled... later
81 */
4d8947de 82#ifdef FIO_HAVE_FADVISE
ebac4655
JA
83#define fadvise(fd, off, len, advice) \
84 posix_fadvise((fd), (off_t)(off), (len), (advice))
4d8947de 85#endif
ebac4655 86
c8f025f8
JA
87/*
88 * If you are on an ancient glibc (2.3.2), then define GLIBC_2_3_2 if you want
89 * the affinity helpers to work.
90 */
91#ifndef GLIBC_2_3_2
e8462bd8
JA
92#define fio_setaffinity(pid, cpumask) \
93 sched_setaffinity((pid), sizeof(cpumask), &(cpumask))
ebac4655
JA
94#define fio_getaffinity(pid, ptr) \
95 sched_getaffinity((pid), sizeof(cpu_set_t), (ptr))
c8f025f8 96#else
e8462bd8
JA
97#define fio_setaffinity(pid, cpumask) \
98 sched_setaffinity((pid), &(cpumask))
c8f025f8
JA
99#define fio_getaffinity(pid, ptr) \
100 sched_getaffinity((pid), (ptr))
101#endif
be4ecfdf 102
1f4c0a4f
JA
103#define fio_cpu_clear(mask, cpu) (void) CPU_CLR((cpu), (mask))
104#define fio_cpu_set(mask, cpu) (void) CPU_SET((cpu), (mask))
d2ce18b5
JA
105
106static inline int fio_cpuset_init(os_cpu_mask_t *mask)
107{
108 CPU_ZERO(mask);
109 return 0;
110}
111
112static inline int fio_cpuset_exit(os_cpu_mask_t *mask)
113{
114 return 0;
115}
6d459ee7
JA
116
117#define FIO_MAX_CPUS CPU_SETSIZE
4d8947de 118
ebac4655
JA
119static inline int ioprio_set(int which, int who, int ioprio)
120{
121 return syscall(__NR_ioprio_set, which, who, ioprio);
122}
123
47f767c1
JA
124static inline int gettid(void)
125{
126 return syscall(__NR_gettid);
127}
128
97fbdfa1
JA
129/*
130 * Just check for SPLICE_F_MOVE, if that isn't there, assume the others
131 * aren't either.
132 */
133#ifndef SPLICE_F_MOVE
134#define SPLICE_F_MOVE (0x01) /* move pages instead of copying */
135#define SPLICE_F_NONBLOCK (0x02) /* don't block on the pipe splicing (but */
136 /* we may still block on the fd we splice */
137 /* from/to, of course */
138#define SPLICE_F_MORE (0x04) /* expect more data */
139#define SPLICE_F_GIFT (0x08) /* pages passed in are a gift */
8756e4d4
JA
140
141static inline int splice(int fdin, loff_t *off_in, int fdout, loff_t *off_out,
495ee9b6 142 size_t len, unsigned int flags)
8756e4d4 143{
97fbdfa1 144 return syscall(__NR_sys_splice, fdin, off_in, fdout, off_out, len, flags);
8756e4d4
JA
145}
146
147static inline int tee(int fdin, int fdout, size_t len, unsigned int flags)
148{
97fbdfa1 149 return syscall(__NR_sys_tee, fdin, fdout, len, flags);
8756e4d4
JA
150}
151
152static inline int vmsplice(int fd, const struct iovec *iov,
153 unsigned long nr_segs, unsigned int flags)
154{
97fbdfa1 155 return syscall(__NR_sys_vmsplice, fd, iov, nr_segs, flags);
8756e4d4 156}
97fbdfa1 157#endif
8756e4d4 158
3feedc60
JA
159#define SPLICE_DEF_SIZE (64*1024)
160
b6959b55 161#ifdef FIO_HAVE_SYSLET
bf0dc8fa
IM
162
163struct syslet_uatom;
164struct async_head_user;
165
a4f4fdd7
JA
166/*
167 * syslet stuff
168 */
bf0dc8fa
IM
169static inline struct syslet_uatom *
170async_exec(struct syslet_uatom *atom, struct async_head_user *ahu)
a4f4fdd7 171{
2b52511f 172 return (struct syslet_uatom *) syscall(__NR_async_exec, atom, ahu);
a4f4fdd7
JA
173}
174
bf0dc8fa
IM
175static inline long
176async_wait(unsigned long min_wait_events, unsigned long user_ring_idx,
177 struct async_head_user *ahu)
a4f4fdd7 178{
bf0dc8fa
IM
179 return syscall(__NR_async_wait, min_wait_events,
180 user_ring_idx, ahu);
a4f4fdd7
JA
181}
182
7756b0d0 183static inline long async_thread(void *event, struct async_head_user *ahu)
a4f4fdd7 184{
7756b0d0 185 return syscall(__NR_async_thread, event, ahu);
a4f4fdd7
JA
186}
187
188static inline long umem_add(unsigned long *uptr, unsigned long inc)
189{
190 return syscall(__NR_umem_add, uptr, inc);
191}
b6959b55 192#endif /* FIO_HAVE_SYSLET */
a4f4fdd7 193
6cefbe33
JA
194enum {
195 IOPRIO_CLASS_NONE,
196 IOPRIO_CLASS_RT,
197 IOPRIO_CLASS_BE,
198 IOPRIO_CLASS_IDLE,
199};
200
ebac4655
JA
201enum {
202 IOPRIO_WHO_PROCESS = 1,
203 IOPRIO_WHO_PGRP,
204 IOPRIO_WHO_USER,
205};
206
6cefbe33 207#define IOPRIO_BITS 16
ebac4655
JA
208#define IOPRIO_CLASS_SHIFT 13
209
210#ifndef BLKGETSIZE64
211#define BLKGETSIZE64 _IOR(0x12,114,size_t)
212#endif
213
e5b401d4
JA
214#ifndef BLKFLSBUF
215#define BLKFLSBUF _IO(0x12,97)
216#endif
217
a5f3027c
JA
218#ifndef BLKDISCARD
219#define BLKDISCARD _IO(0x12,119)
220#endif
221
e2e58886 222static inline int blockdev_invalidate_cache(struct fio_file *f)
e5b401d4 223{
ecc314ba 224 return ioctl(f->fd, BLKFLSBUF);
e5b401d4
JA
225}
226
ecc314ba 227static inline int blockdev_size(struct fio_file *f, unsigned long long *bytes)
ebac4655 228{
ecc314ba 229 if (!ioctl(f->fd, BLKGETSIZE64, bytes))
ebac4655
JA
230 return 0;
231
232 return errno;
233}
234
32cd46a0
JA
235static inline unsigned long long os_phys_mem(void)
236{
237 long pagesize, pages;
238
239 pagesize = sysconf(_SC_PAGESIZE);
240 pages = sysconf(_SC_PHYS_PAGES);
241 if (pages == -1 || pagesize == -1)
242 return 0;
243
244 return (unsigned long long) pages * (unsigned long long) pagesize;
245}
246
6dfd46b9
JA
247static inline void os_random_seed(unsigned long seed, os_random_state_t *rs)
248{
249 srand48_r(seed, rs);
250}
251
252static inline long os_random_long(os_random_state_t *rs)
253{
254 long val;
255
256 lrand48_r(rs, &val);
257 return val;
258}
259
8cc7afa9 260static inline int fio_lookup_raw(dev_t dev, int *majdev, int *mindev)
07e5b264
JA
261{
262 struct raw_config_request rq;
263 int fd;
264
265 if (major(dev) != RAW_MAJOR)
8cc7afa9 266 return 1;
07e5b264
JA
267
268 /*
269 * we should be able to find /dev/rawctl or /dev/raw/rawctl
270 */
271 fd = open("/dev/rawctl", O_RDONLY);
272 if (fd < 0) {
273 fd = open("/dev/raw/rawctl", O_RDONLY);
274 if (fd < 0)
8cc7afa9 275 return 1;
07e5b264
JA
276 }
277
278 rq.raw_minor = minor(dev);
279 if (ioctl(fd, RAW_GETBIND, &rq) < 0) {
280 close(fd);
8cc7afa9 281 return 1;
07e5b264
JA
282 }
283
284 close(fd);
285 *majdev = rq.block_major;
286 *mindev = rq.block_minor;
8cc7afa9 287 return 0;
07e5b264
JA
288}
289
5b6f5c66 290#ifdef O_NOATIME
5921e80c 291#define FIO_O_NOATIME O_NOATIME
5b6f5c66
JA
292#else
293#define FIO_O_NOATIME 0
294#endif
5921e80c 295
a1c58075
JA
296#ifdef MADV_REMOVE
297#define FIO_MADV_FREE MADV_REMOVE
298#endif
299
610a730c
JA
300#if __BYTE_ORDER == __LITTLE_ENDIAN
301#define FIO_LITTLE_ENDIAN
302#elif __BYTE_ORDER == __BIG_ENDIAN
303#define FIO_BIG_ENDIAN
304#else
305#error "Unknown endianness"
306#endif
307
ff245192
JA
308#define fio_swap16(x) __bswap_16(x)
309#define fio_swap32(x) __bswap_32(x)
310#define fio_swap64(x) __bswap_64(x)
311
eb7ccf38
JA
312#define CACHE_LINE_FILE \
313 "/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size"
314
315static inline int arch_cache_line_size(void)
316{
317 char size[32];
318 int fd, ret;
319
320 fd = open(CACHE_LINE_FILE, O_RDONLY);
321 if (fd < 0)
322 return -1;
323
324 ret = read(fd, size, sizeof(size));
ec76f15a
JA
325
326 close(fd);
327
eb7ccf38
JA
328 if (ret <= 0)
329 return -1;
ec76f15a
JA
330 else
331 return atoi(size);
eb7ccf38
JA
332}
333
2e3bd4c2
JA
334static inline unsigned long long get_fs_size(const char *path)
335{
336 unsigned long long ret;
337 struct statfs s;
338
339 if (statfs(path, &s) < 0)
340 return -1ULL;
341
342 ret = s.f_bsize;
343 ret *= (unsigned long long) s.f_bfree;
344 return ret;
345}
346
a5f3027c
JA
347static inline int os_trim(int fd, unsigned long long start,
348 unsigned long long len)
349{
350 uint64_t range[2];
351
352 range[0] = start;
353 range[1] = len;
354
355 if (!ioctl(fd, BLKDISCARD, range))
356 return 0;
357
358 return errno;
359}
360
ebac4655 361#endif