net: fix receiver start time
[fio.git] / filesetup.c
1 #include <unistd.h>
2 #include <fcntl.h>
3 #include <string.h>
4 #include <assert.h>
5 #include <dirent.h>
6 #include <libgen.h>
7 #include <sys/stat.h>
8 #include <sys/mman.h>
9 #include <sys/types.h>
10
11 #include "fio.h"
12 #include "smalloc.h"
13 #include "filehash.h"
14 #include "os/os.h"
15 #include "hash.h"
16 #include "lib/axmap.h"
17
18 #ifdef FIO_HAVE_LINUX_FALLOCATE
19 #include <linux/falloc.h>
20 #endif
21
22 static int root_warn;
23
24 static inline void clear_error(struct thread_data *td)
25 {
26         td->error = 0;
27         td->verror[0] = '\0';
28 }
29
30 /*
31  * Leaves f->fd open on success, caller must close
32  */
33 static int extend_file(struct thread_data *td, struct fio_file *f)
34 {
35         int r, new_layout = 0, unlink_file = 0, flags;
36         unsigned long long left;
37         unsigned int bs;
38         char *b;
39
40         if (read_only) {
41                 log_err("fio: refusing extend of file due to read-only\n");
42                 return 0;
43         }
44
45         /*
46          * check if we need to lay the file out complete again. fio
47          * does that for operations involving reads, or for writes
48          * where overwrite is set
49          */
50         if (td_read(td) || (td_write(td) && td->o.overwrite) ||
51             (td_write(td) && td->io_ops->flags & FIO_NOEXTEND))
52                 new_layout = 1;
53         if (td_write(td) && !td->o.overwrite)
54                 unlink_file = 1;
55
56         if (unlink_file || new_layout) {
57                 dprint(FD_FILE, "layout unlink %s\n", f->file_name);
58                 if ((unlink(f->file_name) < 0) && (errno != ENOENT)) {
59                         td_verror(td, errno, "unlink");
60                         return 1;
61                 }
62         }
63
64         flags = O_WRONLY | O_CREAT;
65         if (new_layout)
66                 flags |= O_TRUNC;
67
68         dprint(FD_FILE, "open file %s, flags %x\n", f->file_name, flags);
69         f->fd = open(f->file_name, flags, 0644);
70         if (f->fd < 0) {
71                 td_verror(td, errno, "open");
72                 return 1;
73         }
74
75 #ifdef FIO_HAVE_FALLOCATE
76         if (!td->o.fill_device) {
77                 switch (td->o.fallocate_mode) {
78                 case FIO_FALLOCATE_NONE:
79                         break;
80                 case FIO_FALLOCATE_POSIX:
81                         dprint(FD_FILE, "posix_fallocate file %s size %llu\n",
82                                  f->file_name, f->real_file_size);
83
84                         r = posix_fallocate(f->fd, 0, f->real_file_size);
85                         if (r > 0) {
86                                 log_err("fio: posix_fallocate fails: %s\n",
87                                                 strerror(r));
88                         }
89                         break;
90 #ifdef FIO_HAVE_LINUX_FALLOCATE
91                 case FIO_FALLOCATE_KEEP_SIZE:
92                         dprint(FD_FILE,
93                                 "fallocate(FALLOC_FL_KEEP_SIZE) "
94                                 "file %s size %llu\n",
95                                 f->file_name, f->real_file_size);
96
97                         r = fallocate(f->fd, FALLOC_FL_KEEP_SIZE, 0,
98                                         f->real_file_size);
99                         if (r != 0) {
100                                 td_verror(td, errno, "fallocate");
101                         }
102                         break;
103 #endif /* FIO_HAVE_LINUX_FALLOCATE */
104                 default:
105                         log_err("fio: unknown fallocate mode: %d\n",
106                                 td->o.fallocate_mode);
107                         assert(0);
108                 }
109         }
110 #endif /* FIO_HAVE_FALLOCATE */
111
112         if (!new_layout)
113                 goto done;
114
115         /*
116          * The size will be -1ULL when fill_device is used, so don't truncate
117          * or fallocate this file, just write it
118          */
119         if (!td->o.fill_device) {
120                 dprint(FD_FILE, "truncate file %s, size %llu\n", f->file_name,
121                                                         f->real_file_size);
122                 if (ftruncate(f->fd, f->real_file_size) == -1) {
123                         td_verror(td, errno, "ftruncate");
124                         goto err;
125                 }
126         }
127
128         b = malloc(td->o.max_bs[DDIR_WRITE]);
129         memset(b, 0, td->o.max_bs[DDIR_WRITE]);
130
131         left = f->real_file_size;
132         while (left && !td->terminate) {
133                 bs = td->o.max_bs[DDIR_WRITE];
134                 if (bs > left)
135                         bs = left;
136
137                 r = write(f->fd, b, bs);
138
139                 if (r > 0) {
140                         left -= r;
141                         continue;
142                 } else {
143                         if (r < 0) {
144                                 int __e = errno;
145
146                                 if (__e == ENOSPC) {
147                                         if (td->o.fill_device)
148                                                 break;
149                                         log_info("fio: ENOSPC on laying out "
150                                                  "file, stopping\n");
151                                         break;
152                                 }
153                                 td_verror(td, errno, "write");
154                         } else
155                                 td_verror(td, EIO, "write");
156
157                         break;
158                 }
159         }
160
161         if (td->terminate) {
162                 dprint(FD_FILE, "terminate unlink %s\n", f->file_name);
163                 unlink(f->file_name);
164         } else if (td->o.create_fsync) {
165                 if (fsync(f->fd) < 0) {
166                         td_verror(td, errno, "fsync");
167                         goto err;
168                 }
169         }
170         if (td->o.fill_device && !td_write(td)) {
171                 fio_file_clear_size_known(f);
172                 if (td_io_get_file_size(td, f))
173                         goto err;
174                 if (f->io_size > f->real_file_size)
175                         f->io_size = f->real_file_size;
176         }
177
178         free(b);
179 done:
180         return 0;
181 err:
182         close(f->fd);
183         f->fd = -1;
184         return 1;
185 }
186
187 static int pre_read_file(struct thread_data *td, struct fio_file *f)
188 {
189         int r, did_open = 0, old_runstate;
190         unsigned long long left;
191         unsigned int bs;
192         char *b;
193
194         if (td->io_ops->flags & FIO_PIPEIO)
195                 return 0;
196
197         if (!fio_file_open(f)) {
198                 if (td->io_ops->open_file(td, f)) {
199                         log_err("fio: cannot pre-read, failed to open file\n");
200                         return 1;
201                 }
202                 did_open = 1;
203         }
204
205         old_runstate = td->runstate;
206         td_set_runstate(td, TD_PRE_READING);
207
208         bs = td->o.max_bs[DDIR_READ];
209         b = malloc(bs);
210         memset(b, 0, bs);
211
212         lseek(f->fd, f->file_offset, SEEK_SET);
213         left = f->io_size;
214
215         while (left && !td->terminate) {
216                 if (bs > left)
217                         bs = left;
218
219                 r = read(f->fd, b, bs);
220
221                 if (r == (int) bs) {
222                         left -= bs;
223                         continue;
224                 } else {
225                         td_verror(td, EIO, "pre_read");
226                         break;
227                 }
228         }
229
230         td_set_runstate(td, old_runstate);
231
232         if (did_open)
233                 td->io_ops->close_file(td, f);
234         free(b);
235         return 0;
236 }
237
238 static unsigned long long get_rand_file_size(struct thread_data *td)
239 {
240         unsigned long long ret, sized;
241         unsigned long r;
242
243         if (td->o.use_os_rand) {
244                 r = os_random_long(&td->file_size_state);
245                 sized = td->o.file_size_high - td->o.file_size_low;
246                 ret = (unsigned long long) ((double) sized * (r / (OS_RAND_MAX + 1.0)));
247         } else {
248                 r = __rand(&td->__file_size_state);
249                 sized = td->o.file_size_high - td->o.file_size_low;
250                 ret = (unsigned long long) ((double) sized * (r / (FRAND_MAX + 1.0)));
251         }
252
253         ret += td->o.file_size_low;
254         ret -= (ret % td->o.rw_min_bs);
255         return ret;
256 }
257
258 static int file_size(struct thread_data *td, struct fio_file *f)
259 {
260         struct stat st;
261
262         if (stat(f->file_name, &st) == -1) {
263                 td_verror(td, errno, "fstat");
264                 return 1;
265         }
266
267         f->real_file_size = st.st_size;
268         return 0;
269 }
270
271 static int bdev_size(struct thread_data *td, struct fio_file *f)
272 {
273         unsigned long long bytes = 0;
274         int r;
275
276         if (td->io_ops->open_file(td, f)) {
277                 log_err("fio: failed opening blockdev %s for size check\n",
278                         f->file_name);
279                 return 1;
280         }
281
282         r = blockdev_size(f, &bytes);
283         if (r) {
284                 td_verror(td, r, "blockdev_size");
285                 goto err;
286         }
287
288         if (!bytes) {
289                 log_err("%s: zero sized block device?\n", f->file_name);
290                 goto err;
291         }
292
293         f->real_file_size = bytes;
294         td->io_ops->close_file(td, f);
295         return 0;
296 err:
297         td->io_ops->close_file(td, f);
298         return 1;
299 }
300
301 static int char_size(struct thread_data *td, struct fio_file *f)
302 {
303 #ifdef FIO_HAVE_CHARDEV_SIZE
304         unsigned long long bytes = 0;
305         int r;
306
307         if (td->io_ops->open_file(td, f)) {
308                 log_err("fio: failed opening blockdev %s for size check\n",
309                         f->file_name);
310                 return 1;
311         }
312
313         r = chardev_size(f, &bytes);
314         if (r) {
315                 td_verror(td, r, "chardev_size");
316                 goto err;
317         }
318
319         if (!bytes) {
320                 log_err("%s: zero sized char device?\n", f->file_name);
321                 goto err;
322         }
323
324         f->real_file_size = bytes;
325         td->io_ops->close_file(td, f);
326         return 0;
327 err:
328         td->io_ops->close_file(td, f);
329         return 1;
330 #else
331         f->real_file_size = -1ULL;
332         return 0;
333 #endif
334 }
335
336 static int get_file_size(struct thread_data *td, struct fio_file *f)
337 {
338         int ret = 0;
339
340         if (fio_file_size_known(f))
341                 return 0;
342
343         if (f->filetype == FIO_TYPE_FILE)
344                 ret = file_size(td, f);
345         else if (f->filetype == FIO_TYPE_BD)
346                 ret = bdev_size(td, f);
347         else if (f->filetype == FIO_TYPE_CHAR)
348                 ret = char_size(td, f);
349         else
350                 f->real_file_size = -1;
351
352         if (ret)
353                 return ret;
354
355         if (f->file_offset > f->real_file_size) {
356                 log_err("%s: offset extends end (%llu > %llu)\n", td->o.name,
357                                         f->file_offset, f->real_file_size);
358                 return 1;
359         }
360
361         fio_file_set_size_known(f);
362         return 0;
363 }
364
365 static int __file_invalidate_cache(struct thread_data *td, struct fio_file *f,
366                                    unsigned long long off,
367                                    unsigned long long len)
368 {
369         int ret = 0;
370
371         if (len == -1ULL)
372                 len = f->io_size;
373         if (off == -1ULL)
374                 off = f->file_offset;
375
376         if (len == -1ULL || off == -1ULL)
377                 return 0;
378
379         dprint(FD_IO, "invalidate cache %s: %llu/%llu\n", f->file_name, off,
380                                                                 len);
381
382         /*
383          * FIXME: add blockdev flushing too
384          */
385         if (f->mmap_ptr) {
386                 ret = posix_madvise(f->mmap_ptr, f->mmap_sz, POSIX_MADV_DONTNEED);
387 #ifdef FIO_MADV_FREE
388                 (void) posix_madvise(f->mmap_ptr, f->mmap_sz, FIO_MADV_FREE);
389 #endif
390         } else if (f->filetype == FIO_TYPE_FILE) {
391                 ret = posix_fadvise(f->fd, off, len, POSIX_FADV_DONTNEED);
392         } else if (f->filetype == FIO_TYPE_BD) {
393                 ret = blockdev_invalidate_cache(f);
394                 if (ret < 0 && errno == EACCES && geteuid()) {
395                         if (!root_warn) {
396                                 log_err("fio: only root may flush block "
397                                         "devices. Cache flush bypassed!\n");
398                                 root_warn = 1;
399                         }
400                         ret = 0;
401                 }
402         } else if (f->filetype == FIO_TYPE_CHAR || f->filetype == FIO_TYPE_PIPE)
403                 ret = 0;
404
405         if (ret < 0) {
406                 td_verror(td, errno, "invalidate_cache");
407                 return 1;
408         } else if (ret > 0) {
409                 td_verror(td, ret, "invalidate_cache");
410                 return 1;
411         }
412
413         return ret;
414
415 }
416
417 int file_invalidate_cache(struct thread_data *td, struct fio_file *f)
418 {
419         if (!fio_file_open(f))
420                 return 0;
421
422         return __file_invalidate_cache(td, f, -1ULL, -1ULL);
423 }
424
425 int generic_close_file(struct thread_data fio_unused *td, struct fio_file *f)
426 {
427         int ret = 0;
428
429         dprint(FD_FILE, "fd close %s\n", f->file_name);
430
431         remove_file_hash(f);
432
433         if (close(f->fd) < 0)
434                 ret = errno;
435
436         f->fd = -1;
437         return ret;
438 }
439
440 int file_lookup_open(struct fio_file *f, int flags)
441 {
442         struct fio_file *__f;
443         int from_hash;
444
445         __f = lookup_file_hash(f->file_name);
446         if (__f) {
447                 dprint(FD_FILE, "found file in hash %s\n", f->file_name);
448                 /*
449                  * racy, need the __f->lock locked
450                  */
451                 f->lock = __f->lock;
452                 f->lock_owner = __f->lock_owner;
453                 f->lock_batch = __f->lock_batch;
454                 f->lock_ddir = __f->lock_ddir;
455                 from_hash = 1;
456         } else {
457                 dprint(FD_FILE, "file not found in hash %s\n", f->file_name);
458                 from_hash = 0;
459         }
460
461         f->fd = open(f->file_name, flags, 0600);
462         return from_hash;
463 }
464
465 int generic_open_file(struct thread_data *td, struct fio_file *f)
466 {
467         int is_std = 0;
468         int flags = 0;
469         int from_hash = 0;
470
471         dprint(FD_FILE, "fd open %s\n", f->file_name);
472
473         if (td_trim(td) && f->filetype != FIO_TYPE_BD) {
474                 log_err("fio: trim only applies to block device\n");
475                 return 1;
476         }
477
478         if (!strcmp(f->file_name, "-")) {
479                 if (td_rw(td)) {
480                         log_err("fio: can't read/write to stdin/out\n");
481                         return 1;
482                 }
483                 is_std = 1;
484
485                 /*
486                  * move output logging to stderr, if we are writing to stdout
487                  */
488                 if (td_write(td))
489                         f_out = stderr;
490         }
491
492         if (td_trim(td))
493                 goto skip_flags;
494         if (td->o.odirect)
495                 flags |= OS_O_DIRECT;
496         if (td->o.sync_io)
497                 flags |= O_SYNC;
498         if (td->o.create_on_open)
499                 flags |= O_CREAT;
500 skip_flags:
501         if (f->filetype != FIO_TYPE_FILE)
502                 flags |= FIO_O_NOATIME;
503
504 open_again:
505         if (td_write(td)) {
506                 if (!read_only)
507                         flags |= O_RDWR;
508
509                 if (f->filetype == FIO_TYPE_FILE)
510                         flags |= O_CREAT;
511
512                 if (is_std)
513                         f->fd = dup(STDOUT_FILENO);
514                 else
515                         from_hash = file_lookup_open(f, flags);
516         } else if (td_read(td)) {
517                 if (f->filetype == FIO_TYPE_CHAR && !read_only)
518                         flags |= O_RDWR;
519                 else
520                         flags |= O_RDONLY;
521
522                 if (is_std)
523                         f->fd = dup(STDIN_FILENO);
524                 else
525                         from_hash = file_lookup_open(f, flags);
526         } else { //td trim
527                 flags |= O_RDWR;
528                 from_hash = file_lookup_open(f, flags);
529         }
530
531         if (f->fd == -1) {
532                 char buf[FIO_VERROR_SIZE];
533                 int __e = errno;
534
535                 if (__e == EPERM && (flags & FIO_O_NOATIME)) {
536                         flags &= ~FIO_O_NOATIME;
537                         goto open_again;
538                 }
539
540                 snprintf(buf, sizeof(buf) - 1, "open(%s)", f->file_name);
541
542                 if (__e == EINVAL && (flags & OS_O_DIRECT)) {
543                         log_err("fio: looks like your file system does not " \
544                                 "support direct=1/buffered=0\n");
545                 }
546
547                 td_verror(td, __e, buf);
548         }
549
550         if (!from_hash && f->fd != -1) {
551                 if (add_file_hash(f)) {
552                         int fio_unused ret;
553
554                         /*
555                          * OK to ignore, we haven't done anything with it
556                          */
557                         ret = generic_close_file(td, f);
558                         goto open_again;
559                 }
560         }
561
562         return 0;
563 }
564
565 int generic_get_file_size(struct thread_data *td, struct fio_file *f)
566 {
567         return get_file_size(td, f);
568 }
569
570 /*
571  * open/close all files, so that ->real_file_size gets set
572  */
573 static int get_file_sizes(struct thread_data *td)
574 {
575         struct fio_file *f;
576         unsigned int i;
577         int err = 0;
578
579         for_each_file(td, f, i) {
580                 dprint(FD_FILE, "get file size for %p/%d/%p\n", f, i,
581                                                                 f->file_name);
582
583                 if (td_io_get_file_size(td, f)) {
584                         if (td->error != ENOENT) {
585                                 log_err("%s\n", td->verror);
586                                 err = 1;
587                         }
588                         clear_error(td);
589                 }
590
591                 if (f->real_file_size == -1ULL && td->o.size)
592                         f->real_file_size = td->o.size / td->o.nr_files;
593         }
594
595         return err;
596 }
597
598 struct fio_mount {
599         struct flist_head list;
600         const char *base;
601         char __base[256];
602         unsigned int key;
603 };
604
605 /*
606  * Get free number of bytes for each file on each unique mount.
607  */
608 static unsigned long long get_fs_free_counts(struct thread_data *td)
609 {
610         struct flist_head *n, *tmp;
611         unsigned long long ret = 0;
612         struct fio_mount *fm;
613         FLIST_HEAD(list);
614         struct fio_file *f;
615         unsigned int i;
616
617         for_each_file(td, f, i) {
618                 struct stat sb;
619                 char buf[256];
620
621                 if (f->filetype == FIO_TYPE_BD || f->filetype == FIO_TYPE_CHAR) {
622                         if (f->real_file_size != -1ULL)
623                                 ret += f->real_file_size;
624                         continue;
625                 } else if (f->filetype != FIO_TYPE_FILE)
626                         continue;
627
628                 strcpy(buf, f->file_name);
629
630                 if (stat(buf, &sb) < 0) {
631                         if (errno != ENOENT)
632                                 break;
633                         strcpy(buf, ".");
634                         if (stat(buf, &sb) < 0)
635                                 break;
636                 }
637
638                 fm = NULL;
639                 flist_for_each(n, &list) {
640                         fm = flist_entry(n, struct fio_mount, list);
641                         if (fm->key == sb.st_dev)
642                                 break;
643
644                         fm = NULL;
645                 }
646
647                 if (fm)
648                         continue;
649
650                 fm = malloc(sizeof(*fm));
651                 strcpy(fm->__base, buf);
652                 fm->base = basename(fm->__base);
653                 fm->key = sb.st_dev;
654                 flist_add(&fm->list, &list);
655         }
656
657         flist_for_each_safe(n, tmp, &list) {
658                 unsigned long long sz;
659
660                 fm = flist_entry(n, struct fio_mount, list);
661                 flist_del(&fm->list);
662
663                 sz = get_fs_size(fm->base);
664                 if (sz && sz != -1ULL)
665                         ret += sz;
666
667                 free(fm);
668         }
669
670         return ret;
671 }
672
673 unsigned long long get_start_offset(struct thread_data *td)
674 {
675         return td->o.start_offset +
676                 (td->thread_number - 1) * td->o.offset_increment;
677 }
678
679 /*
680  * Open the files and setup files sizes, creating files if necessary.
681  */
682 int setup_files(struct thread_data *td)
683 {
684         unsigned long long total_size, extend_size;
685         struct fio_file *f;
686         unsigned int i;
687         int err = 0, need_extend;
688
689         dprint(FD_FILE, "setup files\n");
690
691         if (td->o.read_iolog_file)
692                 goto done;
693
694         /*
695          * if ioengine defines a setup() method, it's responsible for
696          * opening the files and setting f->real_file_size to indicate
697          * the valid range for that file.
698          */
699         if (td->io_ops->setup)
700                 err = td->io_ops->setup(td);
701         else
702                 err = get_file_sizes(td);
703
704         if (err)
705                 return err;
706
707         /*
708          * check sizes. if the files/devices do not exist and the size
709          * isn't passed to fio, abort.
710          */
711         total_size = 0;
712         for_each_file(td, f, i) {
713                 if (f->real_file_size == -1ULL)
714                         total_size = -1ULL;
715                 else
716                         total_size += f->real_file_size;
717         }
718
719         if (td->o.fill_device)
720                 td->fill_device_size = get_fs_free_counts(td);
721
722         /*
723          * device/file sizes are zero and no size given, punt
724          */
725         if ((!total_size || total_size == -1ULL) && !td->o.size &&
726             !(td->io_ops->flags & FIO_NOIO) && !td->o.fill_device) {
727                 log_err("%s: you need to specify size=\n", td->o.name);
728                 td_verror(td, EINVAL, "total_file_size");
729                 return 1;
730         }
731
732         /*
733          * now file sizes are known, so we can set ->io_size. if size= is
734          * not given, ->io_size is just equal to ->real_file_size. if size
735          * is given, ->io_size is size / nr_files.
736          */
737         extend_size = total_size = 0;
738         need_extend = 0;
739         for_each_file(td, f, i) {
740                 f->file_offset = get_start_offset(td);
741
742                 if (!td->o.file_size_low) {
743                         /*
744                          * no file size range given, file size is equal to
745                          * total size divided by number of files. if that is
746                          * zero, set it to the real file size.
747                          */
748                         f->io_size = td->o.size / td->o.nr_files;
749                         if (!f->io_size)
750                                 f->io_size = f->real_file_size - f->file_offset;
751                 } else if (f->real_file_size < td->o.file_size_low ||
752                            f->real_file_size > td->o.file_size_high) {
753                         if (f->file_offset > td->o.file_size_low)
754                                 goto err_offset;
755                         /*
756                          * file size given. if it's fixed, use that. if it's a
757                          * range, generate a random size in-between.
758                          */
759                         if (td->o.file_size_low == td->o.file_size_high) {
760                                 f->io_size = td->o.file_size_low
761                                                 - f->file_offset;
762                         } else {
763                                 f->io_size = get_rand_file_size(td)
764                                                 - f->file_offset;
765                         }
766                 } else
767                         f->io_size = f->real_file_size - f->file_offset;
768
769                 if (f->io_size == -1ULL)
770                         total_size = -1ULL;
771                 else {
772                         if (td->o.size_percent)
773                                 f->io_size = (f->io_size * td->o.size_percent) / 100;
774                         total_size += f->io_size;
775                 }
776
777                 if (f->filetype == FIO_TYPE_FILE &&
778                     (f->io_size + f->file_offset) > f->real_file_size &&
779                     !(td->io_ops->flags & FIO_DISKLESSIO)) {
780                         if (!td->o.create_on_open) {
781                                 need_extend++;
782                                 extend_size += (f->io_size + f->file_offset);
783                         } else
784                                 f->real_file_size = f->io_size + f->file_offset;
785                         fio_file_set_extend(f);
786                 }
787         }
788
789         if (!td->o.size || td->o.size > total_size)
790                 td->o.size = total_size;
791
792         /*
793          * See if we need to extend some files
794          */
795         if (need_extend) {
796                 temp_stall_ts = 1;
797                 if (output_format == FIO_OUTPUT_NORMAL)
798                         log_info("%s: Laying out IO file(s) (%u file(s) /"
799                                  " %lluMB)\n", td->o.name, need_extend,
800                                         extend_size >> 20);
801
802                 for_each_file(td, f, i) {
803                         unsigned long long old_len = -1ULL, extend_len = -1ULL;
804
805                         if (!fio_file_extend(f))
806                                 continue;
807
808                         assert(f->filetype == FIO_TYPE_FILE);
809                         fio_file_clear_extend(f);
810                         if (!td->o.fill_device) {
811                                 old_len = f->real_file_size;
812                                 extend_len = f->io_size + f->file_offset -
813                                                 old_len;
814                         }
815                         f->real_file_size = (f->io_size + f->file_offset);
816                         err = extend_file(td, f);
817                         if (err)
818                                 break;
819
820                         err = __file_invalidate_cache(td, f, old_len,
821                                                                 extend_len);
822                         close(f->fd);
823                         f->fd = -1;
824                         if (err)
825                                 break;
826                 }
827                 temp_stall_ts = 0;
828         }
829
830         if (err)
831                 return err;
832
833         if (!td->o.zone_size)
834                 td->o.zone_size = td->o.size;
835
836         /*
837          * iolog already set the total io size, if we read back
838          * stored entries.
839          */
840         if (!td->o.read_iolog_file)
841                 td->total_io_size = td->o.size * td->o.loops;
842
843 done:
844         if (td->o.create_only)
845                 td->done = 1;
846
847         return 0;
848 err_offset:
849         log_err("%s: you need to specify valid offset=\n", td->o.name);
850         return 1;
851 }
852
853 int pre_read_files(struct thread_data *td)
854 {
855         struct fio_file *f;
856         unsigned int i;
857
858         dprint(FD_FILE, "pre_read files\n");
859
860         for_each_file(td, f, i) {
861                 pre_read_file(td, f);
862         }
863
864         return 1;
865 }
866
867 static int __init_rand_distribution(struct thread_data *td, struct fio_file *f)
868 {
869         unsigned int range_size, seed;
870         unsigned long nranges;
871
872         range_size = min(td->o.min_bs[DDIR_READ], td->o.min_bs[DDIR_WRITE]);
873
874         nranges = (f->real_file_size + range_size - 1) / range_size;
875
876         seed = jhash(f->file_name, strlen(f->file_name), 0) * td->thread_number;
877         if (!td->o.rand_repeatable)
878                 seed = td->rand_seeds[4];
879
880         if (td->o.random_distribution == FIO_RAND_DIST_ZIPF)
881                 zipf_init(&f->zipf, nranges, td->o.zipf_theta, seed);
882         else
883                 pareto_init(&f->zipf, nranges, td->o.pareto_h, seed);
884
885         return 1;
886 }
887
888 static int init_rand_distribution(struct thread_data *td)
889 {
890         struct fio_file *f;
891         unsigned int i;
892         int state;
893
894         if (td->o.random_distribution == FIO_RAND_DIST_RANDOM)
895                 return 0;
896
897         state = td->runstate;
898         td_set_runstate(td, TD_SETTING_UP);
899         for_each_file(td, f, i)
900                 __init_rand_distribution(td, f);
901         td_set_runstate(td, state);
902
903         return 1;
904 }
905
906 int init_random_map(struct thread_data *td)
907 {
908         unsigned long long blocks;
909         struct fio_file *f;
910         unsigned int i;
911
912         if (init_rand_distribution(td))
913                 return 0;
914         if (!td_random(td))
915                 return 0;
916
917         for_each_file(td, f, i) {
918                 blocks = (f->real_file_size + td->o.rw_min_bs - 1) /
919                                 (unsigned long long) td->o.rw_min_bs;
920                 if (td->o.random_generator == FIO_RAND_GEN_LFSR) {
921                         if (!lfsr_init(&f->lfsr, blocks))
922                                 continue;
923                 } else if (!td->o.norandommap) {
924                         f->io_axmap = axmap_new(blocks);
925                         if (f->io_axmap)
926                                 continue;
927                 } else if (td->o.norandommap)
928                         continue;
929
930                 if (!td->o.softrandommap) {
931                         log_err("fio: failed allocating random map. If running"
932                                 " a large number of jobs, try the 'norandommap'"
933                                 " option or set 'softrandommap'. Or give"
934                                 " a larger --alloc-size to fio.\n");
935                         return 1;
936                 }
937
938                 log_info("fio: file %s failed allocating random map. Running "
939                          "job without.\n", f->file_name);
940         }
941
942         return 0;
943 }
944
945 void close_files(struct thread_data *td)
946 {
947         struct fio_file *f;
948         unsigned int i;
949
950         for_each_file(td, f, i) {
951                 if (fio_file_open(f))
952                         td_io_close_file(td, f);
953         }
954 }
955
956 void close_and_free_files(struct thread_data *td)
957 {
958         struct fio_file *f;
959         unsigned int i;
960
961         dprint(FD_FILE, "close files\n");
962
963         for_each_file(td, f, i) {
964                 if (td->o.unlink && f->filetype == FIO_TYPE_FILE) {
965                         dprint(FD_FILE, "free unlink %s\n", f->file_name);
966                         unlink(f->file_name);
967                 }
968
969                 if (fio_file_open(f))
970                         td_io_close_file(td, f);
971
972                 remove_file_hash(f);
973
974                 sfree(f->file_name);
975                 f->file_name = NULL;
976                 axmap_free(f->io_axmap);
977                 f->io_axmap = NULL;
978                 sfree(f);
979         }
980
981         td->o.filename = NULL;
982         free(td->files);
983         td->files_index = 0;
984         td->files = NULL;
985         td->o.nr_files = 0;
986 }
987
988 static void get_file_type(struct fio_file *f)
989 {
990         struct stat sb;
991
992         if (!strcmp(f->file_name, "-"))
993                 f->filetype = FIO_TYPE_PIPE;
994         else
995                 f->filetype = FIO_TYPE_FILE;
996
997         /* \\.\ is the device namespace in Windows, where every file is
998          * a block device */
999         if (strncmp(f->file_name, "\\\\.\\", 4) == 0)
1000                 f->filetype = FIO_TYPE_BD;
1001
1002         if (!stat(f->file_name, &sb)) {
1003                 if (S_ISBLK(sb.st_mode))
1004                         f->filetype = FIO_TYPE_BD;
1005                 else if (S_ISCHR(sb.st_mode))
1006                         f->filetype = FIO_TYPE_CHAR;
1007                 else if (S_ISFIFO(sb.st_mode))
1008                         f->filetype = FIO_TYPE_PIPE;
1009         }
1010 }
1011
1012 int add_file(struct thread_data *td, const char *fname)
1013 {
1014         int cur_files = td->files_index;
1015         char file_name[PATH_MAX];
1016         struct fio_file *f;
1017         int len = 0;
1018
1019         dprint(FD_FILE, "add file %s\n", fname);
1020
1021         f = smalloc(sizeof(*f));
1022         if (!f) {
1023                 log_err("fio: smalloc OOM\n");
1024                 assert(0);
1025         }
1026
1027         f->fd = -1;
1028         fio_file_reset(f);
1029
1030         if (td->files_size <= td->files_index) {
1031                 int new_size = td->o.nr_files + 1;
1032
1033                 dprint(FD_FILE, "resize file array to %d files\n", new_size);
1034
1035                 td->files = realloc(td->files, new_size * sizeof(f));
1036                 td->files_size = new_size;
1037         }
1038         td->files[cur_files] = f;
1039         f->fileno = cur_files;
1040
1041         /*
1042          * init function, io engine may not be loaded yet
1043          */
1044         if (td->io_ops && (td->io_ops->flags & FIO_DISKLESSIO))
1045                 f->real_file_size = -1ULL;
1046
1047         if (td->o.directory)
1048                 len = sprintf(file_name, "%s/", td->o.directory);
1049
1050         sprintf(file_name + len, "%s", fname);
1051         f->file_name = smalloc_strdup(file_name);
1052         if (!f->file_name) {
1053                 log_err("fio: smalloc OOM\n");
1054                 assert(0);
1055         }
1056
1057         get_file_type(f);
1058
1059         switch (td->o.file_lock_mode) {
1060         case FILE_LOCK_NONE:
1061                 break;
1062         case FILE_LOCK_READWRITE:
1063                 f->lock = fio_mutex_rw_init();
1064                 break;
1065         case FILE_LOCK_EXCLUSIVE:
1066                 f->lock = fio_mutex_init(FIO_MUTEX_UNLOCKED);
1067                 break;
1068         default:
1069                 log_err("fio: unknown lock mode: %d\n", td->o.file_lock_mode);
1070                 assert(0);
1071         }
1072
1073         td->files_index++;
1074         if (f->filetype == FIO_TYPE_FILE)
1075                 td->nr_normal_files++;
1076
1077         dprint(FD_FILE, "file %p \"%s\" added at %d\n", f, f->file_name,
1078                                                         cur_files);
1079
1080         return cur_files;
1081 }
1082
1083 int add_file_exclusive(struct thread_data *td, const char *fname)
1084 {
1085         struct fio_file *f;
1086         unsigned int i;
1087
1088         for_each_file(td, f, i) {
1089                 if (!strcmp(f->file_name, fname))
1090                         return i;
1091         }
1092
1093         return add_file(td, fname);
1094 }
1095
1096 void get_file(struct fio_file *f)
1097 {
1098         dprint(FD_FILE, "get file %s, ref=%d\n", f->file_name, f->references);
1099         assert(fio_file_open(f));
1100         f->references++;
1101 }
1102
1103 int put_file(struct thread_data *td, struct fio_file *f)
1104 {
1105         int f_ret = 0, ret = 0;
1106
1107         dprint(FD_FILE, "put file %s, ref=%d\n", f->file_name, f->references);
1108
1109         if (!fio_file_open(f)) {
1110                 assert(f->fd == -1);
1111                 return 0;
1112         }
1113
1114         assert(f->references);
1115         if (--f->references)
1116                 return 0;
1117
1118         if (should_fsync(td) && td->o.fsync_on_close)
1119                 f_ret = fsync(f->fd);
1120
1121         if (td->io_ops->close_file)
1122                 ret = td->io_ops->close_file(td, f);
1123
1124         if (!ret)
1125                 ret = f_ret;
1126
1127         td->nr_open_files--;
1128         fio_file_clear_open(f);
1129         assert(f->fd == -1);
1130         return ret;
1131 }
1132
1133 void lock_file(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir)
1134 {
1135         if (!f->lock || td->o.file_lock_mode == FILE_LOCK_NONE)
1136                 return;
1137
1138         if (f->lock_owner == td && f->lock_batch--)
1139                 return;
1140
1141         if (td->o.file_lock_mode == FILE_LOCK_READWRITE) {
1142                 if (ddir == DDIR_READ)
1143                         fio_mutex_down_read(f->lock);
1144                 else
1145                         fio_mutex_down_write(f->lock);
1146         } else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE)
1147                 fio_mutex_down(f->lock);
1148
1149         f->lock_owner = td;
1150         f->lock_batch = td->o.lockfile_batch;
1151         f->lock_ddir = ddir;
1152 }
1153
1154 void unlock_file(struct thread_data *td, struct fio_file *f)
1155 {
1156         if (!f->lock || td->o.file_lock_mode == FILE_LOCK_NONE)
1157                 return;
1158         if (f->lock_batch)
1159                 return;
1160
1161         if (td->o.file_lock_mode == FILE_LOCK_READWRITE) {
1162                 const int is_read = f->lock_ddir == DDIR_READ;
1163                 int val = fio_mutex_getval(f->lock);
1164
1165                 if ((is_read && val == 1) || (!is_read && val == -1))
1166                         f->lock_owner = NULL;
1167
1168                 if (is_read)
1169                         fio_mutex_up_read(f->lock);
1170                 else
1171                         fio_mutex_up_write(f->lock);
1172         } else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE) {
1173                 int val = fio_mutex_getval(f->lock);
1174
1175                 if (val == 0)
1176                         f->lock_owner = NULL;
1177
1178                 fio_mutex_up(f->lock);
1179         }
1180 }
1181
1182 void unlock_file_all(struct thread_data *td, struct fio_file *f)
1183 {
1184         if (f->lock_owner != td)
1185                 return;
1186
1187         f->lock_batch = 0;
1188         unlock_file(td, f);
1189 }
1190
1191 static int recurse_dir(struct thread_data *td, const char *dirname)
1192 {
1193         struct dirent *dir;
1194         int ret = 0;
1195         DIR *D;
1196
1197         D = opendir(dirname);
1198         if (!D) {
1199                 char buf[FIO_VERROR_SIZE];
1200
1201                 snprintf(buf, FIO_VERROR_SIZE - 1, "opendir(%s)", dirname);
1202                 td_verror(td, errno, buf);
1203                 return 1;
1204         }
1205
1206         while ((dir = readdir(D)) != NULL) {
1207                 char full_path[PATH_MAX];
1208                 struct stat sb;
1209
1210                 if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, ".."))
1211                         continue;
1212
1213                 sprintf(full_path, "%s%s%s", dirname, FIO_OS_PATH_SEPARATOR, dir->d_name);
1214
1215                 if (lstat(full_path, &sb) == -1) {
1216                         if (errno != ENOENT) {
1217                                 td_verror(td, errno, "stat");
1218                                 return 1;
1219                         }
1220                 }
1221
1222                 if (S_ISREG(sb.st_mode)) {
1223                         add_file(td, full_path);
1224                         td->o.nr_files++;
1225                         continue;
1226                 }
1227                 if (!S_ISDIR(sb.st_mode))
1228                         continue;
1229
1230                 ret = recurse_dir(td, full_path);
1231                 if (ret)
1232                         break;
1233         }
1234
1235         closedir(D);
1236         return ret;
1237 }
1238
1239 int add_dir_files(struct thread_data *td, const char *path)
1240 {
1241         int ret = recurse_dir(td, path);
1242
1243         if (!ret)
1244                 log_info("fio: opendir added %d files\n", td->o.nr_files);
1245
1246         return ret;
1247 }
1248
1249 void dup_files(struct thread_data *td, struct thread_data *org)
1250 {
1251         struct fio_file *f;
1252         unsigned int i;
1253
1254         dprint(FD_FILE, "dup files: %d\n", org->files_index);
1255
1256         if (!org->files)
1257                 return;
1258
1259         td->files = malloc(org->files_index * sizeof(f));
1260
1261         for_each_file(org, f, i) {
1262                 struct fio_file *__f;
1263
1264                 __f = smalloc(sizeof(*__f));
1265                 if (!__f) {
1266                         log_err("fio: smalloc OOM\n");
1267                         assert(0);
1268                 }
1269                 __f->fd = -1;
1270                 fio_file_reset(__f);
1271
1272                 if (f->file_name) {
1273                         __f->file_name = smalloc_strdup(f->file_name);
1274                         if (!__f->file_name) {
1275                                 log_err("fio: smalloc OOM\n");
1276                                 assert(0);
1277                         }
1278
1279                         __f->filetype = f->filetype;
1280                 }
1281
1282                 td->files[i] = __f;
1283         }
1284 }
1285
1286 /*
1287  * Returns the index that matches the filename, or -1 if not there
1288  */
1289 int get_fileno(struct thread_data *td, const char *fname)
1290 {
1291         struct fio_file *f;
1292         unsigned int i;
1293
1294         for_each_file(td, f, i)
1295                 if (!strcmp(f->file_name, fname))
1296                         return i;
1297
1298         return -1;
1299 }
1300
1301 /*
1302  * For log usage, where we add/open/close files automatically
1303  */
1304 void free_release_files(struct thread_data *td)
1305 {
1306         close_files(td);
1307         td->files_index = 0;
1308         td->nr_normal_files = 0;
1309 }