70589aa513eaa95d6108664df88d43270d8ba475
[fio.git] / filesetup.c
1 #include <unistd.h>
2 #include <fcntl.h>
3 #include <string.h>
4 #include <assert.h>
5 #include <dirent.h>
6 #include <sys/stat.h>
7 #include <sys/mman.h>
8 #include <sys/types.h>
9
10 #include "fio.h"
11 #include "smalloc.h"
12 #include "filehash.h"
13
14 static int root_warn;
15
16 static inline void clear_error(struct thread_data *td)
17 {
18         td->error = 0;
19         td->verror[0] = '\0';
20 }
21
22 /*
23  * Leaves f->fd open on success, caller must close
24  */
25 static int extend_file(struct thread_data *td, struct fio_file *f)
26 {
27         int r, new_layout = 0, unlink_file = 0, flags;
28         unsigned long long left;
29         unsigned int bs;
30         char *b;
31
32         if (read_only) {
33                 log_err("fio: refusing extend of file due to read-only\n");
34                 return 0;
35         }
36
37         /*
38          * check if we need to lay the file out complete again. fio
39          * does that for operations involving reads, or for writes
40          * where overwrite is set
41          */
42         if (td_read(td) || (td_write(td) && td->o.overwrite) ||
43             (td_write(td) && td->io_ops->flags & FIO_NOEXTEND))
44                 new_layout = 1;
45         if (td_write(td) && !td->o.overwrite)
46                 unlink_file = 1;
47
48         if (unlink_file || new_layout) {
49                 dprint(FD_FILE, "layout unlink %s\n", f->file_name);
50                 if ((unlink(f->file_name) < 0) && (errno != ENOENT)) {
51                         td_verror(td, errno, "unlink");
52                         return 1;
53                 }
54         }
55
56         flags = O_WRONLY | O_CREAT;
57         if (new_layout)
58                 flags |= O_TRUNC;
59
60         dprint(FD_FILE, "open file %s, flags %x\n", f->file_name, flags);
61         f->fd = open(f->file_name, flags, 0644);
62         if (f->fd < 0) {
63                 td_verror(td, errno, "open");
64                 return 1;
65         }
66
67         if (!new_layout)
68                 goto done;
69
70         /*
71          * The size will be -1ULL when fill_device is used, so don't truncate
72          * or fallocate this file, just write it
73          */
74         if (!td->o.fill_device) {
75                 dprint(FD_FILE, "truncate file %s, size %llu\n", f->file_name,
76                                                         f->real_file_size);
77                 if (ftruncate(f->fd, f->real_file_size) == -1) {
78                         td_verror(td, errno, "ftruncate");
79                         goto err;
80                 }
81
82 #ifdef FIO_HAVE_FALLOCATE
83                 dprint(FD_FILE, "fallocate file %s, size %llu\n", f->file_name,
84                                                         f->real_file_size);
85                 r = posix_fallocate(f->fd, 0, f->real_file_size);
86                 if (r < 0) {
87                         log_err("fio: posix_fallocate fails: %s\n",
88                                         strerror(-r));
89                 }
90 #endif
91         }
92
93         b = malloc(td->o.max_bs[DDIR_WRITE]);
94         memset(b, 0, td->o.max_bs[DDIR_WRITE]);
95
96         left = f->real_file_size;
97         while (left && !td->terminate) {
98                 bs = td->o.max_bs[DDIR_WRITE];
99                 if (bs > left)
100                         bs = left;
101
102                 r = write(f->fd, b, bs);
103
104                 if (r > 0) {
105                         left -= r;
106                         continue;
107                 } else {
108                         if (r < 0) {
109                                 int __e = errno;
110
111                                 if (__e == ENOSPC) {
112                                         if (td->o.fill_device)
113                                                 break;
114                                         log_info("fio: ENOSPC on laying out "
115                                                  "file, stopping\n");
116                                         break;
117                                 }
118                                 td_verror(td, errno, "write");
119                         } else
120                                 td_verror(td, EIO, "write");
121
122                         break;
123                 }
124         }
125
126         if (td->terminate) {
127                 dprint(FD_FILE, "terminate unlink %s\n", f->file_name);
128                 unlink(f->file_name);
129         } else if (td->o.create_fsync) {
130                 if (fsync(f->fd) < 0) {
131                         td_verror(td, errno, "fsync");
132                         goto err;
133                 }
134         }
135         if (td->o.fill_device && !td_write(td)) {
136                 fio_file_clear_size_known(f);
137                 if (td_io_get_file_size(td, f))
138                         goto err;
139                 if (f->io_size > f->real_file_size)
140                         f->io_size = f->real_file_size;
141         }
142
143         free(b);
144 done:
145         return 0;
146 err:
147         close(f->fd);
148         f->fd = -1;
149         return 1;
150 }
151
152 static int pre_read_file(struct thread_data *td, struct fio_file *f)
153 {
154         int r, did_open = 0, old_runstate;
155         unsigned long long left;
156         unsigned int bs;
157         char *b;
158
159         if (td->io_ops->flags & FIO_PIPEIO)
160                 return 0;
161
162         if (!fio_file_open(f)) {
163                 if (td->io_ops->open_file(td, f)) {
164                         log_err("fio: cannot pre-read, failed to open file\n");
165                         return 1;
166                 }
167                 did_open = 1;
168         }
169
170         old_runstate = td->runstate;
171         td_set_runstate(td, TD_PRE_READING);
172
173         bs = td->o.max_bs[DDIR_READ];
174         b = malloc(bs);
175         memset(b, 0, bs);
176
177         lseek(f->fd, f->file_offset, SEEK_SET);
178         left = f->io_size;
179
180         while (left && !td->terminate) {
181                 if (bs > left)
182                         bs = left;
183
184                 r = read(f->fd, b, bs);
185
186                 if (r == (int) bs) {
187                         left -= bs;
188                         continue;
189                 } else {
190                         td_verror(td, EIO, "pre_read");
191                         break;
192                 }
193         }
194
195         td_set_runstate(td, old_runstate);
196
197         if (did_open)
198                 td->io_ops->close_file(td, f);
199         free(b);
200         return 0;
201 }
202
203 static unsigned long long get_rand_file_size(struct thread_data *td)
204 {
205         unsigned long long ret, sized;
206         long r;
207
208         r = os_random_long(&td->file_size_state);
209         sized = td->o.file_size_high - td->o.file_size_low;
210         ret = (unsigned long long) ((double) sized * (r / (OS_RAND_MAX + 1.0)));
211         ret += td->o.file_size_low;
212         ret -= (ret % td->o.rw_min_bs);
213         return ret;
214 }
215
216 static int file_size(struct thread_data *td, struct fio_file *f)
217 {
218         struct stat st;
219
220         if (stat(f->file_name, &st) == -1) {
221                 td_verror(td, errno, "fstat");
222                 return 1;
223         }
224
225         f->real_file_size = st.st_size;
226         return 0;
227 }
228
229 static int bdev_size(struct thread_data *td, struct fio_file *f)
230 {
231         unsigned long long bytes;
232         int r;
233
234         if (td->io_ops->open_file(td, f)) {
235                 log_err("fio: failed opening blockdev %s for size check\n",
236                         f->file_name);
237                 return 1;
238         }
239
240         r = blockdev_size(f->fd, &bytes);
241         if (r) {
242                 td_verror(td, r, "blockdev_size");
243                 goto err;
244         }
245
246         if (!bytes) {
247                 log_err("%s: zero sized block device?\n", f->file_name);
248                 goto err;
249         }
250
251         f->real_file_size = bytes;
252         td->io_ops->close_file(td, f);
253         return 0;
254 err:
255         td->io_ops->close_file(td, f);
256         return 1;
257 }
258
259 static int get_file_size(struct thread_data *td, struct fio_file *f)
260 {
261         int ret = 0;
262
263         if (fio_file_size_known(f))
264                 return 0;
265
266         if (f->filetype == FIO_TYPE_FILE)
267                 ret = file_size(td, f);
268         else if (f->filetype == FIO_TYPE_BD)
269                 ret = bdev_size(td, f);
270         else
271                 f->real_file_size = -1;
272
273         if (ret)
274                 return ret;
275
276         if (f->file_offset > f->real_file_size) {
277                 log_err("%s: offset extends end (%Lu > %Lu)\n", td->o.name,
278                                         f->file_offset, f->real_file_size);
279                 return 1;
280         }
281
282         fio_file_set_size_known(f);
283         return 0;
284 }
285
286 static int __file_invalidate_cache(struct thread_data *td, struct fio_file *f,
287                                    unsigned long long off,
288                                    unsigned long long len)
289 {
290         int ret = 0;
291
292         if (len == -1ULL)
293                 len = f->io_size;
294         if (off == -1ULL)
295                 off = f->file_offset;
296
297         if (len == -1ULL || off == -1ULL)
298                 return 0;
299
300         dprint(FD_IO, "invalidate cache %s: %llu/%llu\n", f->file_name, off,
301                                                                 len);
302
303         /*
304          * FIXME: add blockdev flushing too
305          */
306         if (f->mmap_ptr) {
307                 ret = madvise(f->mmap_ptr, f->mmap_sz, MADV_DONTNEED);
308 #ifdef FIO_MADV_FREE
309                 (void) madvise(f->mmap_ptr, f->mmap_sz, FIO_MADV_FREE);
310 #endif
311         } else if (f->filetype == FIO_TYPE_FILE) {
312                 ret = fadvise(f->fd, off, len, POSIX_FADV_DONTNEED);
313         } else if (f->filetype == FIO_TYPE_BD) {
314                 ret = blockdev_invalidate_cache(f->fd);
315                 if (ret < 0 && errno == EACCES && geteuid()) {
316                         if (!root_warn) {
317                                 log_err("fio: only root may flush block "
318                                         "devices. Cache flush bypassed!\n");
319                                 root_warn = 1;
320                         }
321                         ret = 0;
322                 }
323         } else if (f->filetype == FIO_TYPE_CHAR || f->filetype == FIO_TYPE_PIPE)
324                 ret = 0;
325
326         if (ret < 0) {
327                 td_verror(td, errno, "invalidate_cache");
328                 return 1;
329         } else if (ret > 0) {
330                 td_verror(td, ret, "invalidate_cache");
331                 return 1;
332         }
333
334         return ret;
335
336 }
337
338 int file_invalidate_cache(struct thread_data *td, struct fio_file *f)
339 {
340         if (!fio_file_open(f))
341                 return 0;
342
343         return __file_invalidate_cache(td, f, -1ULL, -1ULL);
344 }
345
346 int generic_close_file(struct thread_data fio_unused *td, struct fio_file *f)
347 {
348         int ret = 0;
349
350         dprint(FD_FILE, "fd close %s\n", f->file_name);
351
352         remove_file_hash(f);
353
354         if (close(f->fd) < 0)
355                 ret = errno;
356
357         f->fd = -1;
358         return ret;
359 }
360
361 static int file_lookup_open(struct fio_file *f, int flags)
362 {
363         struct fio_file *__f;
364         int from_hash;
365
366         __f = lookup_file_hash(f->file_name);
367         if (__f) {
368                 dprint(FD_FILE, "found file in hash %s\n", f->file_name);
369                 /*
370                  * racy, need the __f->lock locked
371                  */
372                 f->lock = __f->lock;
373                 f->lock_owner = __f->lock_owner;
374                 f->lock_batch = __f->lock_batch;
375                 f->lock_ddir = __f->lock_ddir;
376                 from_hash = 1;
377         } else {
378                 dprint(FD_FILE, "file not found in hash %s\n", f->file_name);
379                 from_hash = 0;
380         }
381
382         f->fd = open(f->file_name, flags, 0600);
383         return from_hash;
384 }
385
386 int generic_open_file(struct thread_data *td, struct fio_file *f)
387 {
388         int is_std = 0;
389         int flags = 0;
390         int from_hash = 0;
391
392         dprint(FD_FILE, "fd open %s\n", f->file_name);
393
394         if (!strcmp(f->file_name, "-")) {
395                 if (td_rw(td)) {
396                         log_err("fio: can't read/write to stdin/out\n");
397                         return 1;
398                 }
399                 is_std = 1;
400
401                 /*
402                  * move output logging to stderr, if we are writing to stdout
403                  */
404                 if (td_write(td))
405                         f_out = stderr;
406         }
407
408         if (td->o.odirect)
409                 flags |= OS_O_DIRECT;
410         if (td->o.sync_io)
411                 flags |= O_SYNC;
412         if (f->filetype != FIO_TYPE_FILE)
413                 flags |= FIO_O_NOATIME;
414         if (td->o.create_on_open)
415                 flags |= O_CREAT;
416
417 open_again:
418         if (td_write(td)) {
419                 if (!read_only)
420                         flags |= O_RDWR;
421
422                 if (f->filetype == FIO_TYPE_FILE)
423                         flags |= O_CREAT;
424
425                 if (is_std)
426                         f->fd = dup(STDOUT_FILENO);
427                 else
428                         from_hash = file_lookup_open(f, flags);
429         } else {
430                 if (f->filetype == FIO_TYPE_CHAR && !read_only)
431                         flags |= O_RDWR;
432                 else
433                         flags |= O_RDONLY;
434
435                 if (is_std)
436                         f->fd = dup(STDIN_FILENO);
437                 else
438                         from_hash = file_lookup_open(f, flags);
439         }
440
441         if (f->fd == -1) {
442                 char buf[FIO_VERROR_SIZE];
443                 int __e = errno;
444
445                 if (__e == EPERM && (flags & FIO_O_NOATIME)) {
446                         flags &= ~FIO_O_NOATIME;
447                         goto open_again;
448                 }
449
450                 snprintf(buf, sizeof(buf) - 1, "open(%s)", f->file_name);
451
452                 td_verror(td, __e, buf);
453         }
454
455         if (!from_hash && f->fd != -1) {
456                 if (add_file_hash(f)) {
457                         int ret;
458
459                         /*
460                          * OK to ignore, we haven't done anything with it
461                          */
462                         ret = generic_close_file(td, f);
463                         goto open_again;
464                 }
465         }
466
467         return 0;
468 }
469
470 int generic_get_file_size(struct thread_data *td, struct fio_file *f)
471 {
472         return get_file_size(td, f);
473 }
474
475 /*
476  * open/close all files, so that ->real_file_size gets set
477  */
478 static int get_file_sizes(struct thread_data *td)
479 {
480         struct fio_file *f;
481         unsigned int i;
482         int err = 0;
483
484         for_each_file(td, f, i) {
485                 dprint(FD_FILE, "get file size for %p/%d/%p\n", f, i,
486                                                                 f->file_name);
487
488                 if (td_io_get_file_size(td, f)) {
489                         if (td->error != ENOENT) {
490                                 log_err("%s\n", td->verror);
491                                 err = 1;
492                         }
493                         clear_error(td);
494                 }
495
496                 if (f->real_file_size == -1ULL && td->o.size)
497                         f->real_file_size = td->o.size / td->o.nr_files;
498         }
499
500         return err;
501 }
502
503 /*
504  * Open the files and setup files sizes, creating files if necessary.
505  */
506 int setup_files(struct thread_data *td)
507 {
508         unsigned long long total_size, extend_size;
509         struct fio_file *f;
510         unsigned int i;
511         int err = 0, need_extend;
512
513         dprint(FD_FILE, "setup files\n");
514
515         if (td->o.read_iolog_file)
516                 return 0;
517
518         /*
519          * if ioengine defines a setup() method, it's responsible for
520          * opening the files and setting f->real_file_size to indicate
521          * the valid range for that file.
522          */
523         if (td->io_ops->setup)
524                 err = td->io_ops->setup(td);
525         else
526                 err = get_file_sizes(td);
527
528         if (err)
529                 return err;
530
531         /*
532          * check sizes. if the files/devices do not exist and the size
533          * isn't passed to fio, abort.
534          */
535         total_size = 0;
536         for_each_file(td, f, i) {
537                 if (f->real_file_size == -1ULL)
538                         total_size = -1ULL;
539                 else
540                         total_size += f->real_file_size;
541         }
542
543         /*
544          * device/file sizes are zero and no size given, punt
545          */
546         if ((!total_size || total_size == -1ULL) && !td->o.size &&
547             !(td->io_ops->flags & FIO_NOIO) && !td->o.fill_device) {
548                 log_err("%s: you need to specify size=\n", td->o.name);
549                 td_verror(td, EINVAL, "total_file_size");
550                 return 1;
551         }
552
553         /*
554          * now file sizes are known, so we can set ->io_size. if size= is
555          * not given, ->io_size is just equal to ->real_file_size. if size
556          * is given, ->io_size is size / nr_files.
557          */
558         extend_size = total_size = 0;
559         need_extend = 0;
560         for_each_file(td, f, i) {
561                 f->file_offset = td->o.start_offset;
562
563                 if (!td->o.file_size_low) {
564                         /*
565                          * no file size range given, file size is equal to
566                          * total size divided by number of files. if that is
567                          * zero, set it to the real file size.
568                          */
569                         f->io_size = td->o.size / td->o.nr_files;
570                         if (!f->io_size)
571                                 f->io_size = f->real_file_size - f->file_offset;
572                 } else if (f->real_file_size < td->o.file_size_low ||
573                            f->real_file_size > td->o.file_size_high) {
574                         if (f->file_offset > td->o.file_size_low)
575                                 goto err_offset;
576                         /*
577                          * file size given. if it's fixed, use that. if it's a
578                          * range, generate a random size in-between.
579                          */
580                         if (td->o.file_size_low == td->o.file_size_high) {
581                                 f->io_size = td->o.file_size_low
582                                                 - f->file_offset;
583                         } else {
584                                 f->io_size = get_rand_file_size(td)
585                                                 - f->file_offset;
586                         }
587                 } else
588                         f->io_size = f->real_file_size - f->file_offset;
589
590                 if (f->io_size == -1ULL)
591                         total_size = -1ULL;
592                 else
593                         total_size += f->io_size;
594
595                 if (f->filetype == FIO_TYPE_FILE &&
596                     (f->io_size + f->file_offset) > f->real_file_size &&
597                     !(td->io_ops->flags & FIO_DISKLESSIO)) {
598                         if (!td->o.create_on_open) {
599                                 need_extend++;
600                                 extend_size += (f->io_size + f->file_offset);
601                         } else
602                                 f->real_file_size = f->io_size + f->file_offset;
603                         fio_file_set_extend(f);
604                 }
605         }
606
607         if (!td->o.size || td->o.size > total_size)
608                 td->o.size = total_size;
609
610         /*
611          * See if we need to extend some files
612          */
613         if (need_extend) {
614                 temp_stall_ts = 1;
615                 if (!terse_output)
616                         log_info("%s: Laying out IO file(s) (%u file(s) /"
617                                  " %LuMB)\n", td->o.name, need_extend,
618                                         extend_size >> 20);
619
620                 for_each_file(td, f, i) {
621                         unsigned long long old_len = -1ULL, extend_len = -1ULL;
622
623                         if (!fio_file_extend(f))
624                                 continue;
625
626                         assert(f->filetype == FIO_TYPE_FILE);
627                         fio_file_clear_extend(f);
628                         if (!td->o.fill_device) {
629                                 old_len = f->real_file_size;
630                                 extend_len = f->io_size + f->file_offset -
631                                                 old_len;
632                         }
633                         f->real_file_size = (f->io_size + f->file_offset);
634                         err = extend_file(td, f);
635                         if (err)
636                                 break;
637
638                         err = __file_invalidate_cache(td, f, old_len,
639                                                                 extend_len);
640                         close(f->fd);
641                         f->fd = -1;
642                         if (err)
643                                 break;
644                 }
645                 temp_stall_ts = 0;
646         }
647
648         if (err)
649                 return err;
650
651         if (!td->o.zone_size)
652                 td->o.zone_size = td->o.size;
653
654         /*
655          * iolog already set the total io size, if we read back
656          * stored entries.
657          */
658         if (!td->o.read_iolog_file)
659                 td->total_io_size = td->o.size * td->o.loops;
660         return 0;
661 err_offset:
662         log_err("%s: you need to specify valid offset=\n", td->o.name);
663         return 1;
664 }
665
666 int pre_read_files(struct thread_data *td)
667 {
668         struct fio_file *f;
669         unsigned int i;
670
671         dprint(FD_FILE, "pre_read files\n");
672
673         for_each_file(td, f, i) {
674                 pre_read_file(td, f);
675         }
676
677         return 1;
678 }
679
680 int init_random_map(struct thread_data *td)
681 {
682         unsigned long long blocks, num_maps;
683         struct fio_file *f;
684         unsigned int i;
685
686         if (td->o.norandommap || !td_random(td))
687                 return 0;
688
689         for_each_file(td, f, i) {
690                 blocks = (f->real_file_size + td->o.rw_min_bs - 1) /
691                                 (unsigned long long) td->o.rw_min_bs;
692                 num_maps = (blocks + BLOCKS_PER_MAP - 1) /
693                                 (unsigned long long) BLOCKS_PER_MAP;
694                 f->file_map = smalloc(num_maps * sizeof(int));
695                 if (f->file_map) {
696                         f->num_maps = num_maps;
697                         continue;
698                 }
699                 if (!td->o.softrandommap) {
700                         log_err("fio: failed allocating random map. If running"
701                                 " a large number of jobs, try the 'norandommap'"
702                                 " option or set 'softrandommap'. Or give"
703                                 " a larger --alloc-size to fio.\n");
704                         return 1;
705                 }
706
707                 log_info("fio: file %s failed allocating random map. Running "
708                          "job without.\n", f->file_name);
709                 f->num_maps = 0;
710         }
711
712         return 0;
713 }
714
715 void close_files(struct thread_data *td)
716 {
717         struct fio_file *f;
718         unsigned int i;
719
720         for_each_file(td, f, i) {
721                 if (fio_file_open(f))
722                         td_io_close_file(td, f);
723         }
724 }
725
726 void close_and_free_files(struct thread_data *td)
727 {
728         struct fio_file *f;
729         unsigned int i;
730
731         dprint(FD_FILE, "close files\n");
732
733         for_each_file(td, f, i) {
734                 if (td->o.unlink && f->filetype == FIO_TYPE_FILE) {
735                         dprint(FD_FILE, "free unlink %s\n", f->file_name);
736                         unlink(f->file_name);
737                 }
738
739                 if (fio_file_open(f))
740                         td_io_close_file(td, f);
741
742                 remove_file_hash(f);
743
744                 sfree(f->file_name);
745                 f->file_name = NULL;
746
747                 if (f->file_map) {
748                         sfree(f->file_map);
749                         f->file_map = NULL;
750                 }
751                 sfree(f);
752         }
753
754         td->o.filename = NULL;
755         free(td->files);
756         td->files_index = 0;
757         td->files = NULL;
758         td->o.nr_files = 0;
759 }
760
761 static void get_file_type(struct fio_file *f)
762 {
763         struct stat sb;
764
765         if (!strcmp(f->file_name, "-"))
766                 f->filetype = FIO_TYPE_PIPE;
767         else
768                 f->filetype = FIO_TYPE_FILE;
769
770         if (!lstat(f->file_name, &sb)) {
771                 if (S_ISBLK(sb.st_mode))
772                         f->filetype = FIO_TYPE_BD;
773                 else if (S_ISCHR(sb.st_mode))
774                         f->filetype = FIO_TYPE_CHAR;
775                 else if (S_ISFIFO(sb.st_mode))
776                         f->filetype = FIO_TYPE_PIPE;
777         }
778 }
779
780 int add_file(struct thread_data *td, const char *fname)
781 {
782         int cur_files = td->files_index;
783         char file_name[PATH_MAX];
784         struct fio_file *f;
785         int len = 0;
786
787         dprint(FD_FILE, "add file %s\n", fname);
788
789         f = smalloc(sizeof(*f));
790         if (!f) {
791                 log_err("fio: smalloc OOM\n");
792                 assert(0);
793         }
794
795         f->fd = -1;
796
797         if (td->files_size <= td->files_index) {
798                 int new_size = td->o.nr_files + 1;
799
800                 dprint(FD_FILE, "resize file array to %d files\n", new_size);
801
802                 td->files = realloc(td->files, new_size * sizeof(f));
803                 td->files_size = new_size;
804         }
805         td->files[cur_files] = f;
806
807         /*
808          * init function, io engine may not be loaded yet
809          */
810         if (td->io_ops && (td->io_ops->flags & FIO_DISKLESSIO))
811                 f->real_file_size = -1ULL;
812
813         if (td->o.directory)
814                 len = sprintf(file_name, "%s/", td->o.directory);
815
816         sprintf(file_name + len, "%s", fname);
817         f->file_name = smalloc_strdup(file_name);
818         if (!f->file_name) {
819                 log_err("fio: smalloc OOM\n");
820                 assert(0);
821         }
822
823         get_file_type(f);
824
825         switch (td->o.file_lock_mode) {
826         case FILE_LOCK_NONE:
827                 break;
828         case FILE_LOCK_READWRITE:
829                 f->lock = fio_mutex_rw_init();
830                 break;
831         case FILE_LOCK_EXCLUSIVE:
832                 f->lock = fio_mutex_init(1);
833                 break;
834         default:
835                 log_err("fio: unknown lock mode: %d\n", td->o.file_lock_mode);
836                 assert(0);
837         }
838
839         td->files_index++;
840         if (f->filetype == FIO_TYPE_FILE)
841                 td->nr_normal_files++;
842
843         dprint(FD_FILE, "file %p \"%s\" added at %d\n", f, f->file_name,
844                                                         cur_files);
845
846         return cur_files;
847 }
848
849 void get_file(struct fio_file *f)
850 {
851         dprint(FD_FILE, "get file %s, ref=%d\n", f->file_name, f->references);
852         assert(fio_file_open(f));
853         f->references++;
854 }
855
856 int put_file(struct thread_data *td, struct fio_file *f)
857 {
858         int f_ret = 0, ret = 0;
859
860         dprint(FD_FILE, "put file %s, ref=%d\n", f->file_name, f->references);
861
862         if (!fio_file_open(f)) {
863                 assert(f->fd == -1);
864                 return 0;
865         }
866
867         assert(f->references);
868         if (--f->references)
869                 return 0;
870
871         if (should_fsync(td) && td->o.fsync_on_close)
872                 f_ret = fsync(f->fd);
873
874         if (td->io_ops->close_file)
875                 ret = td->io_ops->close_file(td, f);
876
877         if (!ret)
878                 ret = f_ret;
879
880         td->nr_open_files--;
881         fio_file_clear_open(f);
882         assert(f->fd == -1);
883         return ret;
884 }
885
886 void lock_file(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir)
887 {
888         if (!f->lock || td->o.file_lock_mode == FILE_LOCK_NONE)
889                 return;
890
891         if (f->lock_owner == td && f->lock_batch--)
892                 return;
893
894         if (td->o.file_lock_mode == FILE_LOCK_READWRITE) {
895                 if (ddir == DDIR_READ)
896                         fio_mutex_down_read(f->lock);
897                 else
898                         fio_mutex_down_write(f->lock);
899         } else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE)
900                 fio_mutex_down(f->lock);
901
902         f->lock_owner = td;
903         f->lock_batch = td->o.lockfile_batch;
904         f->lock_ddir = ddir;
905 }
906
907 void unlock_file(struct thread_data *td, struct fio_file *f)
908 {
909         if (!f->lock || td->o.file_lock_mode == FILE_LOCK_NONE)
910                 return;
911         if (f->lock_batch)
912                 return;
913
914         if (td->o.file_lock_mode == FILE_LOCK_READWRITE) {
915                 const int is_read = f->lock_ddir == DDIR_READ;
916                 int val = fio_mutex_getval(f->lock);
917
918                 if ((is_read && val == 1) || (!is_read && val == -1))
919                         f->lock_owner = NULL;
920
921                 if (is_read)
922                         fio_mutex_up_read(f->lock);
923                 else
924                         fio_mutex_up_write(f->lock);
925         } else if (td->o.file_lock_mode == FILE_LOCK_EXCLUSIVE) {
926                 int val = fio_mutex_getval(f->lock);
927
928                 if (val == 0)
929                         f->lock_owner = NULL;
930
931                 fio_mutex_up(f->lock);
932         }
933 }
934
935 void unlock_file_all(struct thread_data *td, struct fio_file *f)
936 {
937         if (f->lock_owner != td)
938                 return;
939
940         f->lock_batch = 0;
941         unlock_file(td, f);
942 }
943
944 static int recurse_dir(struct thread_data *td, const char *dirname)
945 {
946         struct dirent *dir;
947         int ret = 0;
948         DIR *D;
949
950         D = opendir(dirname);
951         if (!D) {
952                 char buf[FIO_VERROR_SIZE];
953
954                 snprintf(buf, FIO_VERROR_SIZE - 1, "opendir(%s)", dirname);
955                 td_verror(td, errno, buf);
956                 return 1;
957         }
958
959         while ((dir = readdir(D)) != NULL) {
960                 char full_path[PATH_MAX];
961                 struct stat sb;
962
963                 if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, ".."))
964                         continue;
965
966                 sprintf(full_path, "%s/%s", dirname, dir->d_name);
967
968                 if (lstat(full_path, &sb) == -1) {
969                         if (errno != ENOENT) {
970                                 td_verror(td, errno, "stat");
971                                 return 1;
972                         }
973                 }
974
975                 if (S_ISREG(sb.st_mode)) {
976                         add_file(td, full_path);
977                         td->o.nr_files++;
978                         continue;
979                 }
980                 if (!S_ISDIR(sb.st_mode))
981                         continue;
982
983                 ret = recurse_dir(td, full_path);
984                 if (ret)
985                         break;
986         }
987
988         closedir(D);
989         return ret;
990 }
991
992 int add_dir_files(struct thread_data *td, const char *path)
993 {
994         int ret = recurse_dir(td, path);
995
996         if (!ret)
997                 log_info("fio: opendir added %d files\n", td->o.nr_files);
998
999         return ret;
1000 }
1001
1002 void dup_files(struct thread_data *td, struct thread_data *org)
1003 {
1004         struct fio_file *f;
1005         unsigned int i;
1006
1007         dprint(FD_FILE, "dup files: %d\n", org->files_index);
1008
1009         if (!org->files)
1010                 return;
1011
1012         td->files = malloc(org->files_index * sizeof(f));
1013
1014         for_each_file(org, f, i) {
1015                 struct fio_file *__f;
1016
1017                 __f = smalloc(sizeof(*__f));
1018                 if (!__f) {
1019                         log_err("fio: smalloc OOM\n");
1020                         assert(0);
1021                 }
1022                 __f->fd = -1;
1023
1024                 if (f->file_name) {
1025                         __f->file_name = smalloc_strdup(f->file_name);
1026                         if (!__f->file_name) {
1027                                 log_err("fio: smalloc OOM\n");
1028                                 assert(0);
1029                         }
1030
1031                         __f->filetype = f->filetype;
1032                 }
1033
1034                 td->files[i] = __f;
1035         }
1036 }
1037
1038 /*
1039  * Returns the index that matches the filename, or -1 if not there
1040  */
1041 int get_fileno(struct thread_data *td, const char *fname)
1042 {
1043         struct fio_file *f;
1044         unsigned int i;
1045
1046         for_each_file(td, f, i)
1047                 if (!strcmp(f->file_name, fname))
1048                         return i;
1049
1050         return -1;
1051 }
1052
1053 /*
1054  * For log usage, where we add/open/close files automatically
1055  */
1056 void free_release_files(struct thread_data *td)
1057 {
1058         close_files(td);
1059         td->files_index = 0;
1060         td->nr_normal_files = 0;
1061 }