Add 'filesize' option
[fio.git] / filesetup.c
CommitLineData
53cdc686
JA
1#include <unistd.h>
2#include <fcntl.h>
3#include <string.h>
4#include <assert.h>
bbf6b540 5#include <dirent.h>
53cdc686
JA
6#include <sys/stat.h>
7#include <sys/mman.h>
bbf6b540 8#include <sys/types.h>
53cdc686
JA
9
10#include "fio.h"
11#include "os.h"
12
25205e97
JA
13/*
14 * Check if the file exists and it's large enough.
15 */
16static int file_ok(struct thread_data *td, struct fio_file *f)
53cdc686 17{
b2a15192 18 struct stat st;
53cdc686 19
af52b345 20 if (f->filetype != FIO_TYPE_FILE ||
b5af8293 21 (td->io_ops->flags & FIO_DISKLESSIO))
b2a15192
JA
22 return 0;
23
fa01d139 24 if (lstat(f->file_name, &st) == -1)
25205e97 25 return 1;
fa01d139
JA
26
27 /*
28 * if it's a special file, size is always ok for now
29 */
30 if (S_ISCHR(st.st_mode) || S_ISBLK(st.st_mode))
31 return 0;
32 if (st.st_size < (off_t) f->file_size)
25205e97
JA
33 return 1;
34
35 return 0;
36}
37
38static int create_file(struct thread_data *td, struct fio_file *f)
39{
40 unsigned long long left;
41 unsigned int bs;
42 char *b;
43 int r;
b2a15192 44
53cdc686
JA
45 f->fd = open(f->file_name, O_WRONLY | O_CREAT | O_TRUNC, 0644);
46 if (f->fd < 0) {
e1161c32 47 td_verror(td, errno, "open");
53cdc686
JA
48 return 1;
49 }
50
51 if (ftruncate(f->fd, f->file_size) == -1) {
e1161c32 52 td_verror(td, errno, "ftruncate");
53cdc686
JA
53 goto err;
54 }
55
40f8298c 56 if (posix_fallocate(f->fd, 0, f->file_size) < 0) {
e1161c32 57 td_verror(td, errno, "posix_fallocate");
40f8298c
JA
58 goto err;
59 }
60
a00735e6
JA
61 b = malloc(td->max_bs[DDIR_WRITE]);
62 memset(b, 0, td->max_bs[DDIR_WRITE]);
53cdc686
JA
63
64 left = f->file_size;
65 while (left && !td->terminate) {
a00735e6 66 bs = td->max_bs[DDIR_WRITE];
53cdc686
JA
67 if (bs > left)
68 bs = left;
69
70 r = write(f->fd, b, bs);
71
72 if (r == (int) bs) {
73 left -= bs;
74 continue;
75 } else {
76 if (r < 0)
e1161c32 77 td_verror(td, errno, "write");
53cdc686 78 else
e1161c32 79 td_verror(td, EIO, "write");
53cdc686
JA
80
81 break;
82 }
83 }
84
85 if (td->terminate)
86 unlink(f->file_name);
87 else if (td->create_fsync)
88 fsync(f->fd);
89
90 free(b);
91 close(f->fd);
92 f->fd = -1;
93 return 0;
94err:
95 close(f->fd);
96 f->fd = -1;
97 return 1;
98}
99
9c60ce64
JA
100static unsigned long long set_rand_file_size(struct thread_data *td,
101 unsigned long long total_size)
102{
103 unsigned long long upper = total_size;
104 unsigned long long ret;
105 long r;
106
107 if (upper > td->file_size_high)
108 upper = td->file_size_high;
109 else if (upper < td->file_size_low)
110 return 0;
111 else if (!upper)
112 return 0;
113
114 r = os_random_long(&td->file_size_state);
115 ret = td->file_size_low + (unsigned long long) ((double) upper * (r / (RAND_MAX + 1.0)));
116 ret -= (ret % td->rw_min_bs);
117 if (ret > upper)
118 ret = upper;
119 return ret;
120}
121
53cdc686
JA
122static int create_files(struct thread_data *td)
123{
124 struct fio_file *f;
af52b345 125 int err, need_create, can_extend;
9c60ce64 126 unsigned long long total_file_size, local_file_size;
e407bec2 127 unsigned int i, new_files;
53cdc686 128
e407bec2 129 new_files = 0;
96d32d51 130 total_file_size = td->total_file_size;
1549441c 131 for_each_file(td, f, i) {
96d32d51
JA
132 unsigned long long s;
133
134 f->file_offset = td->start_offset;
135
1549441c
JA
136 if (f->filetype != FIO_TYPE_FILE)
137 continue;
138
96d32d51
JA
139 if (f->flags & FIO_FILE_EXISTS) {
140 s = f->file_size;
141 if (s > total_file_size)
142 s = total_file_size;
143
144 total_file_size -= s;
e407bec2
JA
145 } else
146 new_files++;
1549441c 147 }
f697125a 148
53cdc686
JA
149 /*
150 * unless specifically asked for overwrite, let normal io extend it
151 */
0263882a
JA
152 can_extend = !td->overwrite && !(td->io_ops->flags & FIO_NOEXTEND);
153 if (can_extend)
53cdc686 154 return 0;
53cdc686 155
25205e97 156 need_create = 0;
9c60ce64
JA
157 local_file_size = total_file_size;
158 if (!local_file_size)
159 local_file_size = -1;
160
af52b345
JA
161 for_each_file(td, f, i) {
162 int file_there;
163
164 if (f->filetype != FIO_TYPE_FILE)
165 continue;
96d32d51
JA
166 if (f->flags & FIO_FILE_EXISTS)
167 continue;
168
9c60ce64
JA
169 if (!td->file_size_low)
170 f->file_size = total_file_size / new_files;
171 else {
172 /*
173 * If we don't have enough space left for a file
174 * of the minimum size, bail.
175 */
176 if (local_file_size < td->file_size_low) {
177 log_info("fio: limited to %d files\n", i);
178 new_files -= (td->nr_files - i);
179 td->nr_files = i;
180 break;
181 }
182
183 f->file_size = set_rand_file_size(td, local_file_size);
184 local_file_size -= f->file_size;
185 }
eff6861d 186
af52b345 187 file_there = !file_ok(td, f);
eff6861d 188
af52b345
JA
189 if (file_there && td_write(td) && !td->overwrite) {
190 unlink(f->file_name);
191 file_there = 0;
eff6861d 192 }
af52b345
JA
193
194 need_create += !file_there;
eff6861d 195 }
25205e97 196
25205e97
JA
197 if (!need_create)
198 return 0;
199
f1027063
JA
200 if (!td->total_file_size) {
201 log_err("Need size for create\n");
e1161c32 202 td_verror(td, EINVAL, "file_size");
f1027063
JA
203 return 1;
204 }
205
53cdc686 206 temp_stall_ts = 1;
9c60ce64 207 fprintf(f_out, "%s: Laying out IO file(s) (%u files / %LuMiB)\n",
e407bec2 208 td->name, new_files,
e407bec2 209 total_file_size >> 20);
53cdc686
JA
210
211 err = 0;
212 for_each_file(td, f, i) {
9b031dc8
JA
213 /*
214 * Only unlink files that we created.
215 */
f11bd94d 216 f->flags &= ~FIO_FILE_UNLINK;
25205e97 217 if (file_ok(td, f)) {
f11bd94d
JA
218 if (td->unlink)
219 f->flags |= FIO_FILE_UNLINK;
220
25205e97
JA
221 err = create_file(td, f);
222 if (err)
223 break;
224 }
53cdc686
JA
225 }
226
227 temp_stall_ts = 0;
228 return err;
229}
230
231static int file_size(struct thread_data *td, struct fio_file *f)
232{
233 struct stat st;
234
235 if (td->overwrite) {
236 if (fstat(f->fd, &st) == -1) {
e1161c32 237 td_verror(td, errno, "fstat");
53cdc686
JA
238 return 1;
239 }
240
241 f->real_file_size = st.st_size;
242
243 if (!f->file_size || f->file_size > f->real_file_size)
244 f->file_size = f->real_file_size;
745508dd
JA
245 } else
246 f->real_file_size = f->file_size;
53cdc686 247
53cdc686
JA
248 return 0;
249}
250
251static int bdev_size(struct thread_data *td, struct fio_file *f)
252{
253 unsigned long long bytes;
254 int r;
255
256 r = blockdev_size(f->fd, &bytes);
257 if (r) {
e1161c32 258 td_verror(td, r, "blockdev_size");
53cdc686
JA
259 return 1;
260 }
261
262 f->real_file_size = bytes;
263
264 /*
265 * no extend possibilities, so limit size to device size if too large
266 */
267 if (!f->file_size || f->file_size > f->real_file_size)
268 f->file_size = f->real_file_size;
269
270 f->file_size -= f->file_offset;
271 return 0;
272}
273
274static int get_file_size(struct thread_data *td, struct fio_file *f)
275{
276 int ret = 0;
277
96d32d51
JA
278 if (f->filetype == FIO_TYPE_FILE) {
279 if (!(f->flags & FIO_FILE_EXISTS))
280 ret = file_size(td, f);
281 } else if (f->filetype == FIO_TYPE_BD)
53cdc686
JA
282 ret = bdev_size(td, f);
283 else
284 f->real_file_size = -1;
285
286 if (ret)
287 return ret;
288
289 if (f->file_offset > f->real_file_size) {
290 log_err("%s: offset extends end (%Lu > %Lu)\n", td->name, f->file_offset, f->real_file_size);
291 return 1;
292 }
293
53cdc686
JA
294 return 0;
295}
296
e5b401d4
JA
297int file_invalidate_cache(struct thread_data *td, struct fio_file *f)
298{
299 int ret = 0;
300
da86ccb6 301 if (td->odirect)
b5af8293
JA
302 return 0;
303
e5b401d4
JA
304 /*
305 * FIXME: add blockdev flushing too
306 */
b5af8293 307 if (f->mmap)
e5b401d4 308 ret = madvise(f->mmap, f->file_size, MADV_DONTNEED);
af52b345 309 else if (f->filetype == FIO_TYPE_FILE) {
b5af8293 310 ret = fadvise(f->fd, f->file_offset, f->file_size, POSIX_FADV_DONTNEED);
af52b345 311 } else if (f->filetype == FIO_TYPE_BD) {
b5af8293 312 ret = blockdev_invalidate_cache(f->fd);
af52b345 313 } else if (f->filetype == FIO_TYPE_CHAR)
e5b401d4
JA
314 ret = 0;
315
316 if (ret < 0) {
e1161c32 317 td_verror(td, errno, "invalidate_cache");
e5b401d4
JA
318 return 1;
319 }
320
ad2da605 321 return ret;
e5b401d4
JA
322}
323
b5af8293 324void generic_close_file(struct thread_data fio_unused *td, struct fio_file *f)
53cdc686 325{
b5af8293
JA
326 close(f->fd);
327 f->fd = -1;
53cdc686
JA
328}
329
b5af8293 330int generic_open_file(struct thread_data *td, struct fio_file *f)
53cdc686 331{
53cdc686
JA
332 int flags = 0;
333
2fd233b7
JA
334 if (td->odirect)
335 flags |= OS_O_DIRECT;
336 if (td->sync_io)
337 flags |= O_SYNC;
53cdc686 338
2fd233b7
JA
339 if (td_write(td) || td_rw(td)) {
340 flags |= O_RDWR;
53cdc686 341
af52b345 342 if (f->filetype == FIO_TYPE_FILE)
2fd233b7 343 flags |= O_CREAT;
2fd233b7 344
b5af8293 345 f->fd = open(f->file_name, flags, 0600);
2fd233b7 346 } else {
af52b345 347 if (f->filetype == FIO_TYPE_CHAR)
2fd233b7
JA
348 flags |= O_RDWR;
349 else
350 flags |= O_RDONLY;
351
b5af8293 352 f->fd = open(f->file_name, flags);
53cdc686
JA
353 }
354
355 if (f->fd == -1) {
e1161c32
JA
356 int __e = errno;
357
358 td_verror(td, __e, "open");
359 if (__e == EINVAL && td->odirect)
9d80e114 360 log_err("fio: destination does not support O_DIRECT\n");
8ab53873
JA
361 if (__e == EMFILE)
362 log_err("fio: try reducing/setting openfiles (failed at %u of %u)\n", td->nr_open_files, td->nr_files);
53cdc686
JA
363 return 1;
364 }
365
b5af8293
JA
366 if (get_file_size(td, f))
367 goto err;
368
0263882a 369 if (td->invalidate_cache && file_invalidate_cache(td, f))
b5af8293
JA
370 goto err;
371
372 if (!td_random(td)) {
373 if (fadvise(f->fd, f->file_offset, f->file_size, POSIX_FADV_SEQUENTIAL) < 0) {
374 td_verror(td, errno, "fadvise");
375 goto err;
376 }
377 } else {
378 if (fadvise(f->fd, f->file_offset, f->file_size, POSIX_FADV_RANDOM) < 0) {
379 td_verror(td, errno, "fadvise");
380 goto err;
381 }
bdb4e2e9 382 }
53cdc686
JA
383
384 return 0;
b5af8293
JA
385err:
386 close(f->fd);
387 return 1;
388}
389
21972cde
JA
390int open_files(struct thread_data *td)
391{
392 struct fio_file *f;
af52b345
JA
393 unsigned int i;
394 int err = 0;
21972cde
JA
395
396 for_each_file(td, f, i) {
b5af8293 397 err = td_io_open_file(td, f);
21972cde
JA
398 if (err)
399 break;
b5af8293
JA
400
401 if (td->open_files == td->nr_open_files)
402 break;
21972cde
JA
403 }
404
7abf833d
JA
405 if (!err)
406 return 0;
407
bdb4e2e9 408 for_each_file(td, f, i)
b5af8293 409 td_io_close_file(td, f);
7abf833d 410
21972cde
JA
411 return err;
412}
413
53cdc686
JA
414int setup_files(struct thread_data *td)
415{
416 struct fio_file *f;
af52b345
JA
417 unsigned int i;
418 int err;
53cdc686
JA
419
420 /*
421 * if ioengine defines a setup() method, it's responsible for
422 * setting up everything in the td->files[] area.
423 */
424 if (td->io_ops->setup)
425 return td->io_ops->setup(td);
426
427 if (create_files(td))
428 return 1;
429
21972cde 430 err = open_files(td);
f1027063
JA
431 if (err)
432 return err;
433
0a7eb121
JA
434 /*
435 * Recalculate the total file size now that files are set up.
436 */
437 td->total_file_size = 0;
438 for_each_file(td, f, i)
439 td->total_file_size += f->file_size;
440
f1027063 441 td->io_size = td->total_file_size;
53cdc686
JA
442 if (td->io_size == 0) {
443 log_err("%s: no io blocks\n", td->name);
e1161c32 444 td_verror(td, EINVAL, "total_file_size");
53cdc686
JA
445 return 1;
446 }
447
448 if (!td->zone_size)
449 td->zone_size = td->io_size;
450
451 td->total_io_size = td->io_size * td->loops;
452
bdb4e2e9 453 for_each_file(td, f, i)
b5af8293 454 td_io_close_file(td, f);
21972cde
JA
455
456 return err;
53cdc686
JA
457}
458
459void close_files(struct thread_data *td)
460{
0ab8db89 461 struct fio_file *f;
af52b345 462 unsigned int i;
53cdc686 463
0ab8db89 464 for_each_file(td, f, i) {
1315a68a 465 if (!f->file_name && (f->flags & FIO_FILE_UNLINK) &&
af52b345 466 f->filetype == FIO_TYPE_FILE) {
132ad46d 467 unlink(f->file_name);
1315a68a 468 free(f->file_name);
132ad46d
JA
469 f->file_name = NULL;
470 }
bdb4e2e9 471
b5af8293 472 td_io_close_file(td, f);
b3dc7f07
JA
473
474 if (f->file_map)
475 free(f->file_map);
53cdc686 476 }
b4a6a59a 477
132ad46d 478 td->filename = NULL;
b4a6a59a
JA
479 td->files = NULL;
480 td->nr_files = 0;
53cdc686 481}
af52b345 482
e3bab463 483static void get_file_type(struct fio_file *f)
af52b345
JA
484{
485 struct stat sb;
486
487 f->filetype = FIO_TYPE_FILE;
488
e3bab463 489 if (!lstat(f->file_name, &sb)) {
96d32d51
JA
490 f->flags |= FIO_FILE_EXISTS;
491
af52b345
JA
492 if (S_ISBLK(sb.st_mode))
493 f->filetype = FIO_TYPE_BD;
494 else if (S_ISCHR(sb.st_mode))
495 f->filetype = FIO_TYPE_CHAR;
96d32d51
JA
496 else {
497 /*
498 * might as well do this here, and save a stat later on
499 */
500 f->real_file_size = sb.st_size;
501 f->file_size = f->real_file_size;
502 }
af52b345
JA
503 }
504}
505
506void add_file(struct thread_data *td, const char *fname)
507{
7b4e4fe5 508 int cur_files = td->files_index;
af52b345
JA
509 struct fio_file *f;
510
511 td->files = realloc(td->files, (cur_files + 1) * sizeof(*f));
512
513 f = &td->files[cur_files];
514 memset(f, 0, sizeof(*f));
515 f->fd = -1;
cae61953 516 f->file_name = strdup(fname);
af52b345 517
e3bab463 518 get_file_type(f);
af52b345 519
7b4e4fe5 520 td->files_index++;
1549441c
JA
521 if (f->filetype == FIO_TYPE_FILE)
522 td->nr_normal_files++;
af52b345 523}
0ad920e7
JA
524
525void get_file(struct fio_file *f)
526{
527 f->references++;
528}
529
530void put_file(struct thread_data *td, struct fio_file *f)
531{
532 if (!(f->flags & FIO_FILE_OPEN))
533 return;
534
535 assert(f->references);
536 if (--f->references)
537 return;
538
ebb1415f
JA
539 if (should_fsync(td) && td->fsync_on_close)
540 fsync(f->fd);
541
0ad920e7
JA
542 if (td->io_ops->close_file)
543 td->io_ops->close_file(td, f);
544 td->nr_open_files--;
545 f->flags &= ~FIO_FILE_OPEN;
546}
bbf6b540
JA
547
548static int recurse_dir(struct thread_data *td, const char *dirname)
549{
550 struct dirent *dir;
551 int ret = 0;
552 DIR *D;
553
554 D = opendir(dirname);
555 if (!D) {
556 td_verror(td, errno, "opendir");
557 return 1;
558 }
559
560 while ((dir = readdir(D)) != NULL) {
561 char full_path[PATH_MAX];
562 struct stat sb;
563
e85b2b83
JA
564 if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, ".."))
565 continue;
96d32d51 566
bbf6b540
JA
567 sprintf(full_path, "%s/%s", dirname, dir->d_name);
568
569 if (lstat(full_path, &sb) == -1) {
570 if (errno != ENOENT) {
571 td_verror(td, errno, "stat");
572 return 1;
573 }
574 }
575
576 if (S_ISREG(sb.st_mode)) {
577 add_file(td, full_path);
578 td->nr_files++;
579 continue;
580 }
581
bbf6b540
JA
582 if ((ret = recurse_dir(td, full_path)) != 0)
583 break;
584 }
585
586 closedir(D);
587 return ret;
588}
589
590int add_dir_files(struct thread_data *td, const char *path)
591{
592 return recurse_dir(td, path);
593}