Use log_info() throughout
[fio.git] / filesetup.c
CommitLineData
53cdc686
JA
1#include <unistd.h>
2#include <fcntl.h>
3#include <string.h>
4#include <assert.h>
bbf6b540 5#include <dirent.h>
53cdc686
JA
6#include <sys/stat.h>
7#include <sys/mman.h>
bbf6b540 8#include <sys/types.h>
53cdc686
JA
9
10#include "fio.h"
11#include "os.h"
12
25205e97
JA
13/*
14 * Check if the file exists and it's large enough.
15 */
16static int file_ok(struct thread_data *td, struct fio_file *f)
53cdc686 17{
b2a15192 18 struct stat st;
53cdc686 19
af52b345 20 if (f->filetype != FIO_TYPE_FILE ||
b5af8293 21 (td->io_ops->flags & FIO_DISKLESSIO))
b2a15192
JA
22 return 0;
23
fa01d139 24 if (lstat(f->file_name, &st) == -1)
25205e97 25 return 1;
fa01d139
JA
26
27 /*
28 * if it's a special file, size is always ok for now
29 */
30 if (S_ISCHR(st.st_mode) || S_ISBLK(st.st_mode))
31 return 0;
32 if (st.st_size < (off_t) f->file_size)
25205e97
JA
33 return 1;
34
35 return 0;
36}
37
38static int create_file(struct thread_data *td, struct fio_file *f)
39{
40 unsigned long long left;
41 unsigned int bs;
42 char *b;
43 int r;
b2a15192 44
53cdc686
JA
45 f->fd = open(f->file_name, O_WRONLY | O_CREAT | O_TRUNC, 0644);
46 if (f->fd < 0) {
e1161c32 47 td_verror(td, errno, "open");
53cdc686
JA
48 return 1;
49 }
50
51 if (ftruncate(f->fd, f->file_size) == -1) {
e1161c32 52 td_verror(td, errno, "ftruncate");
53cdc686
JA
53 goto err;
54 }
55
40f8298c 56 if (posix_fallocate(f->fd, 0, f->file_size) < 0) {
e1161c32 57 td_verror(td, errno, "posix_fallocate");
40f8298c
JA
58 goto err;
59 }
60
a00735e6
JA
61 b = malloc(td->max_bs[DDIR_WRITE]);
62 memset(b, 0, td->max_bs[DDIR_WRITE]);
53cdc686
JA
63
64 left = f->file_size;
65 while (left && !td->terminate) {
a00735e6 66 bs = td->max_bs[DDIR_WRITE];
53cdc686
JA
67 if (bs > left)
68 bs = left;
69
70 r = write(f->fd, b, bs);
71
72 if (r == (int) bs) {
73 left -= bs;
74 continue;
75 } else {
76 if (r < 0)
e1161c32 77 td_verror(td, errno, "write");
53cdc686 78 else
e1161c32 79 td_verror(td, EIO, "write");
53cdc686
JA
80
81 break;
82 }
83 }
84
85 if (td->terminate)
86 unlink(f->file_name);
87 else if (td->create_fsync)
88 fsync(f->fd);
89
90 free(b);
91 close(f->fd);
92 f->fd = -1;
93 return 0;
94err:
95 close(f->fd);
96 f->fd = -1;
97 return 1;
98}
99
9c60ce64
JA
100static unsigned long long set_rand_file_size(struct thread_data *td,
101 unsigned long long total_size)
102{
103 unsigned long long upper = total_size;
104 unsigned long long ret;
105 long r;
106
107 if (upper > td->file_size_high)
108 upper = td->file_size_high;
109 else if (upper < td->file_size_low)
110 return 0;
111 else if (!upper)
112 return 0;
113
114 r = os_random_long(&td->file_size_state);
115 ret = td->file_size_low + (unsigned long long) ((double) upper * (r / (RAND_MAX + 1.0)));
116 ret -= (ret % td->rw_min_bs);
117 if (ret > upper)
118 ret = upper;
119 return ret;
120}
121
53cdc686
JA
122static int create_files(struct thread_data *td)
123{
124 struct fio_file *f;
af52b345 125 int err, need_create, can_extend;
9c60ce64 126 unsigned long long total_file_size, local_file_size;
e407bec2 127 unsigned int i, new_files;
53cdc686 128
e407bec2 129 new_files = 0;
96d32d51 130 total_file_size = td->total_file_size;
1549441c 131 for_each_file(td, f, i) {
96d32d51
JA
132 unsigned long long s;
133
134 f->file_offset = td->start_offset;
135
1549441c
JA
136 if (f->filetype != FIO_TYPE_FILE)
137 continue;
138
96d32d51
JA
139 if (f->flags & FIO_FILE_EXISTS) {
140 s = f->file_size;
141 if (s > total_file_size)
142 s = total_file_size;
143
144 total_file_size -= s;
e407bec2
JA
145 } else
146 new_files++;
1549441c 147 }
f697125a 148
53cdc686
JA
149 /*
150 * unless specifically asked for overwrite, let normal io extend it
151 */
0263882a
JA
152 can_extend = !td->overwrite && !(td->io_ops->flags & FIO_NOEXTEND);
153 if (can_extend)
53cdc686 154 return 0;
53cdc686 155
25205e97 156 need_create = 0;
9c60ce64
JA
157 local_file_size = total_file_size;
158 if (!local_file_size)
159 local_file_size = -1;
160
af52b345
JA
161 for_each_file(td, f, i) {
162 int file_there;
163
164 if (f->filetype != FIO_TYPE_FILE)
165 continue;
96d32d51
JA
166 if (f->flags & FIO_FILE_EXISTS)
167 continue;
168
9c60ce64
JA
169 if (!td->file_size_low)
170 f->file_size = total_file_size / new_files;
171 else {
172 /*
173 * If we don't have enough space left for a file
174 * of the minimum size, bail.
175 */
176 if (local_file_size < td->file_size_low) {
177 log_info("fio: limited to %d files\n", i);
178 new_files -= (td->nr_files - i);
179 td->nr_files = i;
180 break;
181 }
182
183 f->file_size = set_rand_file_size(td, local_file_size);
184 local_file_size -= f->file_size;
185 }
eff6861d 186
af52b345 187 file_there = !file_ok(td, f);
eff6861d 188
af52b345
JA
189 if (file_there && td_write(td) && !td->overwrite) {
190 unlink(f->file_name);
191 file_there = 0;
eff6861d 192 }
af52b345
JA
193
194 need_create += !file_there;
eff6861d 195 }
25205e97 196
25205e97
JA
197 if (!need_create)
198 return 0;
199
f1027063
JA
200 if (!td->total_file_size) {
201 log_err("Need size for create\n");
e1161c32 202 td_verror(td, EINVAL, "file_size");
f1027063
JA
203 return 1;
204 }
205
53cdc686 206 temp_stall_ts = 1;
6d86144d
JA
207 log_info("%s: Laying out IO file(s) (%u files / %LuMiB)\n",
208 td->name, new_files, total_file_size >> 20);
53cdc686
JA
209
210 err = 0;
211 for_each_file(td, f, i) {
9b031dc8
JA
212 /*
213 * Only unlink files that we created.
214 */
f11bd94d 215 f->flags &= ~FIO_FILE_UNLINK;
25205e97 216 if (file_ok(td, f)) {
f11bd94d
JA
217 if (td->unlink)
218 f->flags |= FIO_FILE_UNLINK;
219
25205e97
JA
220 err = create_file(td, f);
221 if (err)
222 break;
223 }
53cdc686
JA
224 }
225
226 temp_stall_ts = 0;
227 return err;
228}
229
230static int file_size(struct thread_data *td, struct fio_file *f)
231{
232 struct stat st;
233
234 if (td->overwrite) {
235 if (fstat(f->fd, &st) == -1) {
e1161c32 236 td_verror(td, errno, "fstat");
53cdc686
JA
237 return 1;
238 }
239
240 f->real_file_size = st.st_size;
241
242 if (!f->file_size || f->file_size > f->real_file_size)
243 f->file_size = f->real_file_size;
745508dd
JA
244 } else
245 f->real_file_size = f->file_size;
53cdc686 246
53cdc686
JA
247 return 0;
248}
249
250static int bdev_size(struct thread_data *td, struct fio_file *f)
251{
252 unsigned long long bytes;
253 int r;
254
255 r = blockdev_size(f->fd, &bytes);
256 if (r) {
e1161c32 257 td_verror(td, r, "blockdev_size");
53cdc686
JA
258 return 1;
259 }
260
261 f->real_file_size = bytes;
262
263 /*
264 * no extend possibilities, so limit size to device size if too large
265 */
266 if (!f->file_size || f->file_size > f->real_file_size)
267 f->file_size = f->real_file_size;
268
269 f->file_size -= f->file_offset;
270 return 0;
271}
272
273static int get_file_size(struct thread_data *td, struct fio_file *f)
274{
275 int ret = 0;
276
96d32d51
JA
277 if (f->filetype == FIO_TYPE_FILE) {
278 if (!(f->flags & FIO_FILE_EXISTS))
279 ret = file_size(td, f);
280 } else if (f->filetype == FIO_TYPE_BD)
53cdc686
JA
281 ret = bdev_size(td, f);
282 else
283 f->real_file_size = -1;
284
285 if (ret)
286 return ret;
287
288 if (f->file_offset > f->real_file_size) {
289 log_err("%s: offset extends end (%Lu > %Lu)\n", td->name, f->file_offset, f->real_file_size);
290 return 1;
291 }
292
53cdc686
JA
293 return 0;
294}
295
e5b401d4
JA
296int file_invalidate_cache(struct thread_data *td, struct fio_file *f)
297{
298 int ret = 0;
299
da86ccb6 300 if (td->odirect)
b5af8293
JA
301 return 0;
302
e5b401d4
JA
303 /*
304 * FIXME: add blockdev flushing too
305 */
b5af8293 306 if (f->mmap)
e5b401d4 307 ret = madvise(f->mmap, f->file_size, MADV_DONTNEED);
af52b345 308 else if (f->filetype == FIO_TYPE_FILE) {
b5af8293 309 ret = fadvise(f->fd, f->file_offset, f->file_size, POSIX_FADV_DONTNEED);
af52b345 310 } else if (f->filetype == FIO_TYPE_BD) {
b5af8293 311 ret = blockdev_invalidate_cache(f->fd);
af52b345 312 } else if (f->filetype == FIO_TYPE_CHAR)
e5b401d4
JA
313 ret = 0;
314
315 if (ret < 0) {
e1161c32 316 td_verror(td, errno, "invalidate_cache");
e5b401d4
JA
317 return 1;
318 }
319
ad2da605 320 return ret;
e5b401d4
JA
321}
322
b5af8293 323void generic_close_file(struct thread_data fio_unused *td, struct fio_file *f)
53cdc686 324{
b5af8293
JA
325 close(f->fd);
326 f->fd = -1;
53cdc686
JA
327}
328
b5af8293 329int generic_open_file(struct thread_data *td, struct fio_file *f)
53cdc686 330{
53cdc686
JA
331 int flags = 0;
332
2fd233b7
JA
333 if (td->odirect)
334 flags |= OS_O_DIRECT;
335 if (td->sync_io)
336 flags |= O_SYNC;
53cdc686 337
2fd233b7
JA
338 if (td_write(td) || td_rw(td)) {
339 flags |= O_RDWR;
53cdc686 340
af52b345 341 if (f->filetype == FIO_TYPE_FILE)
2fd233b7 342 flags |= O_CREAT;
2fd233b7 343
b5af8293 344 f->fd = open(f->file_name, flags, 0600);
2fd233b7 345 } else {
af52b345 346 if (f->filetype == FIO_TYPE_CHAR)
2fd233b7
JA
347 flags |= O_RDWR;
348 else
349 flags |= O_RDONLY;
350
b5af8293 351 f->fd = open(f->file_name, flags);
53cdc686
JA
352 }
353
354 if (f->fd == -1) {
e1161c32
JA
355 int __e = errno;
356
357 td_verror(td, __e, "open");
358 if (__e == EINVAL && td->odirect)
9d80e114 359 log_err("fio: destination does not support O_DIRECT\n");
8ab53873
JA
360 if (__e == EMFILE)
361 log_err("fio: try reducing/setting openfiles (failed at %u of %u)\n", td->nr_open_files, td->nr_files);
53cdc686
JA
362 return 1;
363 }
364
b5af8293
JA
365 if (get_file_size(td, f))
366 goto err;
367
0263882a 368 if (td->invalidate_cache && file_invalidate_cache(td, f))
b5af8293
JA
369 goto err;
370
371 if (!td_random(td)) {
372 if (fadvise(f->fd, f->file_offset, f->file_size, POSIX_FADV_SEQUENTIAL) < 0) {
373 td_verror(td, errno, "fadvise");
374 goto err;
375 }
376 } else {
377 if (fadvise(f->fd, f->file_offset, f->file_size, POSIX_FADV_RANDOM) < 0) {
378 td_verror(td, errno, "fadvise");
379 goto err;
380 }
bdb4e2e9 381 }
53cdc686
JA
382
383 return 0;
b5af8293
JA
384err:
385 close(f->fd);
386 return 1;
387}
388
21972cde
JA
389int open_files(struct thread_data *td)
390{
391 struct fio_file *f;
af52b345
JA
392 unsigned int i;
393 int err = 0;
21972cde
JA
394
395 for_each_file(td, f, i) {
b5af8293 396 err = td_io_open_file(td, f);
21972cde
JA
397 if (err)
398 break;
b5af8293
JA
399
400 if (td->open_files == td->nr_open_files)
401 break;
21972cde
JA
402 }
403
7abf833d
JA
404 if (!err)
405 return 0;
406
bdb4e2e9 407 for_each_file(td, f, i)
b5af8293 408 td_io_close_file(td, f);
7abf833d 409
21972cde
JA
410 return err;
411}
412
53cdc686
JA
413int setup_files(struct thread_data *td)
414{
415 struct fio_file *f;
af52b345
JA
416 unsigned int i;
417 int err;
53cdc686
JA
418
419 /*
420 * if ioengine defines a setup() method, it's responsible for
421 * setting up everything in the td->files[] area.
422 */
423 if (td->io_ops->setup)
424 return td->io_ops->setup(td);
425
426 if (create_files(td))
427 return 1;
428
21972cde 429 err = open_files(td);
f1027063
JA
430 if (err)
431 return err;
432
0a7eb121
JA
433 /*
434 * Recalculate the total file size now that files are set up.
435 */
436 td->total_file_size = 0;
437 for_each_file(td, f, i)
438 td->total_file_size += f->file_size;
439
f1027063 440 td->io_size = td->total_file_size;
53cdc686
JA
441 if (td->io_size == 0) {
442 log_err("%s: no io blocks\n", td->name);
e1161c32 443 td_verror(td, EINVAL, "total_file_size");
53cdc686
JA
444 return 1;
445 }
446
447 if (!td->zone_size)
448 td->zone_size = td->io_size;
449
450 td->total_io_size = td->io_size * td->loops;
451
bdb4e2e9 452 for_each_file(td, f, i)
b5af8293 453 td_io_close_file(td, f);
21972cde
JA
454
455 return err;
53cdc686
JA
456}
457
458void close_files(struct thread_data *td)
459{
0ab8db89 460 struct fio_file *f;
af52b345 461 unsigned int i;
53cdc686 462
0ab8db89 463 for_each_file(td, f, i) {
1315a68a 464 if (!f->file_name && (f->flags & FIO_FILE_UNLINK) &&
af52b345 465 f->filetype == FIO_TYPE_FILE) {
132ad46d 466 unlink(f->file_name);
1315a68a 467 free(f->file_name);
132ad46d
JA
468 f->file_name = NULL;
469 }
bdb4e2e9 470
b5af8293 471 td_io_close_file(td, f);
b3dc7f07
JA
472
473 if (f->file_map)
474 free(f->file_map);
53cdc686 475 }
b4a6a59a 476
132ad46d 477 td->filename = NULL;
b4a6a59a
JA
478 td->files = NULL;
479 td->nr_files = 0;
53cdc686 480}
af52b345 481
e3bab463 482static void get_file_type(struct fio_file *f)
af52b345
JA
483{
484 struct stat sb;
485
486 f->filetype = FIO_TYPE_FILE;
487
e3bab463 488 if (!lstat(f->file_name, &sb)) {
96d32d51
JA
489 f->flags |= FIO_FILE_EXISTS;
490
af52b345
JA
491 if (S_ISBLK(sb.st_mode))
492 f->filetype = FIO_TYPE_BD;
493 else if (S_ISCHR(sb.st_mode))
494 f->filetype = FIO_TYPE_CHAR;
96d32d51
JA
495 else {
496 /*
497 * might as well do this here, and save a stat later on
498 */
499 f->real_file_size = sb.st_size;
500 f->file_size = f->real_file_size;
501 }
af52b345
JA
502 }
503}
504
505void add_file(struct thread_data *td, const char *fname)
506{
7b4e4fe5 507 int cur_files = td->files_index;
af52b345
JA
508 struct fio_file *f;
509
510 td->files = realloc(td->files, (cur_files + 1) * sizeof(*f));
511
512 f = &td->files[cur_files];
513 memset(f, 0, sizeof(*f));
514 f->fd = -1;
cae61953 515 f->file_name = strdup(fname);
af52b345 516
e3bab463 517 get_file_type(f);
af52b345 518
7b4e4fe5 519 td->files_index++;
1549441c
JA
520 if (f->filetype == FIO_TYPE_FILE)
521 td->nr_normal_files++;
af52b345 522}
0ad920e7
JA
523
524void get_file(struct fio_file *f)
525{
526 f->references++;
527}
528
529void put_file(struct thread_data *td, struct fio_file *f)
530{
531 if (!(f->flags & FIO_FILE_OPEN))
532 return;
533
534 assert(f->references);
535 if (--f->references)
536 return;
537
ebb1415f
JA
538 if (should_fsync(td) && td->fsync_on_close)
539 fsync(f->fd);
540
0ad920e7
JA
541 if (td->io_ops->close_file)
542 td->io_ops->close_file(td, f);
543 td->nr_open_files--;
544 f->flags &= ~FIO_FILE_OPEN;
545}
bbf6b540
JA
546
547static int recurse_dir(struct thread_data *td, const char *dirname)
548{
549 struct dirent *dir;
550 int ret = 0;
551 DIR *D;
552
553 D = opendir(dirname);
554 if (!D) {
555 td_verror(td, errno, "opendir");
556 return 1;
557 }
558
559 while ((dir = readdir(D)) != NULL) {
560 char full_path[PATH_MAX];
561 struct stat sb;
562
e85b2b83
JA
563 if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, ".."))
564 continue;
96d32d51 565
bbf6b540
JA
566 sprintf(full_path, "%s/%s", dirname, dir->d_name);
567
568 if (lstat(full_path, &sb) == -1) {
569 if (errno != ENOENT) {
570 td_verror(td, errno, "stat");
571 return 1;
572 }
573 }
574
575 if (S_ISREG(sb.st_mode)) {
576 add_file(td, full_path);
577 td->nr_files++;
578 continue;
579 }
580
bbf6b540
JA
581 if ((ret = recurse_dir(td, full_path)) != 0)
582 break;
583 }
584
585 closedir(D);
586 return ret;
587}
588
589int add_dir_files(struct thread_data *td, const char *path)
590{
591 return recurse_dir(td, path);
592}