Allow explicit setting of a number of files
[fio.git] / filesetup.c
CommitLineData
53cdc686
JA
1#include <unistd.h>
2#include <fcntl.h>
3#include <string.h>
4#include <assert.h>
5#include <sys/stat.h>
6#include <sys/mman.h>
7
8#include "fio.h"
9#include "os.h"
10
25205e97
JA
11/*
12 * Check if the file exists and it's large enough.
13 */
14static int file_ok(struct thread_data *td, struct fio_file *f)
53cdc686 15{
b2a15192 16 struct stat st;
53cdc686 17
af52b345 18 if (f->filetype != FIO_TYPE_FILE ||
b5af8293 19 (td->io_ops->flags & FIO_DISKLESSIO))
b2a15192
JA
20 return 0;
21
fa01d139 22 if (lstat(f->file_name, &st) == -1)
25205e97 23 return 1;
fa01d139
JA
24
25 /*
26 * if it's a special file, size is always ok for now
27 */
28 if (S_ISCHR(st.st_mode) || S_ISBLK(st.st_mode))
29 return 0;
30 if (st.st_size < (off_t) f->file_size)
25205e97
JA
31 return 1;
32
33 return 0;
34}
35
36static int create_file(struct thread_data *td, struct fio_file *f)
37{
38 unsigned long long left;
39 unsigned int bs;
40 char *b;
41 int r;
b2a15192 42
53cdc686
JA
43 f->fd = open(f->file_name, O_WRONLY | O_CREAT | O_TRUNC, 0644);
44 if (f->fd < 0) {
e1161c32 45 td_verror(td, errno, "open");
53cdc686
JA
46 return 1;
47 }
48
49 if (ftruncate(f->fd, f->file_size) == -1) {
e1161c32 50 td_verror(td, errno, "ftruncate");
53cdc686
JA
51 goto err;
52 }
53
40f8298c 54 if (posix_fallocate(f->fd, 0, f->file_size) < 0) {
e1161c32 55 td_verror(td, errno, "posix_fallocate");
40f8298c
JA
56 goto err;
57 }
58
a00735e6
JA
59 b = malloc(td->max_bs[DDIR_WRITE]);
60 memset(b, 0, td->max_bs[DDIR_WRITE]);
53cdc686
JA
61
62 left = f->file_size;
63 while (left && !td->terminate) {
a00735e6 64 bs = td->max_bs[DDIR_WRITE];
53cdc686
JA
65 if (bs > left)
66 bs = left;
67
68 r = write(f->fd, b, bs);
69
70 if (r == (int) bs) {
71 left -= bs;
72 continue;
73 } else {
74 if (r < 0)
e1161c32 75 td_verror(td, errno, "write");
53cdc686 76 else
e1161c32 77 td_verror(td, EIO, "write");
53cdc686
JA
78
79 break;
80 }
81 }
82
83 if (td->terminate)
84 unlink(f->file_name);
85 else if (td->create_fsync)
86 fsync(f->fd);
87
88 free(b);
89 close(f->fd);
90 f->fd = -1;
91 return 0;
92err:
93 close(f->fd);
94 f->fd = -1;
95 return 1;
96}
97
98static int create_files(struct thread_data *td)
99{
100 struct fio_file *f;
af52b345
JA
101 int err, need_create, can_extend;
102 unsigned int i;
53cdc686 103
f697125a
JA
104 for_each_file(td, f, i)
105 f->file_size = td->total_file_size / td->nr_files;
106
53cdc686
JA
107 /*
108 * unless specifically asked for overwrite, let normal io extend it
109 */
0263882a
JA
110 can_extend = !td->overwrite && !(td->io_ops->flags & FIO_NOEXTEND);
111 if (can_extend)
53cdc686 112 return 0;
53cdc686 113
25205e97 114 need_create = 0;
af52b345
JA
115 for_each_file(td, f, i) {
116 int file_there;
117
118 if (f->filetype != FIO_TYPE_FILE)
119 continue;
eff6861d 120
af52b345 121 file_there = !file_ok(td, f);
eff6861d 122
af52b345
JA
123 if (file_there && td_write(td) && !td->overwrite) {
124 unlink(f->file_name);
125 file_there = 0;
eff6861d 126 }
af52b345
JA
127
128 need_create += !file_there;
eff6861d 129 }
25205e97 130
25205e97
JA
131 if (!need_create)
132 return 0;
133
f1027063
JA
134 if (!td->total_file_size) {
135 log_err("Need size for create\n");
e1161c32 136 td_verror(td, EINVAL, "file_size");
f1027063
JA
137 return 1;
138 }
139
53cdc686 140 temp_stall_ts = 1;
1e97cce9 141 fprintf(f_out, "%s: Laying out IO file(s) (%u x %LuMiB == %LuMiB)\n",
13f8e2d2
JA
142 td->name, td->nr_uniq_files,
143 (td->total_file_size >> 20) / td->nr_uniq_files,
25205e97 144 td->total_file_size >> 20);
53cdc686
JA
145
146 err = 0;
147 for_each_file(td, f, i) {
9b031dc8
JA
148 /*
149 * Only unlink files that we created.
150 */
151 f->unlink = 0;
25205e97 152 if (file_ok(td, f)) {
9b031dc8 153 f->unlink = td->unlink;
25205e97
JA
154 err = create_file(td, f);
155 if (err)
156 break;
157 }
53cdc686
JA
158 }
159
160 temp_stall_ts = 0;
161 return err;
162}
163
164static int file_size(struct thread_data *td, struct fio_file *f)
165{
166 struct stat st;
167
168 if (td->overwrite) {
169 if (fstat(f->fd, &st) == -1) {
e1161c32 170 td_verror(td, errno, "fstat");
53cdc686
JA
171 return 1;
172 }
173
174 f->real_file_size = st.st_size;
175
176 if (!f->file_size || f->file_size > f->real_file_size)
177 f->file_size = f->real_file_size;
745508dd
JA
178 } else
179 f->real_file_size = f->file_size;
53cdc686 180
53cdc686
JA
181 return 0;
182}
183
184static int bdev_size(struct thread_data *td, struct fio_file *f)
185{
186 unsigned long long bytes;
187 int r;
188
189 r = blockdev_size(f->fd, &bytes);
190 if (r) {
e1161c32 191 td_verror(td, r, "blockdev_size");
53cdc686
JA
192 return 1;
193 }
194
195 f->real_file_size = bytes;
196
197 /*
198 * no extend possibilities, so limit size to device size if too large
199 */
200 if (!f->file_size || f->file_size > f->real_file_size)
201 f->file_size = f->real_file_size;
202
203 f->file_size -= f->file_offset;
204 return 0;
205}
206
207static int get_file_size(struct thread_data *td, struct fio_file *f)
208{
209 int ret = 0;
210
af52b345 211 if (f->filetype == FIO_TYPE_FILE)
53cdc686 212 ret = file_size(td, f);
af52b345 213 else if (f->filetype == FIO_TYPE_BD)
53cdc686
JA
214 ret = bdev_size(td, f);
215 else
216 f->real_file_size = -1;
217
218 if (ret)
219 return ret;
220
221 if (f->file_offset > f->real_file_size) {
222 log_err("%s: offset extends end (%Lu > %Lu)\n", td->name, f->file_offset, f->real_file_size);
223 return 1;
224 }
225
53cdc686
JA
226 return 0;
227}
228
e5b401d4
JA
229int file_invalidate_cache(struct thread_data *td, struct fio_file *f)
230{
231 int ret = 0;
232
da86ccb6 233 if (td->odirect)
b5af8293
JA
234 return 0;
235
e5b401d4
JA
236 /*
237 * FIXME: add blockdev flushing too
238 */
b5af8293 239 if (f->mmap)
e5b401d4 240 ret = madvise(f->mmap, f->file_size, MADV_DONTNEED);
af52b345 241 else if (f->filetype == FIO_TYPE_FILE) {
b5af8293 242 ret = fadvise(f->fd, f->file_offset, f->file_size, POSIX_FADV_DONTNEED);
af52b345 243 } else if (f->filetype == FIO_TYPE_BD) {
b5af8293 244 ret = blockdev_invalidate_cache(f->fd);
af52b345 245 } else if (f->filetype == FIO_TYPE_CHAR)
e5b401d4
JA
246 ret = 0;
247
248 if (ret < 0) {
e1161c32 249 td_verror(td, errno, "invalidate_cache");
e5b401d4
JA
250 return 1;
251 }
252
ad2da605 253 return ret;
e5b401d4
JA
254}
255
b5af8293 256void generic_close_file(struct thread_data fio_unused *td, struct fio_file *f)
53cdc686 257{
b5af8293
JA
258 close(f->fd);
259 f->fd = -1;
53cdc686
JA
260}
261
b5af8293 262int generic_open_file(struct thread_data *td, struct fio_file *f)
53cdc686 263{
53cdc686
JA
264 int flags = 0;
265
2fd233b7
JA
266 if (td->odirect)
267 flags |= OS_O_DIRECT;
268 if (td->sync_io)
269 flags |= O_SYNC;
53cdc686 270
2fd233b7
JA
271 if (td_write(td) || td_rw(td)) {
272 flags |= O_RDWR;
53cdc686 273
af52b345 274 if (f->filetype == FIO_TYPE_FILE)
2fd233b7 275 flags |= O_CREAT;
2fd233b7 276
b5af8293 277 f->fd = open(f->file_name, flags, 0600);
2fd233b7 278 } else {
af52b345 279 if (f->filetype == FIO_TYPE_CHAR)
2fd233b7
JA
280 flags |= O_RDWR;
281 else
282 flags |= O_RDONLY;
283
b5af8293 284 f->fd = open(f->file_name, flags);
53cdc686
JA
285 }
286
287 if (f->fd == -1) {
e1161c32
JA
288 int __e = errno;
289
290 td_verror(td, __e, "open");
291 if (__e == EINVAL && td->odirect)
9d80e114 292 log_err("fio: destination does not support O_DIRECT\n");
53cdc686
JA
293 return 1;
294 }
295
b5af8293
JA
296 if (get_file_size(td, f))
297 goto err;
298
0263882a 299 if (td->invalidate_cache && file_invalidate_cache(td, f))
b5af8293
JA
300 goto err;
301
302 if (!td_random(td)) {
303 if (fadvise(f->fd, f->file_offset, f->file_size, POSIX_FADV_SEQUENTIAL) < 0) {
304 td_verror(td, errno, "fadvise");
305 goto err;
306 }
307 } else {
308 if (fadvise(f->fd, f->file_offset, f->file_size, POSIX_FADV_RANDOM) < 0) {
309 td_verror(td, errno, "fadvise");
310 goto err;
311 }
bdb4e2e9 312 }
53cdc686
JA
313
314 return 0;
b5af8293
JA
315err:
316 close(f->fd);
317 return 1;
318}
319
21972cde
JA
320int open_files(struct thread_data *td)
321{
322 struct fio_file *f;
af52b345
JA
323 unsigned int i;
324 int err = 0;
21972cde
JA
325
326 for_each_file(td, f, i) {
b5af8293 327 err = td_io_open_file(td, f);
21972cde
JA
328 if (err)
329 break;
b5af8293
JA
330
331 if (td->open_files == td->nr_open_files)
332 break;
21972cde
JA
333 }
334
7abf833d
JA
335 if (!err)
336 return 0;
337
bdb4e2e9 338 for_each_file(td, f, i)
b5af8293 339 td_io_close_file(td, f);
7abf833d 340
21972cde
JA
341 return err;
342}
343
53cdc686
JA
344int setup_files(struct thread_data *td)
345{
346 struct fio_file *f;
af52b345
JA
347 unsigned int i;
348 int err;
53cdc686
JA
349
350 /*
351 * if ioengine defines a setup() method, it's responsible for
352 * setting up everything in the td->files[] area.
353 */
354 if (td->io_ops->setup)
355 return td->io_ops->setup(td);
356
357 if (create_files(td))
358 return 1;
359
21972cde 360 err = open_files(td);
f1027063
JA
361 if (err)
362 return err;
363
0a7eb121
JA
364 /*
365 * Recalculate the total file size now that files are set up.
366 */
367 td->total_file_size = 0;
368 for_each_file(td, f, i)
369 td->total_file_size += f->file_size;
370
0f14fef3
JA
371 td->total_file_size = (td->total_file_size * td->nr_files) / td->open_files;
372
f1027063 373 td->io_size = td->total_file_size;
53cdc686
JA
374 if (td->io_size == 0) {
375 log_err("%s: no io blocks\n", td->name);
e1161c32 376 td_verror(td, EINVAL, "total_file_size");
53cdc686
JA
377 return 1;
378 }
379
380 if (!td->zone_size)
381 td->zone_size = td->io_size;
382
383 td->total_io_size = td->io_size * td->loops;
384
bdb4e2e9 385 for_each_file(td, f, i)
b5af8293 386 td_io_close_file(td, f);
21972cde
JA
387
388 return err;
53cdc686
JA
389}
390
391void close_files(struct thread_data *td)
392{
0ab8db89 393 struct fio_file *f;
af52b345 394 unsigned int i;
53cdc686 395
0ab8db89 396 for_each_file(td, f, i) {
9b031dc8 397 if (!td->filename && f->unlink &&
af52b345 398 f->filetype == FIO_TYPE_FILE) {
132ad46d 399 unlink(f->file_name);
132ad46d
JA
400 f->file_name = NULL;
401 }
bdb4e2e9 402
b5af8293 403 td_io_close_file(td, f);
b3dc7f07
JA
404
405 if (f->file_map)
406 free(f->file_map);
53cdc686 407 }
b4a6a59a 408
132ad46d 409 td->filename = NULL;
b4a6a59a
JA
410 free(td->files);
411 td->files = NULL;
412 td->nr_files = 0;
53cdc686 413}
af52b345
JA
414
415static void get_file_type(struct thread_data *td, struct fio_file *f)
416{
417 struct stat sb;
418
419 f->filetype = FIO_TYPE_FILE;
420
421 if (!lstat(td->filename, &sb)) {
422 if (S_ISBLK(sb.st_mode))
423 f->filetype = FIO_TYPE_BD;
424 else if (S_ISCHR(sb.st_mode))
425 f->filetype = FIO_TYPE_CHAR;
426 }
427}
428
429void add_file(struct thread_data *td, const char *fname)
430{
431 int cur_files = td->open_files;
432 struct fio_file *f;
433
434 td->files = realloc(td->files, (cur_files + 1) * sizeof(*f));
435
436 f = &td->files[cur_files];
437 memset(f, 0, sizeof(*f));
438 f->fd = -1;
439 f->file_name = fname;
440
441 get_file_type(td, f);
442
443 td->open_files++;
444 td->nr_uniq_files = td->open_files;
445}