- if (td->thinktime)
- usec_sleep(td, td->thinktime);
-
- if (should_fsync(td) && td->fsync_blocks &&
- (td->io_blocks[DDIR_WRITE] % td->fsync_blocks) == 0)
- sync_td(td);
- }
-
- if (td->cur_depth)
- cleanup_pending_aio(td);
-
- if (should_fsync(td) && td->end_fsync)
- sync_td(td);
-}
-
-static void cleanup_io(struct thread_data *td)
-{
- if (td->io_cleanup)
- td->io_cleanup(td);
-}
-
-static int init_io(struct thread_data *td)
-{
- if (td->io_engine == FIO_SYNCIO)
- return fio_syncio_init(td);
- else if (td->io_engine == FIO_MMAPIO)
- return fio_mmapio_init(td);
- else if (td->io_engine == FIO_LIBAIO)
- return fio_libaio_init(td);
- else if (td->io_engine == FIO_POSIXAIO)
- return fio_posixaio_init(td);
- else if (td->io_engine == FIO_SGIO)
- return fio_sgio_init(td);
- else if (td->io_engine == FIO_SPLICEIO)
- return fio_spliceio_init(td);
- else {
- fprintf(stderr, "bad io_engine %d\n", td->io_engine);
- return 1;
- }
-}
-
-static void cleanup_io_u(struct thread_data *td)
-{
- struct list_head *entry, *n;
- struct io_u *io_u;
-
- list_for_each_safe(entry, n, &td->io_u_freelist) {
- io_u = list_entry(entry, struct io_u, list);
-
- list_del(&io_u->list);
- free(io_u);
- }
-
- if (td->mem_type == MEM_MALLOC)
- free(td->orig_buffer);
- else if (td->mem_type == MEM_SHM) {
- struct shmid_ds sbuf;
-
- shmdt(td->orig_buffer);
- shmctl(td->shm_id, IPC_RMID, &sbuf);
- } else if (td->mem_type == MEM_MMAP)
- munmap(td->orig_buffer, td->orig_buffer_size);
- else
- fprintf(stderr, "Bad memory type %d\n", td->mem_type);
-
- td->orig_buffer = NULL;
-}
-
-static int init_io_u(struct thread_data *td)
-{
- struct io_u *io_u;
- int i, max_units;
- char *p;
-
- if (td->io_engine & FIO_SYNCIO)
- max_units = 1;
- else
- max_units = td->iodepth;
-
- td->orig_buffer_size = td->max_bs * max_units + MASK;
-
- if (td->mem_type == MEM_MALLOC)
- td->orig_buffer = malloc(td->orig_buffer_size);
- else if (td->mem_type == MEM_SHM) {
- td->shm_id = shmget(IPC_PRIVATE, td->orig_buffer_size, IPC_CREAT | 0600);
- if (td->shm_id < 0) {
- td_verror(td, errno);
- perror("shmget");
- return 1;
- }
-
- td->orig_buffer = shmat(td->shm_id, NULL, 0);
- if (td->orig_buffer == (void *) -1) {
- td_verror(td, errno);
- perror("shmat");
- td->orig_buffer = NULL;
- return 1;
- }
- } else if (td->mem_type == MEM_MMAP) {
- td->orig_buffer = mmap(NULL, td->orig_buffer_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | OS_MAP_ANON, 0, 0);
- if (td->orig_buffer == MAP_FAILED) {
- td_verror(td, errno);
- perror("mmap");
- td->orig_buffer = NULL;
- return 1;
- }
- }
-
- p = ALIGN(td->orig_buffer);
- for (i = 0; i < max_units; i++) {
- io_u = malloc(sizeof(*io_u));
- memset(io_u, 0, sizeof(*io_u));
- INIT_LIST_HEAD(&io_u->list);
-
- io_u->buf = p + td->max_bs * i;
- io_u->index = i;
- list_add(&io_u->list, &td->io_u_freelist);
- }
-
- return 0;
-}
-
-static void cleanup_allocs(struct thread_data *td)
-{
- if (td->directory)
- free(td->directory);
- if (td->iolog_file)
- free(td->iolog_file);
- if (td->exec_prerun)
- free(td->exec_prerun);
- if (td->exec_postrun)
- free(td->exec_postrun);
- if (td->ioscheduler)
- free(td->ioscheduler);
- if (td->sysfs_root)
- free(td->sysfs_root);
-}
-
-static int create_file(struct thread_data *td, unsigned long long size,
- int extend)
-{
- unsigned long long left;
- unsigned int bs;
- int r, oflags;
- char *b;
-
- /*
- * unless specifically asked for overwrite, let normal io extend it
- */
- if (td_write(td) && !td->overwrite)
- return 0;
-
- if (!size) {
- fprintf(stderr, "Need size for create\n");
- td_verror(td, EINVAL);
- return 1;
- }
-
- if (!extend) {
- oflags = O_CREAT | O_TRUNC;
- printf("Client%d: Laying out IO file (%LuMiB)\n", td->thread_number, size >> 20);
- } else {
- oflags = O_APPEND;
- printf("Client%d: Extending IO file (%Lu -> %LuMiB)\n", td->thread_number, (td->file_size - size) >> 20, td->file_size >> 20);
- }
-
- td->fd = open(td->file_name, O_WRONLY | oflags, 0644);
- if (td->fd < 0) {
- td_verror(td, errno);
- return 1;
- }
-
- if (!extend && ftruncate(td->fd, td->file_size) == -1) {
- td_verror(td, errno);
- return 1;
- }
-
- td->io_size = td->file_size;
- b = malloc(td->max_bs);
- memset(b, 0, td->max_bs);
-
- left = size;
- while (left && !td->terminate) {
- bs = td->max_bs;
- if (bs > left)
- bs = left;
-
- r = write(td->fd, b, bs);
-
- if (r == (int) bs) {
- left -= bs;
- continue;
- } else {
- if (r < 0)
- td_verror(td, errno);
- else
- td_verror(td, EIO);
-
- break;
- }
- }
-
- if (td->terminate)
- unlink(td->file_name);
- else if (td->create_fsync)
- fsync(td->fd);
-
- close(td->fd);
- td->fd = -1;
- free(b);
- return 0;
-}
-
-static int file_size(struct thread_data *td)
-{
- struct stat st;
-
- if (fstat(td->fd, &st) == -1) {
- td_verror(td, errno);
- return 1;
- }
-
- td->real_file_size = st.st_size;
-
- if (!td->file_size || td->file_size > td->real_file_size)
- td->file_size = td->real_file_size;
-
- td->file_size -= td->file_offset;
- return 0;
-}
-
-static int bdev_size(struct thread_data *td)
-{
- unsigned long long bytes;
- int r;
-
- r = blockdev_size(td->fd, &bytes);
- if (r) {
- td_verror(td, r);
- return 1;
- }
-
- td->real_file_size = bytes;
-
- /*
- * no extend possibilities, so limit size to device size if too large
- */
- if (!td->file_size || td->file_size > td->real_file_size)
- td->file_size = td->real_file_size;
-
- td->file_size -= td->file_offset;
- return 0;
-}
-
-static int get_file_size(struct thread_data *td)
-{
- int ret = 0;
-
- if (td->filetype == FIO_TYPE_FILE)
- ret = file_size(td);
- else if (td->filetype == FIO_TYPE_BD)
- ret = bdev_size(td);
- else
- td->real_file_size = -1;
-
- if (ret)
- return ret;
-
- if (td->file_offset > td->real_file_size) {
- fprintf(stderr, "Client%d: offset extends end (%Lu > %Lu)\n", td->thread_number, td->file_offset, td->real_file_size);
- return 1;
- }
-
- td->io_size = td->file_size;
- if (td->io_size == 0) {
- fprintf(stderr, "Client%d: no io blocks\n", td->thread_number);
- td_verror(td, EINVAL);
- return 1;
- }
-
- if (!td->zone_size)
- td->zone_size = td->io_size;
-
- td->total_io_size = td->io_size * td->loops;
- return 0;
-}
-
-static int setup_file_mmap(struct thread_data *td)
-{
- int flags;
-
- if (td_rw(td))
- flags = PROT_READ | PROT_WRITE;
- else if (td_write(td)) {
- flags = PROT_WRITE;
-
- if (td->verify != VERIFY_NONE)
- flags |= PROT_READ;
- } else
- flags = PROT_READ;
-
- td->mmap = mmap(NULL, td->file_size, flags, MAP_SHARED, td->fd, td->file_offset);
- if (td->mmap == MAP_FAILED) {
- td->mmap = NULL;
- td_verror(td, errno);
- return 1;
- }
-
- if (td->invalidate_cache) {
- if (madvise(td->mmap, td->file_size, MADV_DONTNEED) < 0) {
- td_verror(td, errno);
- return 1;
- }
- }
-
- if (td->sequential) {
- if (madvise(td->mmap, td->file_size, MADV_SEQUENTIAL) < 0) {
- td_verror(td, errno);
- return 1;
- }
- } else {
- if (madvise(td->mmap, td->file_size, MADV_RANDOM) < 0) {
- td_verror(td, errno);
- return 1;
- }
- }
-
- return 0;
-}
-
-static int setup_file_plain(struct thread_data *td)
-{
- if (td->invalidate_cache) {
- if (fadvise(td->fd, td->file_offset, td->file_size, POSIX_FADV_DONTNEED) < 0) {
- td_verror(td, errno);
- return 1;
- }
- }
-
- if (td->sequential) {
- if (fadvise(td->fd, td->file_offset, td->file_size, POSIX_FADV_SEQUENTIAL) < 0) {
- td_verror(td, errno);
- return 1;
- }
- } else {
- if (fadvise(td->fd, td->file_offset, td->file_size, POSIX_FADV_RANDOM) < 0) {
- td_verror(td, errno);
- return 1;
- }
- }
-
- return 0;
-}
-
-static int setup_file(struct thread_data *td)
-{
- struct stat st;
- int flags = 0;
-
- if (stat(td->file_name, &st) == -1) {
- if (errno != ENOENT) {
- td_verror(td, errno);
- return 1;
- }
- if (!td->create_file) {
- td_verror(td, ENOENT);
- return 1;
- }
- if (create_file(td, td->file_size, 0))
- return 1;
- } else if (td->filetype == FIO_TYPE_FILE) {
- if (st.st_size < (off_t) td->file_size) {
- if (create_file(td, td->file_size - st.st_size, 1))
- return 1;
- }
- }
-
- if (td->odirect)
- flags |= O_DIRECT;
-
- if (td_write(td) || td_rw(td)) {
- if (td->filetype == FIO_TYPE_FILE) {
- if (!td->overwrite)
- flags |= O_TRUNC;
-
- flags |= O_CREAT;
- }
- if (td->sync_io)
- flags |= O_SYNC;
-
- flags |= O_RDWR;
-
- td->fd = open(td->file_name, flags, 0600);
- } else {
- if (td->filetype == FIO_TYPE_CHAR)
- flags |= O_RDWR;
- else
- flags |= O_RDONLY;
-
- td->fd = open(td->file_name, flags);
- }
-
- if (td->fd == -1) {
- td_verror(td, errno);
- return 1;
- }
-
- if (get_file_size(td))
- return 1;
-
- if (td->io_engine != FIO_MMAPIO)
- return setup_file_plain(td);
- else
- return setup_file_mmap(td);
-}
-
-static int check_dev_match(dev_t dev, char *path)
-{
- unsigned int major, minor;
- char line[256], *p;
- FILE *f;
-
- f = fopen(path, "r");
- if (!f) {
- perror("open path");
- return 1;
- }
-
- p = fgets(line, sizeof(line), f);
- if (!p) {
- fclose(f);
- return 1;
- }
-
- if (sscanf(p, "%u:%u", &major, &minor) != 2) {
- fclose(f);
- return 1;
- }
-
- if (((major << 8) | minor) == dev) {
- fclose(f);
- return 0;
- }
-
- fclose(f);
- return 1;
-}
-
-static int find_block_dir(dev_t dev, char *path)
-{
- struct dirent *dir;
- struct stat st;
- int found = 0;
- DIR *D;
-
- D = opendir(path);
- if (!D)
- return 0;
-
- while ((dir = readdir(D)) != NULL) {
- char full_path[256];
-
- if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, ".."))
- continue;
- if (!strcmp(dir->d_name, "device"))
- continue;
-
- sprintf(full_path, "%s/%s", path, dir->d_name);
-
- if (!strcmp(dir->d_name, "dev")) {
- if (!check_dev_match(dev, full_path)) {
- found = 1;
- break;
- }
- }
-
- if (stat(full_path, &st) == -1) {
- perror("stat");
- break;
- }
-
- if (!S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))
- continue;
-
- found = find_block_dir(dev, full_path);
- if (found) {
- strcpy(path, full_path);
- break;
- }
- }
-
- closedir(D);
- return found;
-}
-
-static int get_io_ticks(struct disk_util *du, struct disk_util_stat *dus)
-{
- unsigned in_flight;
- char line[256];
- FILE *f;
- char *p;
-
- f = fopen(du->path, "r");
- if (!f)
- return 1;
-
- p = fgets(line, sizeof(line), f);
- if (!p) {
- fclose(f);
- return 1;
- }
-
- if (sscanf(p, "%u %u %llu %u %u %u %llu %u %u %u %u\n", &dus->ios[0], &dus->merges[0], &dus->sectors[0], &dus->ticks[0], &dus->ios[1], &dus->merges[1], &dus->sectors[1], &dus->ticks[1], &in_flight, &dus->io_ticks, &dus->time_in_queue) != 11) {
- fclose(f);
- return 1;
- }
-
- fclose(f);
- return 0;
-}
-
-static void update_io_tick_disk(struct disk_util *du)
-{
- struct disk_util_stat __dus, *dus, *ldus;
- struct timeval t;
-
- if (get_io_ticks(du, &__dus))
- return;