X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=fio.c;h=5673d9ef690cc8e8c6d6bbe31dccc5fb3cc937ba;hp=9a1da292f9d5b5ab580948955f36794f921ecd7a;hb=56f9498d8588d42df2a41a7eb7ebcffef013a339;hpb=84585003d025a38b91749cb0d68f6b5653d1f1a3 diff --git a/fio.c b/fio.c index 9a1da292..5673d9ef 100644 --- a/fio.c +++ b/fio.c @@ -45,8 +45,6 @@ int shm_id = 0; int temp_stall_ts; char *fio_inst_prefix = _INST_PREFIX; -extern unsigned long long mlock_size; - #define should_fsync(td) ((td_write(td) || td_rw(td)) && (!(td)->odirect || (td)->override_sync)) static volatile int startup_sem; @@ -56,11 +54,10 @@ static volatile int startup_sem; static void terminate_threads(int group_id) { + struct thread_data *td; int i; - for (i = 0; i < thread_number; i++) { - struct thread_data *td = &threads[i]; - + for_each_td(td, i) { if (group_id == TERMINATE_ALL || groupid == td->groupid) { td->terminate = 1; td->start_delay = 0; @@ -84,134 +81,6 @@ static void sig_handler(int sig) } } -/* - * The ->file_map[] contains a map of blocks we have or have not done io - * to yet. Used to make sure we cover the entire range in a fair fashion. - */ -static int random_map_free(struct thread_data *td, struct fio_file *f, - unsigned long long block) -{ - unsigned int idx = RAND_MAP_IDX(td, f, block); - unsigned int bit = RAND_MAP_BIT(td, f, block); - - return (f->file_map[idx] & (1UL << bit)) == 0; -} - -/* - * Return the next free block in the map. - */ -static int get_next_free_block(struct thread_data *td, struct fio_file *f, - unsigned long long *b) -{ - int i; - - *b = 0; - i = 0; - while ((*b) * td->min_bs < f->file_size) { - if (f->file_map[i] != -1UL) { - *b += ffz(f->file_map[i]); - return 0; - } - - *b += BLOCKS_PER_MAP; - i++; - } - - return 1; -} - -/* - * Mark a given offset as used in the map. - */ -static void mark_random_map(struct thread_data *td, struct fio_file *f, - struct io_u *io_u) -{ - unsigned long long block = io_u->offset / (unsigned long long) td->min_bs; - unsigned int blocks = 0; - - while (blocks < (io_u->buflen / td->min_bs)) { - unsigned int idx, bit; - - if (!random_map_free(td, f, block)) - break; - - idx = RAND_MAP_IDX(td, f, block); - bit = RAND_MAP_BIT(td, f, block); - - assert(idx < f->num_maps); - - f->file_map[idx] |= (1UL << bit); - block++; - blocks++; - } - - if ((blocks * td->min_bs) < io_u->buflen) - io_u->buflen = blocks * td->min_bs; -} - -/* - * For random io, generate a random new block and see if it's used. Repeat - * until we find a free one. For sequential io, just return the end of - * the last io issued. - */ -static int get_next_offset(struct thread_data *td, struct fio_file *f, - unsigned long long *offset) -{ - unsigned long long b, rb; - long r; - - if (!td->sequential) { - unsigned long long max_blocks = td->io_size / td->min_bs; - int loops = 50; - - do { - r = os_random_long(&td->random_state); - b = ((max_blocks - 1) * r / (unsigned long long) (RAND_MAX+1.0)); - rb = b + (f->file_offset / td->min_bs); - loops--; - } while (!random_map_free(td, f, rb) && loops); - - if (!loops) { - if (get_next_free_block(td, f, &b)) - return 1; - } - } else - b = f->last_pos / td->min_bs; - - *offset = (b * td->min_bs) + f->file_offset; - if (*offset > f->file_size) - return 1; - - return 0; -} - -static unsigned int get_next_buflen(struct thread_data *td) -{ - unsigned int buflen; - long r; - - if (td->min_bs == td->max_bs) - buflen = td->min_bs; - else { - r = os_random_long(&td->bsrange_state); - buflen = (1 + (double) (td->max_bs - 1) * r / (RAND_MAX + 1.0)); - buflen = (buflen + td->min_bs - 1) & ~(td->min_bs - 1); - } - - if (buflen > td->io_size - td->this_io_bytes[td->ddir]) { - /* - * if using direct/raw io, we may not be able to - * shrink the size. so just fail it. - */ - if (td->io_ops->flags & FIO_RAWIO) - return 0; - - buflen = td->io_size - td->this_io_bytes[td->ddir]; - } - - return buflen; -} - /* * Check if we are above the minimum rate given. */ @@ -238,8 +107,6 @@ static int check_min_rate(struct thread_data *td, struct timeval *now) rate = (td->this_io_bytes[ddir] - td->rate_bytes) / spent; if (rate < td->ratemin) { fprintf(f_out, "%s: min rate %d not met, got %ldKiB/sec\n", td->name, td->ratemin, rate); - if (rate_quit) - terminate_threads(td->groupid); return 1; } } @@ -259,162 +126,6 @@ static inline int runtime_exceeded(struct thread_data *td, struct timeval *t) return 0; } -/* - * Return the data direction for the next io_u. If the job is a - * mixed read/write workload, check the rwmix cycle and switch if - * necessary. - */ -static int get_rw_ddir(struct thread_data *td) -{ - if (td_rw(td)) { - struct timeval now; - unsigned long elapsed; - - gettimeofday(&now, NULL); - elapsed = mtime_since_now(&td->rwmix_switch); - - /* - * Check if it's time to seed a new data direction. - */ - if (elapsed >= td->rwmixcycle) { - unsigned int v; - long r; - - r = os_random_long(&td->rwmix_state); - v = 1 + (int) (100.0 * (r / (RAND_MAX + 1.0))); - if (v < td->rwmixread) - td->rwmix_ddir = DDIR_READ; - else - td->rwmix_ddir = DDIR_WRITE; - memcpy(&td->rwmix_switch, &now, sizeof(now)); - } - return td->rwmix_ddir; - } else if (td_read(td)) - return DDIR_READ; - else - return DDIR_WRITE; -} - -static int td_io_prep(struct thread_data *td, struct io_u *io_u) -{ - if (td->io_ops->prep && td->io_ops->prep(td, io_u)) - return 1; - - return 0; -} - -void put_io_u(struct thread_data *td, struct io_u *io_u) -{ - io_u->file = NULL; - list_del(&io_u->list); - list_add(&io_u->list, &td->io_u_freelist); - td->cur_depth--; -} - -static int fill_io_u(struct thread_data *td, struct fio_file *f, - struct io_u *io_u) -{ - /* - * If using an iolog, grab next piece if any available. - */ - if (td->read_iolog) - return read_iolog_get(td, io_u); - - /* - * No log, let the seq/rand engine retrieve the next position. - */ - if (!get_next_offset(td, f, &io_u->offset)) { - io_u->buflen = get_next_buflen(td); - - if (io_u->buflen) { - io_u->ddir = get_rw_ddir(td); - - /* - * If using a write iolog, store this entry. - */ - if (td->write_iolog) - write_iolog_put(td, io_u); - - io_u->file = f; - return 0; - } - } - - return 1; -} - -#define queue_full(td) list_empty(&(td)->io_u_freelist) - -struct io_u *__get_io_u(struct thread_data *td) -{ - struct io_u *io_u = NULL; - - if (!queue_full(td)) { - io_u = list_entry(td->io_u_freelist.next, struct io_u, list); - - io_u->error = 0; - io_u->resid = 0; - list_del(&io_u->list); - list_add(&io_u->list, &td->io_u_busylist); - td->cur_depth++; - } - - return io_u; -} - -/* - * Return an io_u to be processed. Gets a buflen and offset, sets direction, - * etc. The returned io_u is fully ready to be prepped and submitted. - */ -static struct io_u *get_io_u(struct thread_data *td, struct fio_file *f) -{ - struct io_u *io_u; - - io_u = __get_io_u(td); - if (!io_u) - return NULL; - - if (td->zone_bytes >= td->zone_size) { - td->zone_bytes = 0; - f->last_pos += td->zone_skip; - } - - if (fill_io_u(td, f, io_u)) { - put_io_u(td, io_u); - return NULL; - } - - if (io_u->buflen + io_u->offset > f->file_size) { - if (td->io_ops->flags & FIO_RAWIO) { - put_io_u(td, io_u); - return NULL; - } - - io_u->buflen = f->file_size - io_u->offset; - } - - if (!io_u->buflen) { - put_io_u(td, io_u); - return NULL; - } - - if (!td->read_iolog && !td->sequential) - mark_random_map(td, f, io_u); - - f->last_pos += io_u->buflen; - - if (td->verify != VERIFY_NONE) - populate_verify_io_u(td, io_u); - - if (td_io_prep(td, io_u)) { - put_io_u(td, io_u); - return NULL; - } - - gettimeofday(&io_u->start_time, NULL); - return io_u; -} - static inline void td_set_runstate(struct thread_data *td, int runstate) { td->runstate = runstate; @@ -441,75 +152,6 @@ static struct fio_file *get_next_file(struct thread_data *td) return f; } -static int td_io_sync(struct thread_data *td, struct fio_file *f) -{ - if (td->io_ops->sync) - return td->io_ops->sync(td, f); - - return 0; -} - -static int td_io_getevents(struct thread_data *td, int min, int max, - struct timespec *t) -{ - return td->io_ops->getevents(td, min, max, t); -} - -static int td_io_queue(struct thread_data *td, struct io_u *io_u) -{ - gettimeofday(&io_u->issue_time, NULL); - - return td->io_ops->queue(td, io_u); -} - -#define iocb_time(iocb) ((unsigned long) (iocb)->data) - -static void io_completed(struct thread_data *td, struct io_u *io_u, - struct io_completion_data *icd) -{ - struct timeval e; - unsigned long msec; - - gettimeofday(&e, NULL); - - if (!io_u->error) { - unsigned int bytes = io_u->buflen - io_u->resid; - const int idx = io_u->ddir; - - td->io_blocks[idx]++; - td->io_bytes[idx] += bytes; - td->zone_bytes += bytes; - td->this_io_bytes[idx] += bytes; - - msec = mtime_since(&io_u->issue_time, &e); - - add_clat_sample(td, idx, msec); - add_bw_sample(td, idx); - - if ((td_rw(td) || td_write(td)) && idx == DDIR_WRITE) - log_io_piece(td, io_u); - - icd->bytes_done[idx] += bytes; - } else - icd->error = io_u->error; -} - -static void ios_completed(struct thread_data *td,struct io_completion_data *icd) -{ - struct io_u *io_u; - int i; - - icd->error = 0; - icd->bytes_done[0] = icd->bytes_done[1] = 0; - - for (i = 0; i < icd->nr; i++) { - io_u = td->io_ops->event(td, i); - - io_completed(td, io_u, icd); - put_io_u(td, io_u); - } -} - /* * When job exits, we can cancel the in-flight IO if we are using async * io. Attempt to do so. @@ -728,7 +370,6 @@ static void do_io(struct thread_data *td) min_evts = 1; } - ret = td_io_getevents(td, min_evts, td->cur_depth, timeout); if (ret < 0) { td_verror(td, -ret); @@ -754,6 +395,8 @@ static void do_io(struct thread_data *td) rate_throttle(td, usec, icd.bytes_done[td->ddir]); if (check_min_rate(td, &e)) { + if (rate_quit) + terminate_threads(td->groupid); td_verror(td, ENOMEM); break; } @@ -781,14 +424,6 @@ static void do_io(struct thread_data *td) } } -static int td_io_init(struct thread_data *td) -{ - if (td->io_ops->init) - return td->io_ops->init(td); - - return 0; -} - static void cleanup_io_u(struct thread_data *td) { struct list_head *entry, *n; @@ -801,19 +436,7 @@ static void cleanup_io_u(struct thread_data *td) free(io_u); } - if (td->mem_type == MEM_MALLOC) - free(td->orig_buffer); - else if (td->mem_type == MEM_SHM) { - struct shmid_ds sbuf; - - shmdt(td->orig_buffer); - shmctl(td->shm_id, IPC_RMID, &sbuf); - } else if (td->mem_type == MEM_MMAP) - munmap(td->orig_buffer, td->orig_buffer_size); - else - log_err("Bad memory type %d\n", td->mem_type); - - td->orig_buffer = NULL; + free_io_mem(td); } static int init_io_u(struct thread_data *td) @@ -832,32 +455,8 @@ static int init_io_u(struct thread_data *td) td->orig_buffer_size = td->max_bs * max_units + MASK; - if (td->mem_type == MEM_MALLOC) - td->orig_buffer = malloc(td->orig_buffer_size); - else if (td->mem_type == MEM_SHM) { - td->shm_id = shmget(IPC_PRIVATE, td->orig_buffer_size, IPC_CREAT | 0600); - if (td->shm_id < 0) { - td_verror(td, errno); - perror("shmget"); - return 1; - } - - td->orig_buffer = shmat(td->shm_id, NULL, 0); - if (td->orig_buffer == (void *) -1) { - td_verror(td, errno); - perror("shmat"); - td->orig_buffer = NULL; - return 1; - } - } else if (td->mem_type == MEM_MMAP) { - td->orig_buffer = mmap(NULL, td->orig_buffer_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | OS_MAP_ANON, 0, 0); - if (td->orig_buffer == MAP_FAILED) { - td_verror(td, errno); - perror("mmap"); - td->orig_buffer = NULL; - return 1; - } - } + if (allocate_io_mem(td)) + return 1; p = ALIGN(td->orig_buffer); for (i = 0; i < max_units; i++) { @@ -1090,14 +689,14 @@ static void *fork_main(int shmid, int offset) */ static void reap_threads(int *nr_running, int *t_rate, int *m_rate) { + struct thread_data *td; int i, cputhreads; /* * reap exited threads (TD_EXITED -> TD_REAPED) */ - for (i = 0, cputhreads = 0; i < thread_number; i++) { - struct thread_data *td = &threads[i]; - + cputhreads = 0; + for_each_td(td, i) { /* * ->io_ops is NULL for a thread that has closed its * io engine @@ -1127,48 +726,6 @@ static void reap_threads(int *nr_running, int *t_rate, int *m_rate) terminate_threads(TERMINATE_ALL); } -static void fio_unpin_memory(void *pinned) -{ - if (pinned) { - if (munlock(pinned, mlock_size) < 0) - perror("munlock"); - munmap(pinned, mlock_size); - } -} - -static void *fio_pin_memory(void) -{ - unsigned long long phys_mem; - void *ptr; - - if (!mlock_size) - return NULL; - - /* - * Don't allow mlock of more than real_mem-128MB - */ - phys_mem = os_phys_mem(); - if (phys_mem) { - if ((mlock_size + 128 * 1024 * 1024) > phys_mem) { - mlock_size = phys_mem - 128 * 1024 * 1024; - fprintf(f_out, "fio: limiting mlocked memory to %lluMiB\n", mlock_size >> 20); - } - } - - ptr = mmap(NULL, mlock_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | OS_MAP_ANON, 0, 0); - if (!ptr) { - perror("malloc locked mem"); - return NULL; - } - if (mlock(ptr, mlock_size) < 0) { - munmap(ptr, mlock_size); - perror("mlock"); - return NULL; - } - - return ptr; -} - /* * Main function for kicking off and reaping jobs, as needed. */ @@ -1177,9 +734,9 @@ static void run_threads(void) struct thread_data *td; unsigned long spent; int i, todo, nr_running, m_rate, t_rate, nr_started; - void *mlocked_mem; - mlocked_mem = fio_pin_memory(); + if (fio_pin_memory()) + return; if (!terse_output) { printf("Starting %d thread%s\n", thread_number, thread_number > 1 ? "s" : ""); @@ -1194,9 +751,7 @@ static void run_threads(void) nr_started = 0; m_rate = t_rate = 0; - for (i = 0; i < thread_number; i++) { - td = &threads[i]; - + for_each_td(td, i) { print_status_init(td->thread_number - 1); init_disk_util(td); @@ -1225,9 +780,7 @@ static void run_threads(void) /* * create threads (TD_NOT_CREATED -> TD_CREATED) */ - for (i = 0; i < thread_number; i++) { - td = &threads[i]; - + for_each_td(td, i) { if (td->runstate != TD_NOT_CREATED) continue; @@ -1316,9 +869,7 @@ static void run_threads(void) /* * start created threads (TD_INITIALIZED -> TD_RUNNING). */ - for (i = 0; i < thread_number; i++) { - td = &threads[i]; - + for_each_td(td, i) { if (td->runstate != TD_INITIALIZED) continue; @@ -1343,7 +894,7 @@ static void run_threads(void) } update_io_ticks(); - fio_unpin_memory(mlocked_mem); + fio_unpin_memory(); } int main(int argc, char *argv[])