X-Git-Url: https://git.kernel.dk/?a=blobdiff_plain;f=fio.c;h=82a18fb89f4d05c8b8f2821e0cbaf82b677dc75b;hb=ef899b6393c43abd655ad2cbc22de18b88a101f8;hp=3c35039a6a4788e25bc1abc4165630cb0335769e;hpb=fc1a471326fb5248a74ed3a206d37884b812c853;p=fio.git diff --git a/fio.c b/fio.c index 3c35039a..82a18fb8 100644 --- a/fio.c +++ b/fio.c @@ -57,12 +57,15 @@ static void update_io_ticks(void); static void disk_util_timer_arm(void); static void print_thread_status(void); +extern unsigned long long mlock_size; + /* * thread life cycle */ enum { TD_NOT_CREATED = 0, TD_CREATED, + TD_INITIALIZED, TD_RUNNING, TD_VERIFYING, TD_EXITED, @@ -74,6 +77,7 @@ enum { static sem_t startup_sem; #define TERMINATE_ALL (-1) +#define JOB_START_TIMEOUT (5 * 1000) static void terminate_threads(int group_id) { @@ -193,7 +197,7 @@ static int get_next_free_block(struct thread_data *td, unsigned long long *b) static void mark_random_map(struct thread_data *td, struct io_u *io_u) { - unsigned long block = io_u->offset / td->min_bs; + unsigned long long block = io_u->offset / (unsigned long long) td->min_bs; unsigned int blocks = 0; while (blocks < (io_u->buflen / td->min_bs)) { @@ -284,12 +288,12 @@ static int get_next_offset(struct thread_data *td, unsigned long long *offset) long r; if (!td->sequential) { - unsigned long max_blocks = td->io_size / td->min_bs; + unsigned long long max_blocks = td->io_size / td->min_bs; int loops = 50; do { lrand48_r(&td->random_state, &r); - b = ((max_blocks - 1) * r / (RAND_MAX+1.0)); + b = ((max_blocks - 1) * r / (unsigned long long) (RAND_MAX+1.0)); rb = b + (td->file_offset / td->min_bs); loops--; } while (!random_map_free(td, rb) && loops); @@ -541,44 +545,32 @@ static void fill_md5(struct verify_header *hdr, void *p, unsigned int len) memcpy(hdr->md5_digest, md5_ctx.hash, sizeof(md5_ctx.hash)); } -unsigned int hweight32(unsigned int w) -{ - unsigned int res = w - ((w >> 1) & 0x55555555); - - res = (res & 0x33333333) + ((res >> 2) & 0x33333333); - res = (res + (res >> 4)) & 0x0F0F0F0F; - res = res + (res >> 8); - - return (res + (res >> 16)) & 0x000000FF; -} - -unsigned long hweight64(unsigned long long w) +static int get_rw_ddir(struct thread_data *td) { -#if __WORDSIZE == 32 - return hweight32((unsigned int)(w >> 32)) + hweight32((unsigned int)w); -#elif __WORDSIZE == 64 - unsigned long long v = w - ((w >> 1) & 0x5555555555555555ul); + if (td_rw(td)) { + struct timeval now; + unsigned long elapsed; - v = (v & 0x3333333333333333ul) + ((v >> 2) & 0x3333333333333333ul); - v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0Ful; - v = v + (v >> 8); - v = v + (v >> 16); + gettimeofday(&now, NULL); + elapsed = mtime_since_now(&td->rwmix_switch); - return (v + (v >> 32)) & 0x00000000000000FFul; -#else -#error __WORDSIZE not defined -#endif -} + /* + * Check if it's time to seed a new data direction. + */ + if (elapsed >= td->rwmixcycle) { + unsigned long v; + long r; -static int get_rw_ddir(struct thread_data *td) -{ - /* - * perhaps cheasy, but use the hamming weight of the position - * as a randomizer for data direction. - */ - if (td_rw(td)) - return hweight64(td->last_pos) & 1; - else if (td_read(td)) + lrand48_r(&td->random_state, &r); + v = 100UL * r / (unsigned long) (RAND_MAX + 1.0); + if (v < td->rwmixread) + td->rwmix_ddir = DDIR_READ; + else + td->rwmix_ddir = DDIR_WRITE; + memcpy(&td->rwmix_switch, &now, sizeof(now)); + } + return td->rwmix_ddir; + } else if (td_read(td)) return DDIR_READ; else return DDIR_WRITE; @@ -624,17 +616,16 @@ void put_io_u(struct thread_data *td, struct io_u *io_u) td->cur_depth--; } -static int fill_io_u(struct thread_data *td, struct io_u *io_u) +static void write_iolog_put(struct thread_data *td, struct io_u *io_u) { - /* - * If using an iolog, grab next piece if any available. - */ - if (td->iolog) { - struct io_piece *ipo; + fprintf(td->iolog_f, "%d,%llu,%u\n", io_u->ddir, io_u->offset, io_u->buflen); +} - if (list_empty(&td->io_log_list)) - return 1; +static int read_iolog_get(struct thread_data *td, struct io_u *io_u) +{ + struct io_piece *ipo; + if (!list_empty(&td->io_log_list)) { ipo = list_entry(td->io_log_list.next, struct io_piece, list); list_del(&ipo->list); io_u->offset = ipo->offset; @@ -644,6 +635,17 @@ static int fill_io_u(struct thread_data *td, struct io_u *io_u) return 0; } + return 1; +} + +static int fill_io_u(struct thread_data *td, struct io_u *io_u) +{ + /* + * If using an iolog, grab next piece if any available. + */ + if (td->read_iolog) + return read_iolog_get(td, io_u); + /* * No log, let the seq/rand engine retrieve the next position. */ @@ -652,6 +654,13 @@ static int fill_io_u(struct thread_data *td, struct io_u *io_u) if (io_u->buflen) { io_u->ddir = get_rw_ddir(td); + + /* + * If using a write iolog, store this entry. + */ + if (td->write_iolog) + write_iolog_put(td, io_u); + return 0; } } @@ -703,7 +712,7 @@ static struct io_u *get_io_u(struct thread_data *td) return NULL; } - if (!td->iolog && !td->sequential) + if (!td->read_iolog && !td->sequential) mark_random_map(td, io_u); td->last_pos += io_u->buflen; @@ -791,6 +800,13 @@ static void log_io_piece(struct thread_data *td, struct io_u *io_u) list_add(&ipo->list, entry); } +static void write_iolog_close(struct thread_data *td) +{ + fflush(td->iolog_f); + fclose(td->iolog_f); + free(td->iolog_buf); +} + static int init_iolog(struct thread_data *td) { unsigned long long offset; @@ -799,15 +815,34 @@ static int init_iolog(struct thread_data *td) FILE *f; int rw, i, reads, writes; - if (!td->iolog) + if (!td->read_iolog && !td->write_iolog) return 0; - f = fopen(td->iolog_file, "r"); + if (td->read_iolog) + f = fopen(td->iolog_file, "r"); + else + f = fopen(td->iolog_file, "w"); + if (!f) { perror("fopen iolog"); + printf("file %s, %d/%d\n", td->iolog_file, td->read_iolog, td->write_iolog); return 1; } + /* + * That's it for writing, setup a log buffer and we're done. + */ + if (td->write_iolog) { + td->iolog_f = f; + td->iolog_buf = malloc(8192); + setvbuf(f, td->iolog_buf, _IOFBF, 8192); + return 0; + } + + /* + * Read in the read iolog and store it, reuse the infrastructure + * for doing verifications. + */ str = malloc(4096); reads = writes = i = 0; while ((p = fgets(str, 4096, f)) != NULL) { @@ -1737,23 +1772,19 @@ static void init_disk_util(struct thread_data *td) return; /* - * for md/dm, there's no queue dir. we already have the right place + * If there's a ../queue/ directory there, we are inside a partition. + * Check if that is the case and jump back. For loop/md/dm etc we + * are already in the right spot. */ - sprintf(tmp, "%s/stat", foo); - if (stat(tmp, &st)) { - /* - * if this is inside a partition dir, jump back to parent - */ - sprintf(tmp, "%s/queue", foo); + sprintf(tmp, "%s/../queue", foo); + if (!stat(tmp, &st)) { + p = dirname(foo); + sprintf(tmp, "%s/queue", p); if (stat(tmp, &st)) { - p = dirname(foo); - sprintf(tmp, "%s/queue", p); - if (stat(tmp, &st)) { - fprintf(stderr, "unknown sysfs layout\n"); - return; - } - sprintf(foo, "%s", p); + fprintf(stderr, "unknown sysfs layout\n"); + return; } + sprintf(foo, "%s", p); } disk_util_add(dev, foo); @@ -1798,7 +1829,6 @@ static void update_rusage_stat(struct thread_data *td) static void *thread_main(void *data) { struct thread_data *td = data; - int ret = 1; if (!td->use_thread) setsid(); @@ -1831,15 +1861,21 @@ static void *thread_main(void *data) } } + if (nice(td->nice) < 0) { + td_verror(td, errno); + goto err; + } + + if (init_random_state(td)) + goto err; + + td_set_runstate(td, TD_INITIALIZED); sem_post(&startup_sem); sem_wait(&td->mutex); if (!td->create_serialize && setup_file(td)) goto err; - if (init_random_state(td)) - goto err; - gettimeofday(&td->epoch, NULL); while (td->loops--) { @@ -1878,14 +1914,14 @@ static void *thread_main(void *data) break; } - ret = 0; - if (td->bw_log) finish_log(td, td->bw_log, "bw"); if (td->slat_log) finish_log(td, td->slat_log, "slat"); if (td->clat_log) finish_log(td, td->clat_log, "clat"); + if (td->write_iolog) + write_iolog_close(td); if (exitall_on_terminate) terminate_threads(td->groupid); @@ -1897,12 +1933,12 @@ err: } if (td->mmap) munmap(td->mmap, td->file_size); + if (td->directory) + free(td->directory); + if (td->iolog_file) + free(td->iolog_file); cleanup_io(td); cleanup_io_u(td); - if (ret) { - sem_post(&startup_sem); - sem_wait(&td->mutex); - } td_set_runstate(td, TD_EXITED); return NULL; @@ -1949,14 +1985,15 @@ static void show_ddir_status(struct thread_data *td, struct group_run_stats *rs, int ddir) { char *ddir_str[] = { "read ", "write" }; - unsigned long min, max, bw; + unsigned long min, max; + unsigned long long bw; double mean, dev; if (!td->runtime[ddir]) return; bw = td->io_bytes[ddir] / td->runtime[ddir]; - printf(" %s: io=%6lluMiB, bw=%6luKiB/s, runt=%6lumsec\n", ddir_str[ddir], td->io_bytes[ddir] >> 20, bw, td->runtime[ddir]); + printf(" %s: io=%6lluMiB, bw=%6lluKiB/s, runt=%6lumsec\n", ddir_str[ddir], td->io_bytes[ddir] >> 20, bw, td->runtime[ddir]); if (calc_lat(&td->slat_stat[ddir], &min, &max, &mean, &dev)) printf(" slat (msec): min=%5lu, max=%5lu, avg=%5.02f, dev=%5.02f\n", min, max, mean, dev); @@ -2037,6 +2074,9 @@ static void check_str_update(struct thread_data *td) case TD_CREATED: c = 'C'; break; + case TD_INITIALIZED: + c = 'I'; + break; case TD_NOT_CREATED: c = 'P'; break; @@ -2108,7 +2148,8 @@ static int thread_eta(struct thread_data *td, unsigned long elapsed) if (td->timeout && eta_sec > (td->timeout - elapsed)) eta_sec = td->timeout - elapsed; - } else if (td->runstate == TD_NOT_CREATED || td->runstate == TD_CREATED) { + } else if (td->runstate == TD_NOT_CREATED || td->runstate == TD_CREATED + || td->runstate == TD_INITIALIZED) { int t_eta = 0, r_eta = 0; /* @@ -2229,11 +2270,60 @@ static void reap_threads(int *nr_running, int *t_rate, int *m_rate) } } +static void fio_unpin_memory(void *pinned) +{ + if (pinned) { + if (munlock(pinned, mlock_size) < 0) + perror("munlock"); + munmap(pinned, mlock_size); + } +} + +static void *fio_pin_memory(void) +{ + long pagesize, pages; + void *ptr; + + if (!mlock_size) + return NULL; + + /* + * Don't allow mlock of more than real_mem-128MB + */ + pagesize = sysconf(_SC_PAGESIZE); + pages = sysconf(_SC_PHYS_PAGES); + if (pages != -1 && pagesize != -1) { + unsigned long long real_mem = pages * pagesize; + + if ((mlock_size + 128 * 1024 * 1024) > real_mem) { + mlock_size = real_mem - 128 * 1024 * 1024; + printf("fio: limiting mlocked memory to %lluMiB\n", + mlock_size >> 20); + } + } + + ptr = mmap(NULL, mlock_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | OS_MAP_ANON, 0, 0); + if (!ptr) { + perror("malloc locked mem"); + return NULL; + } + if (mlock(ptr, mlock_size) < 0) { + munmap(ptr, mlock_size); + perror("mlock"); + return NULL; + } + + return ptr; +} + static void run_threads(void) { struct thread_data *td; unsigned long spent; int i, todo, nr_running, m_rate, t_rate, nr_started; + void *mlocked_mem; + + mlocked_mem = fio_pin_memory(); printf("Starting %d thread%s\n", thread_number, thread_number > 1 ? "s" : ""); fflush(stdout); @@ -2270,6 +2360,10 @@ static void run_threads(void) gettimeofday(&genesis, NULL); while (todo) { + struct thread_data *map[MAX_JOBS]; + struct timeval this_start; + int this_jobs = 0, left; + /* * create threads (TD_NOT_CREATED -> TD_CREATED) */ @@ -2298,9 +2392,13 @@ static void run_threads(void) if (td->stonewall && (nr_started || nr_running)) break; + /* + * Set state to created. Thread will transition + * to TD_INITIALIZED when it's done setting up. + */ td_set_runstate(td, TD_CREATED); + map[this_jobs++] = td; sem_init(&startup_sem, 0, 1); - todo--; nr_started++; if (td->use_thread) { @@ -2319,12 +2417,53 @@ static void run_threads(void) } /* - * start created threads (TD_CREATED -> TD_RUNNING) + * Wait for the started threads to transition to + * TD_INITIALIZED. + */ + printf("fio: Waiting for threads to initialize...\n"); + gettimeofday(&this_start, NULL); + left = this_jobs; + while (left) { + if (mtime_since_now(&this_start) > JOB_START_TIMEOUT) + break; + + usleep(100000); + + for (i = 0; i < this_jobs; i++) { + td = map[i]; + if (!td) + continue; + if (td->runstate == TD_INITIALIZED) { + map[i] = NULL; + left--; + } else if (td->runstate >= TD_EXITED) { + map[i] = NULL; + left--; + todo--; + nr_running++; /* work-around... */ + } + } + } + + if (left) { + fprintf(stderr, "fio: %d jobs failed to start\n", left); + for (i = 0; i < this_jobs; i++) { + td = map[i]; + if (!td) + continue; + kill(td->pid, SIGTERM); + } + break; + } + + /* + * start created threads (TD_INITIALIZED -> TD_RUNNING). */ + printf("fio: Go for launch\n"); for (i = 0; i < thread_number; i++) { td = &threads[i]; - if (td->runstate != TD_CREATED) + if (td->runstate != TD_INITIALIZED) continue; td_set_runstate(td, TD_RUNNING); @@ -2332,6 +2471,7 @@ static void run_threads(void) nr_started--; m_rate += td->ratemin; t_rate += td->rate; + todo--; sem_post(&td->mutex); } @@ -2347,6 +2487,7 @@ static void run_threads(void) } update_io_ticks(); + fio_unpin_memory(mlocked_mem); } static void show_group_stats(struct group_run_stats *rs, int id) @@ -2397,7 +2538,7 @@ static void show_run_stats(void) } for (i = 0; i < thread_number; i++) { - unsigned long rbw, wbw; + unsigned long long rbw, wbw; td = &threads[i]; @@ -2419,9 +2560,9 @@ static void show_run_stats(void) rbw = wbw = 0; if (td->runtime[0]) - rbw = td->io_bytes[0] / td->runtime[0]; + rbw = td->io_bytes[0] / (unsigned long long) td->runtime[0]; if (td->runtime[1]) - wbw = td->io_bytes[1] / td->runtime[1]; + wbw = td->io_bytes[1] / (unsigned long long) td->runtime[1]; if (rbw < rs->min_bw[0]) rs->min_bw[0] = rbw;