int groupid = 0;
int thread_number = 0;
-static char run_str[MAX_JOBS + 1];
int shm_id = 0;
-static struct timeval genesis;
int temp_stall_ts;
char *fio_inst_prefix = _INST_PREFIX;
-static void print_thread_status(void);
-
-extern unsigned long long mlock_size;
-
-/*
- * Thread life cycle. Once a thread has a runstate beyond TD_INITIALIZED, it
- * will never back again. It may cycle between running/verififying/fsyncing.
- * Once the thread reaches TD_EXITED, it is just waiting for the core to
- * reap it.
- */
-enum {
- TD_NOT_CREATED = 0,
- TD_CREATED,
- TD_INITIALIZED,
- TD_RUNNING,
- TD_VERIFYING,
- TD_FSYNCING,
- TD_EXITED,
- TD_REAPED,
-};
-
#define should_fsync(td) ((td_write(td) || td_rw(td)) && (!(td)->odirect || (td)->override_sync))
static volatile int startup_sem;
static void terminate_threads(int group_id)
{
+ struct thread_data *td;
int i;
- for (i = 0; i < thread_number; i++) {
- struct thread_data *td = &threads[i];
-
+ for_each_td(td, i) {
if (group_id == TERMINATE_ALL || groupid == td->groupid) {
td->terminate = 1;
td->start_delay = 0;
rate = (td->this_io_bytes[ddir] - td->rate_bytes) / spent;
if (rate < td->ratemin) {
fprintf(f_out, "%s: min rate %d not met, got %ldKiB/sec\n", td->name, td->ratemin, rate);
- if (rate_quit)
- terminate_threads(td->groupid);
return 1;
}
}
return 0;
}
-static void fill_random_bytes(struct thread_data *td,
- unsigned char *p, unsigned int len)
-{
- unsigned int todo;
- double r;
-
- while (len) {
- r = os_random_double(&td->verify_state);
-
- /*
- * lrand48_r seems to be broken and only fill the bottom
- * 32-bits, even on 64-bit archs with 64-bit longs
- */
- todo = sizeof(r);
- if (todo > len)
- todo = len;
-
- memcpy(p, &r, todo);
-
- len -= todo;
- p += todo;
- }
-}
-
-static void hexdump(void *buffer, int len)
-{
- unsigned char *p = buffer;
- int i;
-
- for (i = 0; i < len; i++)
- fprintf(f_out, "%02x", p[i]);
- fprintf(f_out, "\n");
-}
-
-static int verify_io_u_crc32(struct verify_header *hdr, struct io_u *io_u)
-{
- unsigned char *p = (unsigned char *) io_u->buf;
- unsigned long c;
-
- p += sizeof(*hdr);
- c = crc32(p, hdr->len - sizeof(*hdr));
-
- if (c != hdr->crc32) {
- log_err("crc32: verify failed at %llu/%u\n", io_u->offset, io_u->buflen);
- log_err("crc32: wanted %lx, got %lx\n", hdr->crc32, c);
- return 1;
- }
-
- return 0;
-}
-
-static int verify_io_u_md5(struct verify_header *hdr, struct io_u *io_u)
-{
- unsigned char *p = (unsigned char *) io_u->buf;
- struct md5_ctx md5_ctx;
-
- memset(&md5_ctx, 0, sizeof(md5_ctx));
- p += sizeof(*hdr);
- md5_update(&md5_ctx, p, hdr->len - sizeof(*hdr));
-
- if (memcmp(hdr->md5_digest, md5_ctx.hash, sizeof(md5_ctx.hash))) {
- log_err("md5: verify failed at %llu/%u\n", io_u->offset, io_u->buflen);
- hexdump(hdr->md5_digest, sizeof(hdr->md5_digest));
- hexdump(md5_ctx.hash, sizeof(md5_ctx.hash));
- return 1;
- }
-
- return 0;
-}
-
-static int verify_io_u(struct io_u *io_u)
-{
- struct verify_header *hdr = (struct verify_header *) io_u->buf;
- int ret;
-
- if (hdr->fio_magic != FIO_HDR_MAGIC)
- return 1;
-
- if (hdr->verify_type == VERIFY_MD5)
- ret = verify_io_u_md5(hdr, io_u);
- else if (hdr->verify_type == VERIFY_CRC32)
- ret = verify_io_u_crc32(hdr, io_u);
- else {
- log_err("Bad verify type %d\n", hdr->verify_type);
- ret = 1;
- }
-
- return ret;
-}
-
-static void fill_crc32(struct verify_header *hdr, void *p, unsigned int len)
-{
- hdr->crc32 = crc32(p, len);
-}
-
-static void fill_md5(struct verify_header *hdr, void *p, unsigned int len)
-{
- struct md5_ctx md5_ctx;
-
- memset(&md5_ctx, 0, sizeof(md5_ctx));
- md5_update(&md5_ctx, p, len);
- memcpy(hdr->md5_digest, md5_ctx.hash, sizeof(md5_ctx.hash));
-}
-
/*
* Return the data direction for the next io_u. If the job is a
* mixed read/write workload, check the rwmix cycle and switch if
* Check if it's time to seed a new data direction.
*/
if (elapsed >= td->rwmixcycle) {
- int v;
+ unsigned int v;
long r;
r = os_random_long(&td->rwmix_state);
return DDIR_WRITE;
}
-/*
- * fill body of io_u->buf with random data and add a header with the
- * crc32 or md5 sum of that data.
- */
-static void populate_io_u(struct thread_data *td, struct io_u *io_u)
-{
- unsigned char *p = (unsigned char *) io_u->buf;
- struct verify_header hdr;
-
- hdr.fio_magic = FIO_HDR_MAGIC;
- hdr.len = io_u->buflen;
- p += sizeof(hdr);
- fill_random_bytes(td, p, io_u->buflen - sizeof(hdr));
-
- if (td->verify == VERIFY_MD5) {
- fill_md5(&hdr, p, io_u->buflen - sizeof(hdr));
- hdr.verify_type = VERIFY_MD5;
- } else {
- fill_crc32(&hdr, p, io_u->buflen - sizeof(hdr));
- hdr.verify_type = VERIFY_CRC32;
- }
-
- memcpy(io_u->buf, &hdr, sizeof(hdr));
-}
-
static int td_io_prep(struct thread_data *td, struct io_u *io_u)
{
if (td->io_ops->prep && td->io_ops->prep(td, io_u))
f->last_pos += io_u->buflen;
if (td->verify != VERIFY_NONE)
- populate_io_u(td, io_u);
+ populate_verify_io_u(td, io_u);
if (td_io_prep(td, io_u)) {
put_io_u(td, io_u);
td->runstate = runstate;
}
-static int get_next_verify(struct thread_data *td, struct io_u *io_u)
-{
- struct io_piece *ipo;
-
- if (!list_empty(&td->io_hist_list)) {
- ipo = list_entry(td->io_hist_list.next, struct io_piece, list);
-
- list_del(&ipo->list);
-
- io_u->offset = ipo->offset;
- io_u->buflen = ipo->len;
- io_u->ddir = DDIR_READ;
- free(ipo);
- return 0;
- }
-
- return 1;
-}
-
static struct fio_file *get_next_file(struct thread_data *td)
{
- int old_next_file = td->next_file;
+ unsigned int old_next_file = td->next_file;
struct fio_file *f;
do {
return 0;
}
-static int io_u_getevents(struct thread_data *td, int min, int max,
+static int td_io_getevents(struct thread_data *td, int min, int max,
struct timespec *t)
{
return td->io_ops->getevents(td, min, max, t);
}
-static int io_u_queue(struct thread_data *td, struct io_u *io_u)
+static int td_io_queue(struct thread_data *td, struct io_u *io_u)
{
gettimeofday(&io_u->issue_time, NULL);
/*
* get immediately available events, if any
*/
- r = io_u_getevents(td, 0, td->cur_depth, &ts);
+ r = td_io_getevents(td, 0, td->cur_depth, &ts);
if (r > 0) {
icd.nr = r;
ios_completed(td, &icd);
}
if (td->cur_depth) {
- r = io_u_getevents(td, td->cur_depth, td->cur_depth, NULL);
+ r = td_io_getevents(td, td->cur_depth, td->cur_depth, NULL);
if (r > 0) {
icd.nr = r;
ios_completed(td, &icd);
}
}
-static int do_io_u_verify(struct thread_data *td, struct io_u **io_u)
-{
- struct io_u *v_io_u = *io_u;
- int ret = 0;
-
- if (v_io_u) {
- ret = verify_io_u(v_io_u);
- put_io_u(td, v_io_u);
- *io_u = NULL;
- }
-
- return ret;
-}
-
/*
* The main verify engine. Runs over the writes we previusly submitted,
* reads the blocks back in, and checks the crc/md5 of the data.
*/
-static void do_verify(struct thread_data *td)
+void do_verify(struct thread_data *td)
{
struct timeval t;
struct io_u *io_u, *v_io_u = NULL;
struct io_completion_data icd;
struct fio_file *f;
- int ret;
+ int ret, i;
+
+ /*
+ * sync io first and invalidate cache, to make sure we really
+ * read from disk.
+ */
+ for_each_file(td, f, i) {
+ td_io_sync(td, f);
+ file_invalidate_cache(td, f);
+ }
td_set_runstate(td, TD_VERIFYING);
break;
}
- ret = io_u_queue(td, io_u);
+ ret = td_io_queue(td, io_u);
if (ret) {
put_io_u(td, io_u);
td_verror(td, ret);
if (do_io_u_verify(td, &v_io_u))
break;
- ret = io_u_getevents(td, 1, 1, NULL);
+ ret = td_io_getevents(td, 1, 1, NULL);
if (ret != 1) {
if (ret < 0)
td_verror(td, ret);
struct timeval s, e;
unsigned long usec;
struct fio_file *f;
- int i;
+ int i, ret = 0;
td_set_runstate(td, TD_RUNNING);
while (td->this_io_bytes[td->ddir] < td->io_size) {
struct timespec ts = { .tv_sec = 0, .tv_nsec = 0};
struct timespec *timeout;
- int ret, min_evts = 0;
+ int min_evts = 0;
struct io_u *io_u;
if (td->terminate)
memcpy(&s, &io_u->start_time, sizeof(s));
- ret = io_u_queue(td, io_u);
+ ret = td_io_queue(td, io_u);
if (ret) {
put_io_u(td, io_u);
td_verror(td, ret);
min_evts = 1;
}
- ret = io_u_getevents(td, min_evts, td->cur_depth, timeout);
+
+ ret = td_io_getevents(td, min_evts, td->cur_depth, timeout);
if (ret < 0) {
- td_verror(td, ret);
+ td_verror(td, -ret);
break;
} else if (!ret)
continue;
rate_throttle(td, usec, icd.bytes_done[td->ddir]);
if (check_min_rate(td, &e)) {
+ if (rate_quit)
+ terminate_threads(td->groupid);
td_verror(td, ENOMEM);
break;
}
td_io_sync(td, f);
}
- if (td->cur_depth)
- cleanup_pending_aio(td);
+ if (!ret) {
+ if (td->cur_depth)
+ cleanup_pending_aio(td);
- if (should_fsync(td) && td->end_fsync) {
- td_set_runstate(td, TD_FSYNCING);
- for_each_file(td, f, i)
- td_io_sync(td, f);
+ if (should_fsync(td) && td->end_fsync) {
+ td_set_runstate(td, TD_FSYNCING);
+ for_each_file(td, f, i)
+ td_io_sync(td, f);
+ }
}
}
-static int init_io(struct thread_data *td)
+static int td_io_init(struct thread_data *td)
{
if (td->io_ops->init)
return td->io_ops->init(td);
free(io_u);
}
- if (td->mem_type == MEM_MALLOC)
- free(td->orig_buffer);
- else if (td->mem_type == MEM_SHM) {
- struct shmid_ds sbuf;
-
- shmdt(td->orig_buffer);
- shmctl(td->shm_id, IPC_RMID, &sbuf);
- } else if (td->mem_type == MEM_MMAP)
- munmap(td->orig_buffer, td->orig_buffer_size);
- else
- log_err("Bad memory type %d\n", td->mem_type);
-
- td->orig_buffer = NULL;
+ free_io_mem(td);
}
static int init_io_u(struct thread_data *td)
td->orig_buffer_size = td->max_bs * max_units + MASK;
- if (td->mem_type == MEM_MALLOC)
- td->orig_buffer = malloc(td->orig_buffer_size);
- else if (td->mem_type == MEM_SHM) {
- td->shm_id = shmget(IPC_PRIVATE, td->orig_buffer_size, IPC_CREAT | 0600);
- if (td->shm_id < 0) {
- td_verror(td, errno);
- perror("shmget");
- return 1;
- }
-
- td->orig_buffer = shmat(td->shm_id, NULL, 0);
- if (td->orig_buffer == (void *) -1) {
- td_verror(td, errno);
- perror("shmat");
- td->orig_buffer = NULL;
- return 1;
- }
- } else if (td->mem_type == MEM_MMAP) {
- td->orig_buffer = mmap(NULL, td->orig_buffer_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | OS_MAP_ANON, 0, 0);
- if (td->orig_buffer == MAP_FAILED) {
- td_verror(td, errno);
- perror("mmap");
- td->orig_buffer = NULL;
- return 1;
- }
- }
+ if (allocate_io_mem(td))
+ return 1;
p = ALIGN(td->orig_buffer);
for (i = 0; i < max_units; i++) {
goto err;
}
- if (init_io(td))
+ if (td_io_init(td))
goto err;
if (init_iolog(td))
return NULL;
}
-/*
- * Sets the status of the 'td' in the printed status map.
- */
-static void check_str_update(struct thread_data *td)
-{
- char c = run_str[td->thread_number - 1];
-
- switch (td->runstate) {
- case TD_REAPED:
- c = '_';
- break;
- case TD_EXITED:
- c = 'E';
- break;
- case TD_RUNNING:
- if (td_rw(td)) {
- if (td->sequential)
- c = 'M';
- else
- c = 'm';
- } else if (td_read(td)) {
- if (td->sequential)
- c = 'R';
- else
- c = 'r';
- } else {
- if (td->sequential)
- c = 'W';
- else
- c = 'w';
- }
- break;
- case TD_VERIFYING:
- c = 'V';
- break;
- case TD_FSYNCING:
- c = 'F';
- break;
- case TD_CREATED:
- c = 'C';
- break;
- case TD_INITIALIZED:
- c = 'I';
- break;
- case TD_NOT_CREATED:
- c = 'P';
- break;
- default:
- log_err("state %d\n", td->runstate);
- }
-
- run_str[td->thread_number - 1] = c;
-}
-
-/*
- * Convert seconds to a printable string.
- */
-static void eta_to_str(char *str, int eta_sec)
-{
- unsigned int d, h, m, s;
- static int always_d, always_h;
-
- d = h = m = s = 0;
-
- s = eta_sec % 60;
- eta_sec /= 60;
- m = eta_sec % 60;
- eta_sec /= 60;
- h = eta_sec % 24;
- eta_sec /= 24;
- d = eta_sec;
-
- if (d || always_d) {
- always_d = 1;
- str += sprintf(str, "%02dd:", d);
- }
- if (h || always_h) {
- always_h = 1;
- str += sprintf(str, "%02dh:", h);
- }
-
- str += sprintf(str, "%02dm:", m);
- str += sprintf(str, "%02ds", s);
-}
-
-/*
- * Best effort calculation of the estimated pending runtime of a job.
- */
-static int thread_eta(struct thread_data *td, unsigned long elapsed)
-{
- unsigned long long bytes_total, bytes_done;
- unsigned int eta_sec = 0;
-
- bytes_total = td->total_io_size;
-
- if (td->zone_size && td->zone_skip)
- bytes_total /= (td->zone_skip / td->zone_size);
-
- if (td->runstate == TD_RUNNING || td->runstate == TD_VERIFYING) {
- double perc;
-
- bytes_done = td->io_bytes[DDIR_READ] + td->io_bytes[DDIR_WRITE];
- perc = (double) bytes_done / (double) bytes_total;
- if (perc > 1.0)
- perc = 1.0;
-
- eta_sec = (elapsed * (1.0 / perc)) - elapsed;
-
- if (td->timeout && eta_sec > (td->timeout - elapsed))
- eta_sec = td->timeout - elapsed;
- } else if (td->runstate == TD_NOT_CREATED || td->runstate == TD_CREATED
- || td->runstate == TD_INITIALIZED) {
- int t_eta = 0, r_eta = 0;
-
- /*
- * We can only guess - assume it'll run the full timeout
- * if given, otherwise assume it'll run at the specified rate.
- */
- if (td->timeout)
- t_eta = td->timeout + td->start_delay - elapsed;
- if (td->rate) {
- r_eta = (bytes_total / 1024) / td->rate;
- r_eta += td->start_delay - elapsed;
- }
-
- if (r_eta && t_eta)
- eta_sec = min(r_eta, t_eta);
- else if (r_eta)
- eta_sec = r_eta;
- else if (t_eta)
- eta_sec = t_eta;
- else
- eta_sec = 0;
- } else {
- /*
- * thread is already done or waiting for fsync
- */
- eta_sec = 0;
- }
-
- return eta_sec;
-}
-
-/*
- * Print status of the jobs we know about. This includes rate estimates,
- * ETA, thread state, etc.
- */
-static void print_thread_status(void)
-{
- unsigned long elapsed = time_since_now(&genesis);
- int i, nr_running, nr_pending, t_rate, m_rate, *eta_secs, eta_sec;
- char eta_str[32];
- double perc = 0.0;
-
- if (temp_stall_ts || terse_output)
- return;
-
- eta_secs = malloc(thread_number * sizeof(int));
- memset(eta_secs, 0, thread_number * sizeof(int));
-
- nr_pending = nr_running = t_rate = m_rate = 0;
- for (i = 0; i < thread_number; i++) {
- struct thread_data *td = &threads[i];
-
- if (td->runstate == TD_RUNNING || td->runstate == TD_VERIFYING||
- td->runstate == TD_FSYNCING) {
- nr_running++;
- t_rate += td->rate;
- m_rate += td->ratemin;
- } else if (td->runstate < TD_RUNNING)
- nr_pending++;
-
- if (elapsed >= 3)
- eta_secs[i] = thread_eta(td, elapsed);
- else
- eta_secs[i] = INT_MAX;
-
- check_str_update(td);
- }
-
- if (exitall_on_terminate)
- eta_sec = INT_MAX;
- else
- eta_sec = 0;
-
- for (i = 0; i < thread_number; i++) {
- if (exitall_on_terminate) {
- if (eta_secs[i] < eta_sec)
- eta_sec = eta_secs[i];
- } else {
- if (eta_secs[i] > eta_sec)
- eta_sec = eta_secs[i];
- }
- }
-
- if (eta_sec != INT_MAX && elapsed) {
- perc = (double) elapsed / (double) (elapsed + eta_sec);
- eta_to_str(eta_str, eta_sec);
- }
-
- if (!nr_running && !nr_pending)
- return;
-
- printf("Threads running: %d", nr_running);
- if (m_rate || t_rate)
- printf(", commitrate %d/%dKiB/sec", t_rate, m_rate);
- if (eta_sec != INT_MAX && nr_running) {
- perc *= 100.0;
- printf(": [%s] [%3.2f%% done] [eta %s]", run_str, perc,eta_str);
- }
- printf("\r");
- fflush(stdout);
- free(eta_secs);
-}
-
/*
* Run over the job map and reap the threads that have exited, if any.
*/
static void reap_threads(int *nr_running, int *t_rate, int *m_rate)
{
+ struct thread_data *td;
int i, cputhreads;
/*
* reap exited threads (TD_EXITED -> TD_REAPED)
*/
- for (i = 0, cputhreads = 0; i < thread_number; i++) {
- struct thread_data *td = &threads[i];
-
- if (td->io_ops->flags & FIO_CPUIO)
+ cputhreads = 0;
+ for_each_td(td, i) {
+ /*
+ * ->io_ops is NULL for a thread that has closed its
+ * io engine
+ */
+ if (td->io_ops && td->io_ops->flags & FIO_CPUIO)
cputhreads++;
if (td->runstate != TD_EXITED)
terminate_threads(TERMINATE_ALL);
}
-static void fio_unpin_memory(void *pinned)
-{
- if (pinned) {
- if (munlock(pinned, mlock_size) < 0)
- perror("munlock");
- munmap(pinned, mlock_size);
- }
-}
-
-static void *fio_pin_memory(void)
-{
- unsigned long long phys_mem;
- void *ptr;
-
- if (!mlock_size)
- return NULL;
-
- /*
- * Don't allow mlock of more than real_mem-128MB
- */
- phys_mem = os_phys_mem();
- if (phys_mem) {
- if ((mlock_size + 128 * 1024 * 1024) > phys_mem) {
- mlock_size = phys_mem - 128 * 1024 * 1024;
- fprintf(f_out, "fio: limiting mlocked memory to %lluMiB\n", mlock_size >> 20);
- }
- }
-
- ptr = mmap(NULL, mlock_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | OS_MAP_ANON, 0, 0);
- if (!ptr) {
- perror("malloc locked mem");
- return NULL;
- }
- if (mlock(ptr, mlock_size) < 0) {
- munmap(ptr, mlock_size);
- perror("mlock");
- return NULL;
- }
-
- return ptr;
-}
-
/*
* Main function for kicking off and reaping jobs, as needed.
*/
struct thread_data *td;
unsigned long spent;
int i, todo, nr_running, m_rate, t_rate, nr_started;
- void *mlocked_mem;
- mlocked_mem = fio_pin_memory();
+ if (fio_pin_memory())
+ return;
if (!terse_output) {
printf("Starting %d thread%s\n", thread_number, thread_number > 1 ? "s" : "");
nr_started = 0;
m_rate = t_rate = 0;
- for (i = 0; i < thread_number; i++) {
- td = &threads[i];
-
- run_str[td->thread_number - 1] = 'P';
+ for_each_td(td, i) {
+ print_status_init(td->thread_number - 1);
init_disk_util(td);
}
}
- gettimeofday(&genesis, NULL);
+ time_init();
while (todo) {
struct thread_data *map[MAX_JOBS];
/*
* create threads (TD_NOT_CREATED -> TD_CREATED)
*/
- for (i = 0; i < thread_number; i++) {
- td = &threads[i];
-
+ for_each_td(td, i) {
if (td->runstate != TD_NOT_CREATED)
continue;
}
if (td->start_delay) {
- spent = mtime_since_now(&genesis);
+ spent = mtime_since_genesis();
if (td->start_delay * 1000 > spent)
continue;
/*
* start created threads (TD_INITIALIZED -> TD_RUNNING).
*/
- for (i = 0; i < thread_number; i++) {
- td = &threads[i];
-
+ for_each_td(td, i) {
if (td->runstate != TD_INITIALIZED)
continue;
}
update_io_ticks();
- fio_unpin_memory(mlocked_mem);
+ fio_unpin_memory();
}
int main(int argc, char *argv[])
projects / fio.git / blobdiff