X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=engines%2Fmmap.c;h=14e4013de83cac82d09a1eafe013c18797b08970;hp=bd9a9428661abe38048233c0c56c5be256ee7311;hb=8a68c41c2e7e0a1d5128e7e30ab673a6699c2f45;hpb=ff455a04f4e77d5a41532cda61b6fb82a37c2658 diff --git a/engines/mmap.c b/engines/mmap.c index bd9a9428..14e4013d 100644 --- a/engines/mmap.c +++ b/engines/mmap.c @@ -22,11 +22,17 @@ static unsigned long mmap_map_size; static unsigned long mmap_map_mask; +struct fio_mmap_data { + void *mmap_ptr; + size_t mmap_sz; + off_t mmap_off; +}; + static int fio_mmap_file(struct thread_data *td, struct fio_file *f, size_t length, off_t off) { + struct fio_mmap_data *fmd = FILE_ENG_DATA(f); int flags = 0; - int ret = 0; if (td_rw(td)) flags = PROT_READ | PROT_WRITE; @@ -38,72 +44,130 @@ static int fio_mmap_file(struct thread_data *td, struct fio_file *f, } else flags = PROT_READ; - f->mmap_ptr = mmap(NULL, length, flags, MAP_SHARED, f->fd, off); - if (f->mmap_ptr == MAP_FAILED) { - int err = errno; - - f->mmap_ptr = NULL; - td_verror(td, err, "mmap"); + fmd->mmap_ptr = mmap(NULL, length, flags, MAP_SHARED, f->fd, off); + if (fmd->mmap_ptr == MAP_FAILED) { + fmd->mmap_ptr = NULL; + td_verror(td, errno, "mmap"); goto err; } if (!td_random(td)) { - if (madvise(f->mmap_ptr, length, MADV_SEQUENTIAL) < 0) { + if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_SEQUENTIAL) < 0) { td_verror(td, errno, "madvise"); goto err; } } else { - if (madvise(f->mmap_ptr, length, MADV_RANDOM) < 0) { + if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_RANDOM) < 0) { td_verror(td, errno, "madvise"); goto err; } } + if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_DONTNEED) < 0) { + td_verror(td, errno, "madvise"); + goto err; + } + +#ifdef FIO_MADV_FREE + if (f->filetype == FIO_TYPE_BD) + (void) posix_madvise(fmd->mmap_ptr, fmd->mmap_sz, FIO_MADV_FREE); +#endif + err: - return ret; + if (td->error && fmd->mmap_ptr) + munmap(fmd->mmap_ptr, length); + + return td->error; } -static int fio_mmapio_prep(struct thread_data *td, struct io_u *io_u) +/* + * Just mmap an appropriate portion, we cannot mmap the full extent + */ +static int fio_mmapio_prep_limited(struct thread_data *td, struct io_u *io_u) { struct fio_file *f = io_u->file; - int ret = 0; + struct fio_mmap_data *fmd = FILE_ENG_DATA(f); if (io_u->buflen > mmap_map_size) { log_err("fio: bs too big for mmap engine\n"); - ret = EIO; - goto err; + return EIO; } - if (io_u->offset >= f->mmap_off && - io_u->offset + io_u->buflen < f->mmap_off + f->mmap_sz) - goto done; + fmd->mmap_sz = mmap_map_size; + if (fmd->mmap_sz > f->io_size) + fmd->mmap_sz = f->io_size; - if (f->mmap_ptr) { - if (munmap(f->mmap_ptr, f->mmap_sz) < 0) { - ret = errno; - goto err; - } - f->mmap_ptr = NULL; + fmd->mmap_off = io_u->offset; + + return fio_mmap_file(td, f, fmd->mmap_sz, fmd->mmap_off); +} + +/* + * Attempt to mmap the entire file + */ +static int fio_mmapio_prep_full(struct thread_data *td, struct io_u *io_u) +{ + struct fio_file *f = io_u->file; + struct fio_mmap_data *fmd = FILE_ENG_DATA(f); + int ret; + + if (fio_file_partial_mmap(f)) + return EINVAL; + if (io_u->offset != (size_t) io_u->offset || + f->io_size != (size_t) f->io_size) { + fio_file_set_partial_mmap(f); + return EINVAL; } - f->mmap_sz = mmap_map_size; - if (f->mmap_sz > f->io_size) - f->mmap_sz = f->io_size; + fmd->mmap_sz = f->io_size; + fmd->mmap_off = 0; - f->mmap_off = io_u->offset; + ret = fio_mmap_file(td, f, fmd->mmap_sz, fmd->mmap_off); + if (ret) + fio_file_set_partial_mmap(f); - ret = fio_mmap_file(td, f, f->mmap_sz, f->mmap_off); -done: - if (!ret) - io_u->mmap_data = f->mmap_ptr + io_u->offset - f->mmap_off - - f->file_offset; -err: return ret; } +static int fio_mmapio_prep(struct thread_data *td, struct io_u *io_u) +{ + struct fio_file *f = io_u->file; + struct fio_mmap_data *fmd = FILE_ENG_DATA(f); + int ret; + + /* + * It fits within existing mapping, use it + */ + if (io_u->offset >= fmd->mmap_off && + io_u->offset + io_u->buflen < fmd->mmap_off + fmd->mmap_sz) + goto done; + + /* + * unmap any existing mapping + */ + if (fmd->mmap_ptr) { + if (munmap(fmd->mmap_ptr, fmd->mmap_sz) < 0) + return errno; + fmd->mmap_ptr = NULL; + } + + if (fio_mmapio_prep_full(td, io_u)) { + td_clear_error(td); + ret = fio_mmapio_prep_limited(td, io_u); + if (ret) + return ret; + } + +done: + io_u->mmap_data = fmd->mmap_ptr + io_u->offset - fmd->mmap_off - + f->file_offset; + return 0; +} + static int fio_mmapio_queue(struct thread_data *td, struct io_u *io_u) { struct fio_file *f = io_u->file; + struct fio_mmap_data *fmd = FILE_ENG_DATA(f); fio_ro_check(td, io_u); @@ -112,21 +176,27 @@ static int fio_mmapio_queue(struct thread_data *td, struct io_u *io_u) else if (io_u->ddir == DDIR_WRITE) memcpy(io_u->mmap_data, io_u->xfer_buf, io_u->xfer_buflen); else if (ddir_sync(io_u->ddir)) { - if (msync(f->mmap_ptr, f->mmap_sz, MS_SYNC)) { + if (msync(fmd->mmap_ptr, fmd->mmap_sz, MS_SYNC)) { io_u->error = errno; td_verror(td, io_u->error, "msync"); } + } else if (io_u->ddir == DDIR_TRIM) { + int ret = do_io_u_trim(td, io_u); + + if (!ret) + td_verror(td, io_u->error, "trim"); } + /* * not really direct, but should drop the pages from the cache */ - if (td->o.odirect && !ddir_sync(io_u->ddir)) { + if (td->o.odirect && ddir_rw(io_u->ddir)) { if (msync(io_u->mmap_data, io_u->xfer_buflen, MS_SYNC) < 0) { io_u->error = errno; td_verror(td, io_u->error, "msync"); } - if (madvise(io_u->mmap_data, io_u->xfer_buflen, MADV_DONTNEED) < 0) { + if (posix_madvise(io_u->mmap_data, io_u->xfer_buflen, POSIX_MADV_DONTNEED) < 0) { io_u->error = errno; td_verror(td, io_u->error, "madvise"); } @@ -137,8 +207,16 @@ static int fio_mmapio_queue(struct thread_data *td, struct io_u *io_u) static int fio_mmapio_init(struct thread_data *td) { + struct thread_options *o = &td->o; unsigned long shift, mask; + if ((td->o.rw_min_bs & page_mask) && + (o->odirect || o->fsync_blocks || o->fdatasync_blocks)) { + log_err("fio: mmap options dictate a minimum block size of " + "%llu bytes\n", (unsigned long long) page_size); + return 1; + } + mmap_map_size = MMAP_TOTAL_SZ / td->o.nr_files; mask = mmap_map_size; shift = 0; @@ -148,19 +226,50 @@ static int fio_mmapio_init(struct thread_data *td) break; shift++; } while (1); - + mmap_map_mask = 1UL << shift; return 0; } +static int fio_mmapio_open_file(struct thread_data *td, struct fio_file *f) +{ + struct fio_mmap_data *fmd; + int ret; + + ret = generic_open_file(td, f); + if (ret) + return ret; + + fmd = calloc(1, sizeof(*fmd)); + if (!fmd) { + int fio_unused __ret; + __ret = generic_close_file(td, f); + return 1; + } + + FILE_SET_ENG_DATA(f, fmd); + return 0; +} + +static int fio_mmapio_close_file(struct thread_data *td, struct fio_file *f) +{ + struct fio_mmap_data *fmd = FILE_ENG_DATA(f); + + FILE_SET_ENG_DATA(f, NULL); + free(fmd); + fio_file_clear_partial_mmap(f); + + return generic_close_file(td, f); +} + static struct ioengine_ops ioengine = { .name = "mmap", .version = FIO_IOOPS_VERSION, .init = fio_mmapio_init, .prep = fio_mmapio_prep, .queue = fio_mmapio_queue, - .open_file = generic_open_file, - .close_file = generic_close_file, + .open_file = fio_mmapio_open_file, + .close_file = fio_mmapio_close_file, .get_file_size = generic_get_file_size, .flags = FIO_SYNCIO | FIO_NOEXTEND, };