[fio.git] / engines / mmap.c

/*
 * mmap engine
 *
 * IO engine that reads/writes from files by doing memcpy to/from
 * a memory mapped region of the file.
 *
 */
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <sys/mman.h>

#include "../fio.h"
#include "../verify.h"

/*
 * Limits us to 1GB of mapped files in total
 */
#define MMAP_TOTAL_SZ	(1 * 1024 * 1024 * 1024UL)

static unsigned long mmap_map_size;
static unsigned long mmap_map_mask;

struct fio_mmap_data {
	void *mmap_ptr;
	size_t mmap_sz;
	off_t mmap_off;
};

static int fio_mmap_file(struct thread_data *td, struct fio_file *f,
			 size_t length, off_t off)
{
	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
	int flags = 0;

	if (td_rw(td))
		flags = PROT_READ | PROT_WRITE;
	else if (td_write(td)) {
		flags = PROT_WRITE;

		if (td->o.verify != VERIFY_NONE)
			flags |= PROT_READ;
	} else
		flags = PROT_READ;

	fmd->mmap_ptr = mmap(NULL, length, flags, MAP_SHARED, f->fd, off);
	if (fmd->mmap_ptr == MAP_FAILED) {
		fmd->mmap_ptr = NULL;
		td_verror(td, errno, "mmap");
		goto err;
	}

	if (!td_random(td)) {
		if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_SEQUENTIAL) < 0) {
			td_verror(td, errno, "madvise");
			goto err;
		}
	} else {
		if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_RANDOM) < 0) {
			td_verror(td, errno, "madvise");
			goto err;
		}
	}

err:
	if (td->error && fmd->mmap_ptr)
		munmap(fmd->mmap_ptr, length);

	return td->error;
}

/*
 * Just mmap an appropriate portion, we cannot mmap the full extent
 */
static int fio_mmapio_prep_limited(struct thread_data *td, struct io_u *io_u)
{
	struct fio_file *f = io_u->file;
	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);

	if (io_u->buflen > mmap_map_size) {
		log_err("fio: bs too big for mmap engine\n");
		return EIO;
	}

	fmd->mmap_sz = mmap_map_size;
	if (fmd->mmap_sz  > f->io_size)
		fmd->mmap_sz = f->io_size;

	fmd->mmap_off = io_u->offset;

	return fio_mmap_file(td, f, fmd->mmap_sz, fmd->mmap_off);
}

/*
 * Attempt to mmap the entire file
 */
static int fio_mmapio_prep_full(struct thread_data *td, struct io_u *io_u)
{
	struct fio_file *f = io_u->file;
	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
	int ret;

	if (fio_file_partial_mmap(f))
		return EINVAL;

	fmd->mmap_sz = f->io_size;
	fmd->mmap_off = 0;

	ret = fio_mmap_file(td, f, fmd->mmap_sz, fmd->mmap_off);
	if (ret)
		fio_file_set_partial_mmap(f);

	return ret;
}

static int fio_mmapio_prep(struct thread_data *td, struct io_u *io_u)
{
	struct fio_file *f = io_u->file;
	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
	int ret;

	/*
	 * It fits within existing mapping, use it
	 */
	if (io_u->offset >= fmd->mmap_off &&
	    io_u->offset + io_u->buflen < fmd->mmap_off + fmd->mmap_sz)
		goto done;

	/*
	 * unmap any existing mapping
	 */
	if (fmd->mmap_ptr) {
		if (munmap(fmd->mmap_ptr, fmd->mmap_sz) < 0)
			return errno;
		fmd->mmap_ptr = NULL;
	}

	if (fio_mmapio_prep_full(td, io_u)) {
		td_clear_error(td);
		ret = fio_mmapio_prep_limited(td, io_u);
		if (ret)
			return ret;
	}

done:
	io_u->mmap_data = fmd->mmap_ptr + io_u->offset - fmd->mmap_off -
				f->file_offset;
	return 0;
}

static int fio_mmapio_queue(struct thread_data *td, struct io_u *io_u)
{
	struct fio_file *f = io_u->file;
	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);

	fio_ro_check(td, io_u);

	if (io_u->ddir == DDIR_READ)
		memcpy(io_u->xfer_buf, io_u->mmap_data, io_u->xfer_buflen);
	else if (io_u->ddir == DDIR_WRITE)
		memcpy(io_u->mmap_data, io_u->xfer_buf, io_u->xfer_buflen);
	else if (ddir_sync(io_u->ddir)) {
		if (msync(fmd->mmap_ptr, fmd->mmap_sz, MS_SYNC)) {
			io_u->error = errno;
			td_verror(td, io_u->error, "msync");
		}
	} else if (io_u->ddir == DDIR_TRIM) {
		int ret = do_io_u_trim(td, io_u);

		if (!ret)
			td_verror(td, io_u->error, "trim");
	}


	/*
	 * not really direct, but should drop the pages from the cache
	 */
	if (td->o.odirect && ddir_rw(io_u->ddir)) {
		if (msync(io_u->mmap_data, io_u->xfer_buflen, MS_SYNC) < 0) {
			io_u->error = errno;
			td_verror(td, io_u->error, "msync");
		}
		if (posix_madvise(io_u->mmap_data, io_u->xfer_buflen, POSIX_MADV_DONTNEED) < 0) {
			io_u->error = errno;
			td_verror(td, io_u->error, "madvise");
		}
	}

	return FIO_Q_COMPLETED;
}

static int fio_mmapio_init(struct thread_data *td)
{
	struct thread_options *o = &td->o;
	unsigned long shift, mask;

	if ((td->o.rw_min_bs & page_mask) &&
	    (o->odirect || o->fsync_blocks || o->fdatasync_blocks)) {
		log_err("fio: mmap options dictate a minimum block size of "
			"%llu bytes\n", (unsigned long long) page_size);
		return 1;
	}

	mmap_map_size = MMAP_TOTAL_SZ / td->o.nr_files;
	mask = mmap_map_size;
	shift = 0;
	do {
		mask >>= 1;
		if (!mask)
			break;
		shift++;
	} while (1);

	mmap_map_mask = 1UL << shift;
	return 0;
}

static int fio_mmapio_open_file(struct thread_data *td, struct fio_file *f)
{
	struct fio_mmap_data *fmd;
	int ret;

	ret = generic_open_file(td, f);
	if (ret)
		return ret;

	fmd = calloc(1, sizeof(*fmd));
	if (!fmd) {
		int fio_unused ret;
		ret = generic_close_file(td, f);
		return 1;
	}

	FILE_SET_ENG_DATA(f, fmd);
	return 0;
}

static int fio_mmapio_close_file(struct thread_data *td, struct fio_file *f)
{
	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);

	FILE_SET_ENG_DATA(f, NULL);
	free(fmd);

	return generic_close_file(td, f);
}

static int fio_mmapio_invalidate(struct thread_data *td, struct fio_file *f)
{
	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
	int ret;

	ret = posix_madvise(fmd->mmap_ptr, fmd->mmap_sz, POSIX_MADV_DONTNEED);
#ifdef FIO_MADV_FREE
	if (f->filetype == FIO_TYPE_BD)
		(void) posix_madvise(fmd->mmap_ptr, fmd->mmap_sz, FIO_MADV_FREE);
#endif

	return ret;
}

static struct ioengine_ops ioengine = {
	.name		= "mmap",
	.version	= FIO_IOOPS_VERSION,
	.init		= fio_mmapio_init,
	.prep		= fio_mmapio_prep,
	.queue		= fio_mmapio_queue,
	.open_file	= fio_mmapio_open_file,
	.close_file	= fio_mmapio_close_file,
	.invalidate	= fio_mmapio_invalidate,
	.get_file_size	= generic_get_file_size,
	.flags		= FIO_SYNCIO | FIO_NOEXTEND,
};

static void fio_init fio_mmapio_register(void)
{
	register_ioengine(&ioengine);
}

static void fio_exit fio_mmapio_unregister(void)
{
	unregister_ioengine(&ioengine);
}
Commit	Line	Data
2866c82d	1	/*
da751ca9 JA	2	* mmap engine
	3	*
	4	* IO engine that reads/writes from files by doing memcpy to/from
	5	* a memory mapped region of the file.
2866c82d JA	6	*
	7	*/
	8	#include <stdio.h>
	9	#include <stdlib.h>
	10	#include <unistd.h>
	11	#include <errno.h>
2866c82d	12	#include <sys/mman.h>
5f350952 JA	13
5f350952 JA	14	#include "../fio.h"
4f5af7b2	15	#include "../verify.h"
2866c82d	16
ac893112	17	/*
ff455a04	18	* Limits us to 1GB of mapped files in total
ac893112	19	*/
ff455a04	20	#define MMAP_TOTAL_SZ (1 * 1024 * 1024 * 1024UL)
2866c82d	21
ac893112 JA	22	static unsigned long mmap_map_size;
ac893112 JA	23	static unsigned long mmap_map_mask;
2866c82d	24
03a32636 JA	25	struct fio_mmap_data {
	26	void *mmap_ptr;
	27	size_t mmap_sz;
	28	off_t mmap_off;
	29	};
	30
ac893112 JA	31	static int fio_mmap_file(struct thread_data td, struct fio_file f,
ac893112 JA	32	size_t length, off_t off)
b5af8293	33	{
e19ccb55	34	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
ac893112	35	int flags = 0;
c97d8369	36
b5af8293 JA	37	if (td_rw(td))
	38	flags = PROT_READ \| PROT_WRITE;
	39	else if (td_write(td)) {
	40	flags = PROT_WRITE;
	41
2dc1bbeb	42	if (td->o.verify != VERIFY_NONE)
b5af8293 JA	43	flags \|= PROT_READ;
	44	} else
	45	flags = PROT_READ;
	46
03a32636 JA	47	fmd->mmap_ptr = mmap(NULL, length, flags, MAP_SHARED, f->fd, off);
	48	if (fmd->mmap_ptr == MAP_FAILED) {
	49	fmd->mmap_ptr = NULL;
ed47cbf7	50	td_verror(td, errno, "mmap");
b5af8293 JA	51	goto err;
	52	}
	53
b5af8293	54	if (!td_random(td)) {
03a32636	55	if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_SEQUENTIAL) < 0) {
b5af8293 JA	56	td_verror(td, errno, "madvise");
	57	goto err;
	58	}
	59	} else {
03a32636	60	if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_RANDOM) < 0) {
b5af8293 JA	61	td_verror(td, errno, "madvise");
	62	goto err;
	63	}
	64	}
	65
b5af8293	66	err:
03a32636 JA	67	if (td->error && fmd->mmap_ptr)
03a32636 JA	68	munmap(fmd->mmap_ptr, length);
93bcfd20	69
ed47cbf7	70	return td->error;
b5af8293 JA	71	}
b5af8293 JA	72
ed47cbf7 JA	73	/*
	74	* Just mmap an appropriate portion, we cannot mmap the full extent
	75	*/
	76	static int fio_mmapio_prep_limited(struct thread_data td, struct io_u io_u)
b5af8293	77	{
ac893112	78	struct fio_file *f = io_u->file;
e19ccb55	79	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
6977bcd0	80
ac893112 JA	81	if (io_u->buflen > mmap_map_size) {
ac893112 JA	82	log_err("fio: bs too big for mmap engine\n");
ed47cbf7	83	return EIO;
ac893112 JA	84	}
ac893112 JA	85
03a32636 JA	86	fmd->mmap_sz = mmap_map_size;
	87	if (fmd->mmap_sz > f->io_size)
	88	fmd->mmap_sz = f->io_size;
6977bcd0	89
03a32636	90	fmd->mmap_off = io_u->offset;
ac893112	91
03a32636	92	return fio_mmap_file(td, f, fmd->mmap_sz, fmd->mmap_off);
ed47cbf7 JA	93	}
	94
	95	/*
	96	* Attempt to mmap the entire file
	97	*/
	98	static int fio_mmapio_prep_full(struct thread_data td, struct io_u io_u)
	99	{
	100	struct fio_file *f = io_u->file;
e19ccb55	101	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
ed47cbf7 JA	102	int ret;
	103
	104	if (fio_file_partial_mmap(f))
	105	return EINVAL;
	106
03a32636 JA	107	fmd->mmap_sz = f->io_size;
03a32636 JA	108	fmd->mmap_off = 0;
ed47cbf7	109
03a32636	110	ret = fio_mmap_file(td, f, fmd->mmap_sz, fmd->mmap_off);
ed47cbf7 JA	111	if (ret)
	112	fio_file_set_partial_mmap(f);
	113
6977bcd0	114	return ret;
b5af8293 JA	115	}
b5af8293 JA	116
ed47cbf7 JA	117	static int fio_mmapio_prep(struct thread_data td, struct io_u io_u)
	118	{
	119	struct fio_file *f = io_u->file;
e19ccb55	120	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
ed47cbf7 JA	121	int ret;
ed47cbf7 JA	122
8f933cae JA	123	/*
	124	* It fits within existing mapping, use it
	125	*/
03a32636 JA	126	if (io_u->offset >= fmd->mmap_off &&
03a32636 JA	127	io_u->offset + io_u->buflen < fmd->mmap_off + fmd->mmap_sz)
ed47cbf7 JA	128	goto done;
ed47cbf7 JA	129
8f933cae JA	130	/*
	131	* unmap any existing mapping
	132	*/
03a32636 JA	133	if (fmd->mmap_ptr) {
03a32636 JA	134	if (munmap(fmd->mmap_ptr, fmd->mmap_sz) < 0)
8f933cae	135	return errno;
03a32636	136	fmd->mmap_ptr = NULL;
8f933cae JA	137	}
8f933cae JA	138
ed47cbf7 JA	139	if (fio_mmapio_prep_full(td, io_u)) {
	140	td_clear_error(td);
	141	ret = fio_mmapio_prep_limited(td, io_u);
	142	if (ret)
	143	return ret;
	144	}
	145
	146	done:
03a32636	147	io_u->mmap_data = fmd->mmap_ptr + io_u->offset - fmd->mmap_off -
ed47cbf7 JA	148	f->file_offset;
	149	return 0;
	150	}
	151
ac893112 JA	152	static int fio_mmapio_queue(struct thread_data td, struct io_u io_u)
	153	{
	154	struct fio_file *f = io_u->file;
e19ccb55	155	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
ac893112 JA	156
	157	fio_ro_check(td, io_u);
	158
	159	if (io_u->ddir == DDIR_READ)
	160	memcpy(io_u->xfer_buf, io_u->mmap_data, io_u->xfer_buflen);
	161	else if (io_u->ddir == DDIR_WRITE)
	162	memcpy(io_u->mmap_data, io_u->xfer_buf, io_u->xfer_buflen);
5f9099ea	163	else if (ddir_sync(io_u->ddir)) {
03a32636	164	if (msync(fmd->mmap_ptr, fmd->mmap_sz, MS_SYNC)) {
ac893112 JA	165	io_u->error = errno;
	166	td_verror(td, io_u->error, "msync");
	167	}
ff58fced JA	168	} else if (io_u->ddir == DDIR_TRIM) {
	169	int ret = do_io_u_trim(td, io_u);
	170
	171	if (!ret)
	172	td_verror(td, io_u->error, "trim");
ac893112 JA	173	}
ac893112 JA	174
ff58fced	175
ac893112 JA	176	/*
	177	* not really direct, but should drop the pages from the cache
	178	*/
ff58fced	179	if (td->o.odirect && ddir_rw(io_u->ddir)) {
ac893112 JA	180	if (msync(io_u->mmap_data, io_u->xfer_buflen, MS_SYNC) < 0) {
	181	io_u->error = errno;
	182	td_verror(td, io_u->error, "msync");
	183	}
03e20d68	184	if (posix_madvise(io_u->mmap_data, io_u->xfer_buflen, POSIX_MADV_DONTNEED) < 0) {
ac893112 JA	185	io_u->error = errno;
	186	td_verror(td, io_u->error, "madvise");
	187	}
	188	}
	189
	190	return FIO_Q_COMPLETED;
	191	}
	192
	193	static int fio_mmapio_init(struct thread_data *td)
	194	{
913ea0db	195	struct thread_options *o = &td->o;
ac893112 JA	196	unsigned long shift, mask;
ac893112 JA	197
913ea0db JA	198	if ((td->o.rw_min_bs & page_mask) &&
	199	(o->odirect \|\| o->fsync_blocks \|\| o->fdatasync_blocks)) {
	200	log_err("fio: mmap options dictate a minimum block size of "
4e0a8fa2	201	"%llu bytes\n", (unsigned long long) page_size);
913ea0db JA	202	return 1;
	203	}
	204
ac893112 JA	205	mmap_map_size = MMAP_TOTAL_SZ / td->o.nr_files;
	206	mask = mmap_map_size;
	207	shift = 0;
	208	do {
	209	mask >>= 1;
	210	if (!mask)
	211	break;
	212	shift++;
	213	} while (1);
93bcfd20	214
ac893112 JA	215	mmap_map_mask = 1UL << shift;
	216	return 0;
	217	}
	218
03a32636 JA	219	static int fio_mmapio_open_file(struct thread_data td, struct fio_file f)
	220	{
	221	struct fio_mmap_data *fmd;
	222	int ret;
	223
	224	ret = generic_open_file(td, f);
	225	if (ret)
	226	return ret;
	227
	228	fmd = calloc(1, sizeof(*fmd));
	229	if (!fmd) {
	230	int fio_unused ret;
	231	ret = generic_close_file(td, f);
	232	return 1;
	233	}
	234
e19ccb55	235	FILE_SET_ENG_DATA(f, fmd);
03a32636 JA	236	return 0;
	237	}
	238
	239	static int fio_mmapio_close_file(struct thread_data td, struct fio_file f)
	240	{
e19ccb55	241	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
03a32636	242
e19ccb55	243	FILE_SET_ENG_DATA(f, NULL);
03a32636 JA	244	free(fmd);
	245
	246	return generic_close_file(td, f);
	247	}
	248
	249	static int fio_mmapio_invalidate(struct thread_data td, struct fio_file f)
	250	{
e19ccb55	251	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
03a32636 JA	252	int ret;
	253
	254	ret = posix_madvise(fmd->mmap_ptr, fmd->mmap_sz, POSIX_MADV_DONTNEED);
	255	#ifdef FIO_MADV_FREE
	256	if (f->filetype == FIO_TYPE_BD)
	257	(void) posix_madvise(fmd->mmap_ptr, fmd->mmap_sz, FIO_MADV_FREE);
	258	#endif
	259
	260	return ret;
	261	}
	262
5f350952	263	static struct ioengine_ops ioengine = {
2866c82d JA	264	.name = "mmap",
2866c82d JA	265	.version = FIO_IOOPS_VERSION,
ac893112 JA	266	.init = fio_mmapio_init,
ac893112 JA	267	.prep = fio_mmapio_prep,
2866c82d	268	.queue = fio_mmapio_queue,
03a32636 JA	269	.open_file = fio_mmapio_open_file,
	270	.close_file = fio_mmapio_close_file,
	271	.invalidate = fio_mmapio_invalidate,
df9c26b1	272	.get_file_size = generic_get_file_size,
0263882a	273	.flags = FIO_SYNCIO \| FIO_NOEXTEND,
2866c82d	274	};
5f350952 JA	275
	276	static void fio_init fio_mmapio_register(void)
	277	{
	278	register_ioengine(&ioengine);
	279	}
	280
	281	static void fio_exit fio_mmapio_unregister(void)
	282	{
	283	unregister_ioengine(&ioengine);
	284	}