[fio.git] / engines / mmap.c

/*
 * mmap engine
 *
 * IO engine that reads/writes from files by doing memcpy to/from
 * a memory mapped region of the file.
 *
 */
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/mman.h>

#include "../fio.h"
#include "../verify.h"

/*
 * Limits us to 1GiB of mapped files in total
 */
#define MMAP_TOTAL_SZ	(1 * 1024 * 1024 * 1024UL)

static unsigned long mmap_map_size;

struct fio_mmap_data {
	void *mmap_ptr;
	size_t mmap_sz;
	off_t mmap_off;
};

static bool fio_madvise_file(struct thread_data *td, struct fio_file *f,
			     size_t length)

{
	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);

	if (!td->o.fadvise_hint)
		return true;

	if (!td_random(td)) {
		if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_SEQUENTIAL) < 0) {
			td_verror(td, errno, "madvise");
			return false;
		}
	} else {
		if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_RANDOM) < 0) {
			td_verror(td, errno, "madvise");
			return false;
		}
	}

	return true;
}

static int fio_mmap_file(struct thread_data *td, struct fio_file *f,
			 size_t length, off_t off)
{
	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
	int flags = 0;

	if (td_rw(td) && !td->o.verify_only)
		flags = PROT_READ | PROT_WRITE;
	else if (td_write(td) && !td->o.verify_only) {
		flags = PROT_WRITE;

		if (td->o.verify != VERIFY_NONE)
			flags |= PROT_READ;
	} else
		flags = PROT_READ;

	fmd->mmap_ptr = mmap(NULL, length, flags, MAP_SHARED, f->fd, off);
	if (fmd->mmap_ptr == MAP_FAILED) {
		fmd->mmap_ptr = NULL;
		td_verror(td, errno, "mmap");
		goto err;
	}

	if (!fio_madvise_file(td, f, length))
		goto err;

	if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_DONTNEED) < 0) {
		td_verror(td, errno, "madvise");
		goto err;
	}

#ifdef FIO_MADV_FREE
	if (f->filetype == FIO_TYPE_BLOCK)
		(void) posix_madvise(fmd->mmap_ptr, fmd->mmap_sz, FIO_MADV_FREE);
#endif

err:
	if (td->error && fmd->mmap_ptr)
		munmap(fmd->mmap_ptr, length);

	return td->error;
}

/*
 * Just mmap an appropriate portion, we cannot mmap the full extent
 */
static int fio_mmapio_prep_limited(struct thread_data *td, struct io_u *io_u)
{
	struct fio_file *f = io_u->file;
	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);

	if (io_u->buflen > mmap_map_size) {
		log_err("fio: bs too big for mmap engine\n");
		return EIO;
	}

	fmd->mmap_sz = mmap_map_size;
	if (fmd->mmap_sz  > f->io_size)
		fmd->mmap_sz = f->io_size;

	fmd->mmap_off = io_u->offset;

	return fio_mmap_file(td, f, fmd->mmap_sz, fmd->mmap_off);
}

/*
 * Attempt to mmap the entire file
 */
static int fio_mmapio_prep_full(struct thread_data *td, struct io_u *io_u)
{
	struct fio_file *f = io_u->file;
	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
	int ret;

	if (fio_file_partial_mmap(f))
		return EINVAL;
	if (io_u->offset != (size_t) io_u->offset ||
	    f->io_size != (size_t) f->io_size) {
		fio_file_set_partial_mmap(f);
		return EINVAL;
	}

	fmd->mmap_sz = f->io_size;
	fmd->mmap_off = 0;

	ret = fio_mmap_file(td, f, fmd->mmap_sz, fmd->mmap_off);
	if (ret)
		fio_file_set_partial_mmap(f);

	return ret;
}

static int fio_mmapio_prep(struct thread_data *td, struct io_u *io_u)
{
	struct fio_file *f = io_u->file;
	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
	int ret;

	/*
	 * It fits within existing mapping, use it
	 */
	if (io_u->offset >= fmd->mmap_off &&
	    io_u->offset + io_u->buflen <= fmd->mmap_off + fmd->mmap_sz)
		goto done;

	/*
	 * unmap any existing mapping
	 */
	if (fmd->mmap_ptr) {
		if (munmap(fmd->mmap_ptr, fmd->mmap_sz) < 0)
			return errno;
		fmd->mmap_ptr = NULL;
	}

	if (fio_mmapio_prep_full(td, io_u)) {
		td_clear_error(td);
		ret = fio_mmapio_prep_limited(td, io_u);
		if (ret)
			return ret;
	}

done:
	io_u->mmap_data = fmd->mmap_ptr + io_u->offset - fmd->mmap_off -
				f->file_offset;
	return 0;
}

static enum fio_q_status
fio_mmapio_queue(struct thread_data *td, struct io_u *io_u)
{
	struct fio_file *f = io_u->file;
	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);

	fio_ro_check(td, io_u);

	if (io_u->ddir == DDIR_READ)
		memcpy(io_u->xfer_buf, io_u->mmap_data, io_u->xfer_buflen);
	else if (io_u->ddir == DDIR_WRITE)
		memcpy(io_u->mmap_data, io_u->xfer_buf, io_u->xfer_buflen);
	else if (ddir_sync(io_u->ddir)) {
		if (msync(fmd->mmap_ptr, fmd->mmap_sz, MS_SYNC)) {
			io_u->error = errno;
			td_verror(td, io_u->error, "msync");
		}
	} else if (io_u->ddir == DDIR_TRIM) {
		int ret = do_io_u_trim(td, io_u);

		if (!ret)
			td_verror(td, io_u->error, "trim");
	}


	/*
	 * not really direct, but should drop the pages from the cache
	 */
	if (td->o.odirect && ddir_rw(io_u->ddir)) {
		if (msync(io_u->mmap_data, io_u->xfer_buflen, MS_SYNC) < 0) {
			io_u->error = errno;
			td_verror(td, io_u->error, "msync");
		}
		if (posix_madvise(io_u->mmap_data, io_u->xfer_buflen, POSIX_MADV_DONTNEED) < 0) {
			io_u->error = errno;
			td_verror(td, io_u->error, "madvise");
		}
	}

	return FIO_Q_COMPLETED;
}

static int fio_mmapio_init(struct thread_data *td)
{
	struct thread_options *o = &td->o;

	if ((o->rw_min_bs & page_mask) &&
	    (o->odirect || o->fsync_blocks || o->fdatasync_blocks)) {
		log_err("fio: mmap options dictate a minimum block size of "
			"%llu bytes\n", (unsigned long long) page_size);
		return 1;
	}

	mmap_map_size = MMAP_TOTAL_SZ / o->nr_files;
	return 0;
}

static int fio_mmapio_open_file(struct thread_data *td, struct fio_file *f)
{
	struct fio_mmap_data *fmd;
	int ret;

	ret = generic_open_file(td, f);
	if (ret)
		return ret;

	fmd = calloc(1, sizeof(*fmd));
	if (!fmd) {
		int fio_unused __ret;
		__ret = generic_close_file(td, f);
		return 1;
	}

	FILE_SET_ENG_DATA(f, fmd);
	return 0;
}

static int fio_mmapio_close_file(struct thread_data *td, struct fio_file *f)
{
	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);

	FILE_SET_ENG_DATA(f, NULL);
	free(fmd);
	fio_file_clear_partial_mmap(f);

	return generic_close_file(td, f);
}

static struct ioengine_ops ioengine = {
	.name		= "mmap",
	.version	= FIO_IOOPS_VERSION,
	.init		= fio_mmapio_init,
	.prep		= fio_mmapio_prep,
	.queue		= fio_mmapio_queue,
	.open_file	= fio_mmapio_open_file,
	.close_file	= fio_mmapio_close_file,
	.get_file_size	= generic_get_file_size,
	.flags		= FIO_SYNCIO | FIO_NOEXTEND,
};

static void fio_init fio_mmapio_register(void)
{
	register_ioengine(&ioengine);
}

static void fio_exit fio_mmapio_unregister(void)
{
	unregister_ioengine(&ioengine);
}
Commit	Line	Data
2866c82d	1	/*
da751ca9 JA	2	* mmap engine
	3	*
	4	* IO engine that reads/writes from files by doing memcpy to/from
	5	* a memory mapped region of the file.
2866c82d JA	6	*
	7	*/
	8	#include <stdio.h>
	9	#include <stdlib.h>
2866c82d	10	#include <errno.h>
2866c82d	11	#include <sys/mman.h>
5f350952 JA	12
5f350952 JA	13	#include "../fio.h"
4f5af7b2	14	#include "../verify.h"
2866c82d	15
ac893112	16	/*
420b104a	17	* Limits us to 1GiB of mapped files in total
ac893112	18	*/
ff455a04	19	#define MMAP_TOTAL_SZ (1 * 1024 * 1024 * 1024UL)
2866c82d	20
ac893112	21	static unsigned long mmap_map_size;
2866c82d	22
03a32636 JA	23	struct fio_mmap_data {
	24	void *mmap_ptr;
	25	size_t mmap_sz;
	26	off_t mmap_off;
	27	};
	28
c712c97a JA	29	static bool fio_madvise_file(struct thread_data td, struct fio_file f,
	30	size_t length)
	31
	32	{
	33	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
	34
	35	if (!td->o.fadvise_hint)
	36	return true;
	37
	38	if (!td_random(td)) {
	39	if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_SEQUENTIAL) < 0) {
	40	td_verror(td, errno, "madvise");
	41	return false;
	42	}
	43	} else {
	44	if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_RANDOM) < 0) {
	45	td_verror(td, errno, "madvise");
	46	return false;
	47	}
	48	}
c712c97a JA	49
	50	return true;
	51	}
	52
ac893112 JA	53	static int fio_mmap_file(struct thread_data td, struct fio_file f,
ac893112 JA	54	size_t length, off_t off)
b5af8293	55	{
e19ccb55	56	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
ac893112	57	int flags = 0;
c97d8369	58
81a1ee61	59	if (td_rw(td) && !td->o.verify_only)
b5af8293	60	flags = PROT_READ \| PROT_WRITE;
81a1ee61	61	else if (td_write(td) && !td->o.verify_only) {
b5af8293 JA	62	flags = PROT_WRITE;
b5af8293 JA	63
2dc1bbeb	64	if (td->o.verify != VERIFY_NONE)
b5af8293 JA	65	flags \|= PROT_READ;
	66	} else
	67	flags = PROT_READ;
	68
03a32636 JA	69	fmd->mmap_ptr = mmap(NULL, length, flags, MAP_SHARED, f->fd, off);
	70	if (fmd->mmap_ptr == MAP_FAILED) {
	71	fmd->mmap_ptr = NULL;
ed47cbf7	72	td_verror(td, errno, "mmap");
b5af8293 JA	73	goto err;
	74	}
	75
c712c97a	76	if (!fio_madvise_file(td, f, length))
ecbfcd94	77	goto err;
ecbfcd94	78
c69f6bf3 JA	79	if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_DONTNEED) < 0) {
	80	td_verror(td, errno, "madvise");
	81	goto err;
	82	}
	83
	84	#ifdef FIO_MADV_FREE
	85	if (f->filetype == FIO_TYPE_BLOCK)
	86	(void) posix_madvise(fmd->mmap_ptr, fmd->mmap_sz, FIO_MADV_FREE);
	87	#endif
	88
b5af8293	89	err:
03a32636 JA	90	if (td->error && fmd->mmap_ptr)
03a32636 JA	91	munmap(fmd->mmap_ptr, length);
93bcfd20	92
ed47cbf7	93	return td->error;
b5af8293 JA	94	}
b5af8293 JA	95
ed47cbf7 JA	96	/*
	97	* Just mmap an appropriate portion, we cannot mmap the full extent
	98	*/
	99	static int fio_mmapio_prep_limited(struct thread_data td, struct io_u io_u)
b5af8293	100	{
ac893112	101	struct fio_file *f = io_u->file;
e19ccb55	102	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
6977bcd0	103
ac893112 JA	104	if (io_u->buflen > mmap_map_size) {
ac893112 JA	105	log_err("fio: bs too big for mmap engine\n");
ed47cbf7	106	return EIO;
ac893112 JA	107	}
ac893112 JA	108
03a32636 JA	109	fmd->mmap_sz = mmap_map_size;
	110	if (fmd->mmap_sz > f->io_size)
	111	fmd->mmap_sz = f->io_size;
6977bcd0	112
03a32636	113	fmd->mmap_off = io_u->offset;
ac893112	114
03a32636	115	return fio_mmap_file(td, f, fmd->mmap_sz, fmd->mmap_off);
ed47cbf7 JA	116	}
	117
	118	/*
	119	* Attempt to mmap the entire file
	120	*/
	121	static int fio_mmapio_prep_full(struct thread_data td, struct io_u io_u)
	122	{
	123	struct fio_file *f = io_u->file;
e19ccb55	124	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
ed47cbf7 JA	125	int ret;
	126
	127	if (fio_file_partial_mmap(f))
	128	return EINVAL;
01626837 JA	129	if (io_u->offset != (size_t) io_u->offset \|\|
	130	f->io_size != (size_t) f->io_size) {
	131	fio_file_set_partial_mmap(f);
	132	return EINVAL;
	133	}
ed47cbf7	134
03a32636 JA	135	fmd->mmap_sz = f->io_size;
03a32636 JA	136	fmd->mmap_off = 0;
ed47cbf7	137
03a32636	138	ret = fio_mmap_file(td, f, fmd->mmap_sz, fmd->mmap_off);
ed47cbf7 JA	139	if (ret)
	140	fio_file_set_partial_mmap(f);
	141
6977bcd0	142	return ret;
b5af8293 JA	143	}
b5af8293 JA	144
ed47cbf7 JA	145	static int fio_mmapio_prep(struct thread_data td, struct io_u io_u)
	146	{
	147	struct fio_file *f = io_u->file;
e19ccb55	148	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
ed47cbf7 JA	149	int ret;
ed47cbf7 JA	150
8f933cae JA	151	/*
	152	* It fits within existing mapping, use it
	153	*/
03a32636	154	if (io_u->offset >= fmd->mmap_off &&
f23ff35e	155	io_u->offset + io_u->buflen <= fmd->mmap_off + fmd->mmap_sz)
ed47cbf7 JA	156	goto done;
ed47cbf7 JA	157
8f933cae JA	158	/*
	159	* unmap any existing mapping
	160	*/
03a32636 JA	161	if (fmd->mmap_ptr) {
03a32636 JA	162	if (munmap(fmd->mmap_ptr, fmd->mmap_sz) < 0)
8f933cae	163	return errno;
03a32636	164	fmd->mmap_ptr = NULL;
8f933cae JA	165	}
8f933cae JA	166
ed47cbf7 JA	167	if (fio_mmapio_prep_full(td, io_u)) {
	168	td_clear_error(td);
	169	ret = fio_mmapio_prep_limited(td, io_u);
	170	if (ret)
	171	return ret;
	172	}
	173
	174	done:
03a32636	175	io_u->mmap_data = fmd->mmap_ptr + io_u->offset - fmd->mmap_off -
ed47cbf7 JA	176	f->file_offset;
	177	return 0;
	178	}
	179
d3b07186 BVA	180	static enum fio_q_status
d3b07186 BVA	181	fio_mmapio_queue(struct thread_data td, struct io_u io_u)
ac893112 JA	182	{
ac893112 JA	183	struct fio_file *f = io_u->file;
e19ccb55	184	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
ac893112 JA	185
	186	fio_ro_check(td, io_u);
	187
	188	if (io_u->ddir == DDIR_READ)
	189	memcpy(io_u->xfer_buf, io_u->mmap_data, io_u->xfer_buflen);
	190	else if (io_u->ddir == DDIR_WRITE)
	191	memcpy(io_u->mmap_data, io_u->xfer_buf, io_u->xfer_buflen);
5f9099ea	192	else if (ddir_sync(io_u->ddir)) {
03a32636	193	if (msync(fmd->mmap_ptr, fmd->mmap_sz, MS_SYNC)) {
ac893112 JA	194	io_u->error = errno;
	195	td_verror(td, io_u->error, "msync");
	196	}
ff58fced JA	197	} else if (io_u->ddir == DDIR_TRIM) {
	198	int ret = do_io_u_trim(td, io_u);
	199
	200	if (!ret)
	201	td_verror(td, io_u->error, "trim");
ac893112 JA	202	}
ac893112 JA	203
ff58fced	204
ac893112 JA	205	/*
	206	* not really direct, but should drop the pages from the cache
	207	*/
ff58fced	208	if (td->o.odirect && ddir_rw(io_u->ddir)) {
ac893112 JA	209	if (msync(io_u->mmap_data, io_u->xfer_buflen, MS_SYNC) < 0) {
	210	io_u->error = errno;
	211	td_verror(td, io_u->error, "msync");
	212	}
03e20d68	213	if (posix_madvise(io_u->mmap_data, io_u->xfer_buflen, POSIX_MADV_DONTNEED) < 0) {
ac893112 JA	214	io_u->error = errno;
	215	td_verror(td, io_u->error, "madvise");
	216	}
	217	}
	218
	219	return FIO_Q_COMPLETED;
	220	}
	221
	222	static int fio_mmapio_init(struct thread_data *td)
	223	{
913ea0db	224	struct thread_options *o = &td->o;
ac893112	225
487197d9	226	if ((o->rw_min_bs & page_mask) &&
913ea0db JA	227	(o->odirect \|\| o->fsync_blocks \|\| o->fdatasync_blocks)) {
913ea0db JA	228	log_err("fio: mmap options dictate a minimum block size of "
4e0a8fa2	229	"%llu bytes\n", (unsigned long long) page_size);
913ea0db JA	230	return 1;
	231	}
	232
487197d9	233	mmap_map_size = MMAP_TOTAL_SZ / o->nr_files;
ac893112 JA	234	return 0;
	235	}
	236
03a32636 JA	237	static int fio_mmapio_open_file(struct thread_data td, struct fio_file f)
	238	{
	239	struct fio_mmap_data *fmd;
	240	int ret;
	241
	242	ret = generic_open_file(td, f);
	243	if (ret)
	244	return ret;
	245
	246	fmd = calloc(1, sizeof(*fmd));
	247	if (!fmd) {
8a68c41c JA	248	int fio_unused __ret;
8a68c41c JA	249	__ret = generic_close_file(td, f);
03a32636 JA	250	return 1;
	251	}
	252
e19ccb55	253	FILE_SET_ENG_DATA(f, fmd);
03a32636 JA	254	return 0;
	255	}
	256
	257	static int fio_mmapio_close_file(struct thread_data td, struct fio_file f)
	258	{
e19ccb55	259	struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
03a32636	260
e19ccb55	261	FILE_SET_ENG_DATA(f, NULL);
03a32636	262	free(fmd);
965f0a41	263	fio_file_clear_partial_mmap(f);
03a32636 JA	264
	265	return generic_close_file(td, f);
	266	}
	267
5f350952	268	static struct ioengine_ops ioengine = {
2866c82d JA	269	.name = "mmap",
2866c82d JA	270	.version = FIO_IOOPS_VERSION,
ac893112 JA	271	.init = fio_mmapio_init,
ac893112 JA	272	.prep = fio_mmapio_prep,
2866c82d	273	.queue = fio_mmapio_queue,
03a32636 JA	274	.open_file = fio_mmapio_open_file,
03a32636 JA	275	.close_file = fio_mmapio_close_file,
df9c26b1	276	.get_file_size = generic_get_file_size,
0263882a	277	.flags = FIO_SYNCIO \| FIO_NOEXTEND,
2866c82d	278	};
5f350952 JA	279
	280	static void fio_init fio_mmapio_register(void)
	281	{
	282	register_ioengine(&ioengine);
	283	}
	284
	285	static void fio_exit fio_mmapio_unregister(void)
	286	{
	287	unregister_ioengine(&ioengine);
	288	}