2 * pmemblk: IO engine that uses NVML libpmemblk to read and write data
4 * Copyright (C) 2016 Hewlett Packard Enterprise Development LP
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License,
8 * version 2 as published by the Free Software Foundation..
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public
16 * License along with this program; if not, write to the Free
17 * Software Foundation, Inc., 59 Temple Place, Suite 330,
18 * Boston, MA 02111-1307 USA
25 * IO engine that uses libpmemblk to read and write data
27 * To build as an external engine:
28 * Copy pmemblk.c to the engines/ directory of the fio source code, then:
30 * gcc -Wall -shared -rdynamic -fPIC -o pmemblk.o -D_GNU_SOURCE pmemblk.c
32 * To use as an external engine:
33 * ioengine=/full/path/to/pmemblk.o
35 * Other relevant settings:
40 * filename=/pmem0/fiotestfile,BSIZE,FSIZEMB
42 * thread must be set to 1 for pmemblk as multiple processes cannot
43 * open the same block pool file.
45 * iodepth should be set to 1 as pmemblk is always synchronous.
46 * Use numjobs to scale up.
48 * direct=1 is implied as pmemblk is always direct.
50 * Can set unlink to 1 to remove the block pool file after testing.
52 * When specifying the filename, if the block pool file does not already
53 * exist, then the pmemblk engine can create the pool file if you specify
54 * the block and file sizes. BSIZE is the block size in bytes.
55 * FSIZEMB is the pool file size in MB.
58 * By default, the pmemblk engine will let the system find the libpmemblk.so
59 * that it uses. You can use an alternative libpmemblk by setting the
60 * FIO_PMEMBLK_LIB environment variable to the full path to the desired
82 typedef struct PMEMblkpool_s PMEMblkpool;
84 PMEMblkpool* (*pmemblk_create_ptr)(const char*, size_t, size_t, mode_t) = NULL;
85 PMEMblkpool* (*pmemblk_open_ptr)(const char*, size_t) = NULL;
86 void (*pmemblk_close_ptr)(PMEMblkpool*) = NULL;
87 size_t (*pmemblk_nblock_ptr)(PMEMblkpool*) = NULL;
88 size_t (*pmemblk_bsize_ptr)(PMEMblkpool*) = NULL;
89 int (*pmemblk_read_ptr)(PMEMblkpool*, void*, off_t) = NULL;
90 int (*pmemblk_write_ptr)(PMEMblkpool*, const void*, off_t) = NULL;
100 path = "libpmemblk.so";
102 dl = dlopen(path, RTLD_NOW | RTLD_NODELETE);
106 if (NULL == (pmemblk_create_ptr = dlsym(dl, "pmemblk_create")))
108 if (NULL == (pmemblk_open_ptr = dlsym(dl, "pmemblk_open")))
110 if (NULL == (pmemblk_close_ptr = dlsym(dl, "pmemblk_close")))
112 if (NULL == (pmemblk_nblock_ptr = dlsym(dl, "pmemblk_nblock")))
114 if (NULL == (pmemblk_bsize_ptr = dlsym(dl, "pmemblk_bsize")))
116 if (NULL == (pmemblk_read_ptr = dlsym(dl, "pmemblk_read")))
118 if (NULL == (pmemblk_write_ptr = dlsym(dl, "pmemblk_write")))
124 log_err("fio: unable to load libpmemblk: %s\n", dlerror());
130 } /* load_libpmemblk() */
132 #define pmemblk_create pmemblk_create_ptr
133 #define pmemblk_open pmemblk_open_ptr
134 #define pmemblk_close pmemblk_close_ptr
135 #define pmemblk_nblock pmemblk_nblock_ptr
136 #define pmemblk_bsize pmemblk_bsize_ptr
137 #define pmemblk_read pmemblk_read_ptr
138 #define pmemblk_write pmemblk_write_ptr
141 typedef struct fio_pmemblk_file* fio_pmemblk_file_t;
142 struct fio_pmemblk_file {
143 fio_pmemblk_file_t pmb_next;
146 PMEMblkpool* pmb_pool;
150 #define FIOFILEPMBSET(_f, _v) do { \
151 (_f)->engine_data = (uint64_t)(uintptr_t)(_v); \
153 #define FIOFILEPMBGET(_f) ((fio_pmemblk_file_t)((_f)->engine_data))
155 static fio_pmemblk_file_t Cache = NULL;
157 static pthread_mutex_t CacheLock = PTHREAD_MUTEX_INITIALIZER;
158 #define CACHE_LOCK() \
159 (void)pthread_mutex_lock(&CacheLock)
160 #define CACHE_UNLOCK() \
161 (void)pthread_mutex_unlock(&CacheLock)
163 #define PMB_CREATE (0x0001) /* should create file */
167 fio_pmemblk_cache_lookup(
171 fio_pmemblk_file_t i;
173 for (i = Cache; i != NULL; i = i->pmb_next)
174 if (0 == strcmp(filename, i->pmb_filename))
179 } /* fio_pmemblk_cache_lookup() */
183 fio_pmemblk_cache_insert(
184 fio_pmemblk_file_t pmb
187 pmb->pmb_next = Cache;
192 } /* fio_pmemblk_cache_insert() */
196 fio_pmemblk_cache_remove(
197 fio_pmemblk_file_t pmb
200 fio_pmemblk_file_t i;
203 Cache = Cache->pmb_next;
204 pmb->pmb_next = NULL;
208 for (i = Cache; i != NULL; i = i->pmb_next)
209 if (pmb == i->pmb_next) {
210 i->pmb_next = i->pmb_next->pmb_next;
211 pmb->pmb_next = NULL;
217 } /* fio_pmemblk_cache_remove() */
221 * to control block size and gross file size at the libpmemblk
222 * level, we allow the block size and file size to be appended
225 * path[,bsize,fsizemb]
227 * note that we do not use the fio option "filesize" to dictate
228 * the file size because we can only give libpmemblk the gross
229 * file size, which is different from the net or usable file
230 * size (which is probably what fio wants).
232 * the final path without the parameters is returned in ppath.
233 * the block size and file size are returned in pbsize and fsize.
235 * note that the user should specify the file size in MiB, but
236 * we return bytes from here.
240 const char* pathspec,
251 path = strdup(pathspec);
257 /* extract sizes, if given */
258 s = strrchr(path, ',');
259 if (s && (fsizemb = strtoull(s+1, NULL, 10))) {
261 s = strrchr(path, ',');
262 if (s && (bsize = strtoull(s+1, NULL, 10))) {
266 *pfsize = fsizemb << 20;
271 /* size specs not found */
272 strcpy(path, pathspec);
278 } /* pmb_parse_path() */
284 const char* pathspec,
288 fio_pmemblk_file_t pmb;
293 pmb_parse_path(pathspec, &path, &bsize, &fsize);
299 pmb = fio_pmemblk_cache_lookup(path);
302 /* load libpmemblk if needed */
303 if (NULL == pmemblk_open)
304 if (0 != load_libpmemblk(getenv("FIO_PMEMBLK_LIB")))
307 pmb = malloc(sizeof(*pmb));
311 /* try opening existing first, create it if needed */
312 pmb->pmb_pool = pmemblk_open(path, bsize);
313 if ((NULL == pmb->pmb_pool) &&
315 (flags & PMB_CREATE) &&
318 pmb->pmb_pool = pmemblk_create(path, bsize, fsize, 0644);
320 if (NULL == pmb->pmb_pool) {
321 log_err("fio: enable to open pmemblk pool file (errno %d)\n",
326 pmb->pmb_filename = path;
327 pmb->pmb_next = NULL;
329 pmb->pmb_bsize = pmemblk_bsize(pmb->pmb_pool);
330 pmb->pmb_nblocks = pmemblk_nblock(pmb->pmb_pool);
332 fio_pmemblk_cache_insert(pmb);
335 pmb->pmb_refcnt += 1;
343 if (NULL != pmb->pmb_pool)
344 pmemblk_close(pmb->pmb_pool);
345 pmb->pmb_pool = NULL;
346 pmb->pmb_filename = NULL;
359 fio_pmemblk_file_t pmb,
367 if (!keep && (0 == pmb->pmb_refcnt)) {
368 pmemblk_close(pmb->pmb_pool);
369 pmb->pmb_pool = NULL;
370 free(pmb->pmb_filename);
371 pmb->pmb_filename = NULL;
372 fio_pmemblk_cache_remove(pmb);
383 struct thread_data* td,
387 static int thread_warned = 0;
388 static int odirect_warned = 0;
392 if (!td->o.use_thread) {
393 if (!thread_warned) {
395 log_err("fio: must set thread=1 for pmemblk engine\n");
400 if (!td->o.odirect && !odirect_warned) {
402 log_info("fio: direct == 0, but pmemblk is always direct\n");
405 if (td->o.allow_create)
411 } /* pmb_get_flags() */
415 fio_pmemblk_open_file(
416 struct thread_data* td,
420 fio_pmemblk_file_t pmb;
422 if (0 != pmb_get_flags(td, &flags))
425 pmb = pmb_open(f->file_name, flags);
429 FIOFILEPMBSET(f, pmb);
433 } /* fio_pmemblk_open_file() */
437 fio_pmemblk_close_file(
438 struct thread_data fio_unused* td,
442 fio_pmemblk_file_t pmb = FIOFILEPMBGET(f);
447 FIOFILEPMBSET(f, NULL);
451 } /* fio_pmemblk_close_file() */
455 fio_pmemblk_get_file_size(
456 struct thread_data* td,
461 fio_pmemblk_file_t pmb = FIOFILEPMBGET(f);
463 if (fio_file_size_known(f))
467 if (0 != pmb_get_flags(td, &flags))
469 pmb = pmb_open(f->file_name, flags);
474 f->real_file_size = pmb->pmb_bsize * pmb->pmb_nblocks;
476 fio_file_set_size_known(f);
478 if (NULL == FIOFILEPMBGET(f))
483 } /* fio_pmemblk_get_file_size() */
488 struct thread_data* td,
491 struct fio_file* f = io_u->file;
492 fio_pmemblk_file_t pmb = FIOFILEPMBGET(f);
494 unsigned long long off;
497 int (*blkop)(PMEMblkpool*, void*, off_t) = (void*)pmemblk_write;
499 fio_ro_check(td, io_u);
501 switch (io_u->ddir) {
503 blkop = pmemblk_read;
507 len = io_u->xfer_buflen;
509 io_u->error = EINVAL;
510 if (0 != (off % pmb->pmb_bsize))
512 if (0 != (len % pmb->pmb_bsize))
514 if ((off + len) / pmb->pmb_bsize > pmb->pmb_nblocks)
518 buf = io_u->xfer_buf;
519 off /= pmb->pmb_bsize;
520 len /= pmb->pmb_bsize;
522 if (0 != blkop(pmb->pmb_pool, buf, off)) {
526 buf += pmb->pmb_bsize;
530 off *= pmb->pmb_bsize;
531 len *= pmb->pmb_bsize;
532 io_u->resid = io_u->xfer_buflen - (off - io_u->offset);
536 case DDIR_SYNC_FILE_RANGE:
537 /* we're always sync'd */
541 io_u->error = EINVAL;
545 return FIO_Q_COMPLETED;
547 } /* fio_pmemblk_queue() */
551 fio_pmemblk_unlink_file(
552 struct thread_data* td,
561 * we need our own unlink in case the user has specified
562 * the block and file sizes in the path name. we parse
563 * the file_name to determine the file name we actually used.
566 pmb_parse_path(f->file_name, &path, &bsize, &fsize);
575 } /* fio_pmemblk_unlink_file() */
578 struct ioengine_ops ioengine = {
580 .version = FIO_IOOPS_VERSION,
581 .queue = fio_pmemblk_queue,
582 .open_file = fio_pmemblk_open_file,
583 .close_file = fio_pmemblk_close_file,
584 .get_file_size = fio_pmemblk_get_file_size,
585 .unlink_file = fio_pmemblk_unlink_file,
586 .flags = FIO_SYNCIO | FIO_DISKLESSIO | FIO_NOEXTEND | FIO_NODISKUTIL,
591 fio_init fio_pmemblk_register(void)
593 register_ioengine(&ioengine);
598 fio_exit fio_pmemblk_unregister(void)
600 unregister_ioengine(&ioengine);