2 * pmemblk: IO engine that uses NVML libpmemblk to read and write data
4 * Copyright (C) 2016 Hewlett Packard Enterprise Development LP
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License,
8 * version 2 as published by the Free Software Foundation..
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public
16 * License along with this program; if not, write to the Free
17 * Software Foundation, Inc., 59 Temple Place, Suite 330,
18 * Boston, MA 02111-1307 USA
24 * IO engine that uses libpmemblk to read and write data
29 * Other relevant settings:
34 * filename=/pmem0/fiotestfile,BSIZE,FSIZEMB
36 * thread must be set to 1 for pmemblk as multiple processes cannot
37 * open the same block pool file.
39 * iodepth should be set to 1 as pmemblk is always synchronous.
40 * Use numjobs to scale up.
42 * direct=1 is implied as pmemblk is always direct.
44 * Can set unlink to 1 to remove the block pool file after testing.
46 * When specifying the filename, if the block pool file does not already
47 * exist, then the pmemblk engine can create the pool file if you specify
48 * the block and file sizes. BSIZE is the block size in bytes.
49 * FSIZEMB is the pool file size in MB.
51 * See examples/pmemblk.fio for more.
54 * By default, the pmemblk engine will let the system find the libpmemblk.so
55 * that it uses. You can use an alternative libpmemblk by setting the
56 * FIO_PMEMBLK_LIB environment variable to the full path to the desired
76 typedef struct PMEMblkpool_s PMEMblkpool;
78 PMEMblkpool *(*pmemblk_create) (const char *, size_t, size_t, mode_t) = NULL;
79 PMEMblkpool *(*pmemblk_open) (const char *, size_t) = NULL;
80 void (*pmemblk_close) (PMEMblkpool *) = NULL;
81 size_t(*pmemblk_nblock) (PMEMblkpool *) = NULL;
82 size_t(*pmemblk_bsize) (PMEMblkpool *) = NULL;
83 int (*pmemblk_read) (PMEMblkpool *, void *, off_t) = NULL;
84 int (*pmemblk_write) (PMEMblkpool *, const void *, off_t) = NULL;
86 int load_libpmemblk(const char *path)
91 path = "libpmemblk.so";
93 dl = dlopen(path, RTLD_NOW | RTLD_NODELETE);
97 if (NULL == (pmemblk_create = dlsym(dl, "pmemblk_create")))
99 if (NULL == (pmemblk_open = dlsym(dl, "pmemblk_open")))
101 if (NULL == (pmemblk_close = dlsym(dl, "pmemblk_close")))
103 if (NULL == (pmemblk_nblock = dlsym(dl, "pmemblk_nblock")))
105 if (NULL == (pmemblk_bsize = dlsym(dl, "pmemblk_bsize")))
107 if (NULL == (pmemblk_read = dlsym(dl, "pmemblk_read")))
109 if (NULL == (pmemblk_write = dlsym(dl, "pmemblk_write")))
115 log_err("fio: unable to load libpmemblk: %s\n", dlerror());
121 } /* load_libpmemblk() */
123 typedef struct fio_pmemblk_file *fio_pmemblk_file_t;
124 struct fio_pmemblk_file {
125 fio_pmemblk_file_t pmb_next;
128 PMEMblkpool *pmb_pool;
132 #define FIOFILEPMBSET(_f, _v) do { \
133 (_f)->engine_data = (uint64_t)(uintptr_t)(_v); \
135 #define FIOFILEPMBGET(_f) ((fio_pmemblk_file_t)((_f)->engine_data))
137 static fio_pmemblk_file_t Cache = NULL;
139 static pthread_mutex_t CacheLock = PTHREAD_MUTEX_INITIALIZER;
141 #define PMB_CREATE (0x0001) /* should create file */
143 fio_pmemblk_file_t fio_pmemblk_cache_lookup(const char *filename)
145 fio_pmemblk_file_t i;
147 for (i = Cache; i != NULL; i = i->pmb_next)
148 if (0 == strcmp(filename, i->pmb_filename))
153 } /* fio_pmemblk_cache_lookup() */
155 static void fio_pmemblk_cache_insert(fio_pmemblk_file_t pmb)
157 pmb->pmb_next = Cache;
162 } /* fio_pmemblk_cache_insert() */
164 static void fio_pmemblk_cache_remove(fio_pmemblk_file_t pmb)
166 fio_pmemblk_file_t i;
169 Cache = Cache->pmb_next;
170 pmb->pmb_next = NULL;
174 for (i = Cache; i != NULL; i = i->pmb_next)
175 if (pmb == i->pmb_next) {
176 i->pmb_next = i->pmb_next->pmb_next;
177 pmb->pmb_next = NULL;
183 } /* fio_pmemblk_cache_remove() */
186 * to control block size and gross file size at the libpmemblk
187 * level, we allow the block size and file size to be appended
190 * path[,bsize,fsizemb]
192 * note that we do not use the fio option "filesize" to dictate
193 * the file size because we can only give libpmemblk the gross
194 * file size, which is different from the net or usable file
195 * size (which is probably what fio wants).
197 * the final path without the parameters is returned in ppath.
198 * the block size and file size are returned in pbsize and fsize.
200 * note that the user should specify the file size in MiB, but
201 * we return bytes from here.
204 pmb_parse_path(const char *pathspec,
205 char **ppath, uint64_t * pbsize, uint64_t * pfsize)
212 path = strdup(pathspec);
218 /* extract sizes, if given */
219 s = strrchr(path, ',');
220 if (s && (fsizemb = strtoull(s + 1, NULL, 10))) {
222 s = strrchr(path, ',');
223 if (s && (bsize = strtoull(s + 1, NULL, 10))) {
227 *pfsize = fsizemb << 20;
232 /* size specs not found */
233 strcpy(path, pathspec);
239 } /* pmb_parse_path() */
242 fio_pmemblk_file_t pmb_open(const char *pathspec, int flags)
244 fio_pmemblk_file_t pmb;
249 pmb_parse_path(pathspec, &path, &bsize, &fsize);
253 pthread_mutex_lock(&CacheLock);
255 pmb = fio_pmemblk_cache_lookup(path);
258 /* load libpmemblk if needed */
259 if (NULL == pmemblk_open)
260 if (0 != load_libpmemblk(getenv("FIO_PMEMBLK_LIB")))
263 pmb = malloc(sizeof(*pmb));
267 /* try opening existing first, create it if needed */
268 pmb->pmb_pool = pmemblk_open(path, bsize);
269 if ((NULL == pmb->pmb_pool) &&
271 (flags & PMB_CREATE) && (0 < fsize) && (0 < bsize)) {
273 pmemblk_create(path, bsize, fsize, 0644);
275 if (NULL == pmb->pmb_pool) {
277 ("fio: enable to open pmemblk pool file (errno %d)\n",
282 pmb->pmb_filename = path;
283 pmb->pmb_next = NULL;
285 pmb->pmb_bsize = pmemblk_bsize(pmb->pmb_pool);
286 pmb->pmb_nblocks = pmemblk_nblock(pmb->pmb_pool);
288 fio_pmemblk_cache_insert(pmb);
291 pmb->pmb_refcnt += 1;
293 pthread_mutex_unlock(&CacheLock);
299 if (NULL != pmb->pmb_pool)
300 pmemblk_close(pmb->pmb_pool);
301 pmb->pmb_pool = NULL;
302 pmb->pmb_filename = NULL;
308 pthread_mutex_unlock(&CacheLock);
313 static void pmb_close(fio_pmemblk_file_t pmb, const int keep)
315 pthread_mutex_lock(&CacheLock);
319 if (!keep && (0 == pmb->pmb_refcnt)) {
320 pmemblk_close(pmb->pmb_pool);
321 pmb->pmb_pool = NULL;
322 free(pmb->pmb_filename);
323 pmb->pmb_filename = NULL;
324 fio_pmemblk_cache_remove(pmb);
328 pthread_mutex_unlock(&CacheLock);
332 static int pmb_get_flags(struct thread_data *td, uint64_t * pflags)
334 static int thread_warned = 0;
335 static int odirect_warned = 0;
339 if (!td->o.use_thread) {
340 if (!thread_warned) {
342 log_err("fio: must set thread=1 for pmemblk engine\n");
347 if (!td->o.odirect && !odirect_warned) {
349 log_info("fio: direct == 0, but pmemblk is always direct\n");
352 if (td->o.allow_create)
358 } /* pmb_get_flags() */
360 static int fio_pmemblk_open_file(struct thread_data *td, struct fio_file *f)
363 fio_pmemblk_file_t pmb;
365 if (0 != pmb_get_flags(td, &flags))
368 pmb = pmb_open(f->file_name, flags);
372 FIOFILEPMBSET(f, pmb);
376 } /* fio_pmemblk_open_file() */
379 fio_pmemblk_close_file(struct thread_data fio_unused * td, struct fio_file *f)
381 fio_pmemblk_file_t pmb = FIOFILEPMBGET(f);
386 FIOFILEPMBSET(f, NULL);
390 } /* fio_pmemblk_close_file() */
392 static int fio_pmemblk_get_file_size(struct thread_data *td, struct fio_file *f)
395 fio_pmemblk_file_t pmb = FIOFILEPMBGET(f);
397 if (fio_file_size_known(f))
401 if (0 != pmb_get_flags(td, &flags))
403 pmb = pmb_open(f->file_name, flags);
408 f->real_file_size = pmb->pmb_bsize * pmb->pmb_nblocks;
410 fio_file_set_size_known(f);
412 if (NULL == FIOFILEPMBGET(f))
417 } /* fio_pmemblk_get_file_size() */
419 static int fio_pmemblk_queue(struct thread_data *td, struct io_u *io_u)
421 struct fio_file *f = io_u->file;
422 fio_pmemblk_file_t pmb = FIOFILEPMBGET(f);
424 unsigned long long off;
427 int (*blkop) (PMEMblkpool *, void *, off_t) = (void *)pmemblk_write;
429 fio_ro_check(td, io_u);
431 switch (io_u->ddir) {
433 blkop = pmemblk_read;
437 len = io_u->xfer_buflen;
439 io_u->error = EINVAL;
440 if (0 != (off % pmb->pmb_bsize))
442 if (0 != (len % pmb->pmb_bsize))
444 if ((off + len) / pmb->pmb_bsize > pmb->pmb_nblocks)
448 buf = io_u->xfer_buf;
449 off /= pmb->pmb_bsize;
450 len /= pmb->pmb_bsize;
452 if (0 != blkop(pmb->pmb_pool, buf, off)) {
456 buf += pmb->pmb_bsize;
460 off *= pmb->pmb_bsize;
461 len *= pmb->pmb_bsize;
462 io_u->resid = io_u->xfer_buflen - (off - io_u->offset);
466 case DDIR_SYNC_FILE_RANGE:
467 /* we're always sync'd */
471 io_u->error = EINVAL;
475 return FIO_Q_COMPLETED;
477 } /* fio_pmemblk_queue() */
479 static int fio_pmemblk_unlink_file(struct thread_data *td, struct fio_file *f)
486 * we need our own unlink in case the user has specified
487 * the block and file sizes in the path name. we parse
488 * the file_name to determine the file name we actually used.
491 pmb_parse_path(f->file_name, &path, &bsize, &fsize);
500 } /* fio_pmemblk_unlink_file() */
502 struct ioengine_ops ioengine = {
504 .version = FIO_IOOPS_VERSION,
505 .queue = fio_pmemblk_queue,
506 .open_file = fio_pmemblk_open_file,
507 .close_file = fio_pmemblk_close_file,
508 .get_file_size = fio_pmemblk_get_file_size,
509 .unlink_file = fio_pmemblk_unlink_file,
510 .flags = FIO_SYNCIO | FIO_DISKLESSIO | FIO_NOEXTEND | FIO_NODISKUTIL,
514 fio_init fio_pmemblk_register(void)
516 register_ioengine(&ioengine);
520 fio_exit fio_pmemblk_unregister(void)
522 unregister_ioengine(&ioengine);