engines/io_uring_cmd: allocate enough ranges for async trims

[fio.git] / engines / io_uring.c

diff --git a/engines/io_uring.c b/engines/io_uring.c
index 376a2a270d12ce2239756dcb87b5cacd96d349b5..38c36fdca26063fcbb8aacbda7a9b1ec71fb655b 100644 (file)
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -18,6 +18,7 @@
 #include "../lib/memalign.h"
 #include "../lib/fls.h"
 #include "../lib/roundup.h"
+#include "../verify.h"
 
 #ifdef ARCH_HAVE_IOURING
 
@@ -456,7 +457,9 @@ static struct io_u *fio_ioring_cmd_event(struct thread_data *td, int event)
        struct ioring_options *o = td->eo;
        struct io_uring_cqe *cqe;
        struct io_u *io_u;
+       struct nvme_data *data;
        unsigned index;
+       int ret;
 
        index = (event + ld->cq_ring_off) & ld->cq_ring_mask;
        if (o->cmd_type == FIO_URING_CMD_NVME)
@@ -470,6 +473,15 @@ static struct io_u *fio_ioring_cmd_event(struct thread_data *td, int event)
        else
                io_u->error = 0;
 
+       if (o->cmd_type == FIO_URING_CMD_NVME) {
+               data = FILE_ENG_DATA(io_u->file);
+               if (data->pi_type && (io_u->ddir == DDIR_READ) && !o->pi_act) {
+                       ret = fio_nvme_pi_verify(data, io_u);
+                       if (ret)
+                               io_u->error = ret;
+               }
+       }
+
        return io_u;
 }
 
@@ -1154,10 +1166,13 @@ static int fio_ioring_init(struct thread_data *td)
                        md_size += td->o.mem_align - page_size;
                if (td->o.mem_type == MEM_MALLOC) {
                        ld->md_buf = malloc(md_size);
-                       if (!ld->md_buf)
+                       if (!ld->md_buf) {
+                               free(ld);
                                return 1;
+                       }
                } else {
                        log_err("fio: Only iomem=malloc or mem=malloc is supported\n");
+                       free(ld);
                        return 1;
                }
        }
@@ -1181,7 +1196,7 @@ static int fio_ioring_init(struct thread_data *td)
            td->o.zone_mode == ZONE_MODE_ZBD)
                td->io_ops->flags |= FIO_ASYNCIO_SYNC_TRIM;
        else
-               ld->dsm = calloc(ld->iodepth, sizeof(*ld->dsm));
+               ld->dsm = calloc(td->o.iodepth, sizeof(*ld->dsm));
 
        return 0;
 }
@@ -1190,6 +1205,7 @@ static int fio_ioring_io_u_init(struct thread_data *td, struct io_u *io_u)
 {
        struct ioring_data *ld = td->io_ops_data;
        struct ioring_options *o = td->eo;
+       struct nvme_pi_data *pi_data;
        char *p;
 
        ld->io_u_index[io_u->index] = io_u;
@@ -1198,11 +1214,32 @@ static int fio_ioring_io_u_init(struct thread_data *td, struct io_u *io_u)
                p = PTR_ALIGN(ld->md_buf, page_mask) + td->o.mem_align;
                p += o->md_per_io_size * io_u->index;
                io_u->mmap_data = p;
+
+               if (!o->pi_act) {
+                       pi_data = calloc(1, sizeof(*pi_data));
+                       pi_data->io_flags |= o->prchk;
+                       pi_data->apptag_mask = o->apptag_mask;
+                       pi_data->apptag = o->apptag;
+                       io_u->engine_data = pi_data;
+               }
        }
 
        return 0;
 }
 
+static void fio_ioring_io_u_free(struct thread_data *td, struct io_u *io_u)
+{
+       struct ioring_options *o = td->eo;
+       struct nvme_pi *pi;
+
+       if (!strcmp(td->io_ops->name, "io_uring_cmd") &&
+           (o->cmd_type == FIO_URING_CMD_NVME)) {
+               pi = io_u->engine_data;
+               free(pi);
+               io_u->engine_data = NULL;
+       }
+}
+
 static int fio_ioring_open_file(struct thread_data *td, struct fio_file *f)
 {
        struct ioring_data *ld = td->io_ops_data;
@@ -1263,6 +1300,19 @@ static int fio_ioring_cmd_open_file(struct thread_data *td, struct fio_file *f)
                                return 1;
                        }
                 }
+
+               /*
+                * For extended logical block sizes we cannot use verify when
+                * end to end data protection checks are enabled, as the PI
+                * section of data buffer conflicts with verify.
+                */
+               if (data->ms && data->pi_type && data->lba_ext &&
+                   td->o.verify != VERIFY_NONE) {
+                       log_err("%s: for extended LBA, verify cannot be used when E2E data protection is enabled\n",
+                               f->file_name);
+                       td_verror(td, EINVAL, "fio_ioring_cmd_open_file");
+                       return 1;
+               }
        }
        if (!ld || !o->registerfiles)
                return generic_open_file(td, f);
@@ -1411,6 +1461,7 @@ static struct ioengine_ops ioengine_uring_cmd = {
        .init                   = fio_ioring_init,
        .post_init              = fio_ioring_cmd_post_init,
        .io_u_init              = fio_ioring_io_u_init,
+       .io_u_free              = fio_ioring_io_u_free,
        .prep                   = fio_ioring_cmd_prep,
        .queue                  = fio_ioring_queue,
        .commit                 = fio_ioring_commit,