fio: Introduce the log_prio option

Introduce the log_prio option to expand priority logging from just a
single bit information (priority high vs low) to the full value of the
priority value used to execute IOs. When this option is set, the
priority value is printed as a 16-bits hexadecimal value combining
the I/O priority class and priority level as defined by the
ioprio_value() helper.

Similarly to the log_offset option, this option does not result in
actual I/O priority logging when log_avg_msec is set.

This patch also fixes a problem with the IO_U_F_PRIORITY flag, namely
that this flag is used to indicate that the IO is being executed with a
high priority on the device while at the same time indicating how to
account for the IO completion latency (high_prio clat vs low_prio clat).
With the introduction of the cmdprio_class and cmdprio options, these
assumptions are not necesarilly compatible anymore.

These problems are addressed as follows:
* The priority_bit field of struct iosample is replaced with the
  16-bits priority field representing the full io_u->ioprio value. When
  log_prio is set, the priority field value is logged as is. When
  log_prio is not set, 1 is logged as the entry's priority field if the
  sample priority class is IOPRIO_CLASS_RT, and 0 otherwise.
* IO_U_F_PRIORITY is renamed to IO_U_F_HIGH_PRIO to indicate that a job
  IO has the highest priority within the job context and so must be
  accounted as such using high_prio clat.

While fio final statistics only show accounting of high vs low IO
completion latency statistics, the log_prio option allows a user to
perform more detailed statistical analysis of a workload using
multiple different IO priorities.

Signed-off-by: Damien Le Moal <damien.lemoal@wdc.com>
Signed-off-by: Niklas Cassel <niklas.cassel@wdc.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/cconv.c b/cconv.c
index e3a8c27cf63205bb7eb4eba5c3c0fc7dd703249e..2dc5274ef69c22b8914285d9327ca9c3e6d929cf 100644 (file)
--- a/cconv.c
+++ b/cconv.c
@@ -192,6 +192,7 @@ void convert_thread_options_to_cpu(struct thread_options *o,
        o->log_hist_coarseness = le32_to_cpu(top->log_hist_coarseness);
        o->log_max = le32_to_cpu(top->log_max);
        o->log_offset = le32_to_cpu(top->log_offset);
+       o->log_prio = le32_to_cpu(top->log_prio);
        o->log_gz = le32_to_cpu(top->log_gz);
        o->log_gz_store = le32_to_cpu(top->log_gz_store);
        o->log_unix_epoch = le32_to_cpu(top->log_unix_epoch);
@@ -417,6 +418,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
        top->log_avg_msec = cpu_to_le32(o->log_avg_msec);
        top->log_max = cpu_to_le32(o->log_max);
        top->log_offset = cpu_to_le32(o->log_offset);
+       top->log_prio = cpu_to_le32(o->log_prio);
        top->log_gz = cpu_to_le32(o->log_gz);
        top->log_gz_store = cpu_to_le32(o->log_gz_store);
        top->log_unix_epoch = cpu_to_le32(o->log_unix_epoch);

diff --git a/client.c b/client.c
index 29d8750a5b2b6948bc8206399d6a1be774a22a78..8b230617f79bd0dbab6853e9388a1e7fa8f700ce 100644 (file)
--- a/client.c
+++ b/client.c
@@ -1679,6 +1679,7 @@ static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd,
        ret->log_type           = le32_to_cpu(ret->log_type);
        ret->compressed         = le32_to_cpu(ret->compressed);
        ret->log_offset         = le32_to_cpu(ret->log_offset);
+       ret->log_prio           = le32_to_cpu(ret->log_prio);
        ret->log_hist_coarseness = le32_to_cpu(ret->log_hist_coarseness);
 
        if (*store_direct)
@@ -1696,6 +1697,7 @@ static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd,
                s->data.val     = le64_to_cpu(s->data.val);
                s->__ddir       = __le32_to_cpu(s->__ddir);
                s->bs           = le64_to_cpu(s->bs);
+               s->priority     = le16_to_cpu(s->priority);
 
                if (ret->log_offset) {
                        struct io_sample_offset *so = (void *) s;

diff --git a/engines/filecreate.c b/engines/filecreate.c
index 16c64928162654ad76c6697958b846eb5bb0d985..4bb13c348c1a4113a1425dbf620dc5d74a8264be 100644 (file)
--- a/engines/filecreate.c
+++ b/engines/filecreate.c
@@ -49,7 +49,7 @@ static int open_file(struct thread_data *td, struct fio_file *f)
                uint64_t nsec;
 
                nsec = ntime_since_now(&start);
-               add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0);
+               add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0, false);
        }
 
        return 0;

diff --git a/engines/filedelete.c b/engines/filedelete.c
index 64c586391037cf6b140f59bae351f8f4b08483ce..e882ccf0176726be863a605c81afc0f5fbd8e999 100644 (file)
--- a/engines/filedelete.c
+++ b/engines/filedelete.c
@@ -51,7 +51,7 @@ static int delete_file(struct thread_data *td, struct fio_file *f)
                uint64_t nsec;
 
                nsec = ntime_since_now(&start);
-               add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0);
+               add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0, false);
        }
 
        return 0;

diff --git a/engines/filestat.c b/engines/filestat.c
index 405f028d11cd383aa48d4e6c3840a592a1074b4b..003112474b39f61c94937447a8518d8b8bc25027 100644 (file)
--- a/engines/filestat.c
+++ b/engines/filestat.c
@@ -125,7 +125,7 @@ static int stat_file(struct thread_data *td, struct fio_file *f)
                uint64_t nsec;
 
                nsec = ntime_since_now(&start);
-               add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0);
+               add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0, false);
        }
 
        return 0;

diff --git a/engines/io_uring.c b/engines/io_uring.c
index df2d6c4c7e4a1e3903b4446974dd70abd8700025..27a4a67860956153a450e3aab099dbd47c63b909 100644 (file)
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -463,7 +463,7 @@ static void fio_ioring_prio_prep(struct thread_data *td, struct io_u *io_u)
                         * than the priority set by "prio" and "prioclass"
                         * options.
                         */
-                       io_u->flags |= IO_U_F_PRIORITY;
+                       io_u->flags |= IO_U_F_HIGH_PRIO;
                }
        } else {
                sqe->ioprio = td->ioprio;
@@ -474,9 +474,11 @@ static void fio_ioring_prio_prep(struct thread_data *td, struct io_u *io_u)
                         * is higher (has a lower value) than the async IO
                         * priority.
                         */
-                       io_u->flags |= IO_U_F_PRIORITY;
+                       io_u->flags |= IO_U_F_HIGH_PRIO;
                }
        }
+
+       io_u->ioprio = sqe->ioprio;
 }
 
 static enum fio_q_status fio_ioring_queue(struct thread_data *td,

diff --git a/engines/libaio.c b/engines/libaio.c
index 62c8aed72d0f35f4b4c9586d8e48cc7dc0b8a607..dd655355ba6995725c82f27ec3264690d92c3709 100644 (file)
--- a/engines/libaio.c
+++ b/engines/libaio.c
@@ -215,6 +215,7 @@ static void fio_libaio_prio_prep(struct thread_data *td, struct io_u *io_u)
                ioprio_value(cmdprio->class[ddir], cmdprio->level[ddir]);
 
        if (p && rand_between(&td->prio_state, 0, 99) < p) {
+               io_u->ioprio = cmdprio_value;
                io_u->iocb.aio_reqprio = cmdprio_value;
                io_u->iocb.u.c.flags |= IOCB_FLAG_IOPRIO;
                if (!td->ioprio || cmdprio_value < td->ioprio) {
@@ -222,7 +223,7 @@ static void fio_libaio_prio_prep(struct thread_data *td, struct io_u *io_u)
                         * The async IO priority is higher (has a lower value)
                         * than the default context priority.
                         */
-                       io_u->flags |= IO_U_F_PRIORITY;
+                       io_u->flags |= IO_U_F_HIGH_PRIO;
                }
        } else if (td->ioprio && td->ioprio < cmdprio_value) {
                /*
@@ -230,7 +231,7 @@ static void fio_libaio_prio_prep(struct thread_data *td, struct io_u *io_u)
                 * and this priority is higher (has a lower value) than the
                 * async IO priority.
                 */
-               io_u->flags |= IO_U_F_PRIORITY;
+               io_u->flags |= IO_U_F_HIGH_PRIO;
        }
 }
 

diff --git a/eta.c b/eta.c
index db13cb18103226028ca324cf5acdfb2d03fe4507..ea1781f3b792147636210f38714abb089870a2da 100644 (file)
--- a/eta.c
+++ b/eta.c
@@ -509,7 +509,7 @@ bool calc_thread_status(struct jobs_eta *je, int force)
                memcpy(&rate_prev_time, &now, sizeof(now));
                regrow_agg_logs();
                for_each_rw_ddir(ddir) {
-                       add_agg_sample(sample_val(je->rate[ddir]), ddir, 0, 0);
+                       add_agg_sample(sample_val(je->rate[ddir]), ddir, 0);
                }
        }
 

diff --git a/fio.1 b/fio.1
index 415a91bb25acaa8a594fb977286c5e4089938050..03fddffb1d8cbfedbcc08fe315faa74aae027c6e 100644 (file)
--- a/fio.1
+++ b/fio.1
@@ -3266,6 +3266,11 @@ If this is set, the iolog options will include the byte offset for the I/O
 entry as well as the other data values. Defaults to 0 meaning that
 offsets are not present in logs. Also see \fBLOG FILE FORMATS\fR section.
 .TP
+.BI log_prio \fR=\fPbool
+If this is set, the iolog options will include the I/O priority for the I/O
+entry as well as the other data values. Defaults to 0 meaning that
+I/O priorities are not present in logs. Also see \fBLOG FILE FORMATS\fR section.
+.TP
 .BI log_compression \fR=\fPint
 If this is set, fio will compress the I/O logs as it goes, to keep the
 memory footprint lower. When a log reaches the specified size, that chunk is
@@ -4199,8 +4204,14 @@ The entry's `block size' is always in bytes. The `offset' is the position in byt
 from the start of the file for that particular I/O. The logging of the offset can be
 toggled with \fBlog_offset\fR.
 .P
-`Command priority` is 0 for normal priority and 1 for high priority. This is controlled
-by the ioengine specific \fBcmdprio_percentage\fR.
+If \fBlog_prio\fR is not set, the entry's `Command priority` is 1 for an IO executed
+with the highest RT priority class (\fBprioclass\fR=1 or \fBcmdprio_class\fR=1) and 0
+otherwise. This is controlled by the \fBprioclass\fR option and the ioengine specific
+\fBcmdprio_percentage\fR \fBcmdprio_class\fR options. If \fBlog_prio\fR is set, the
+entry's `Command priority` is the priority set for the IO, as a 16-bits hexadecimal
+number with the lowest 13 bits indicating the priority value (\fBprio\fR and
+\fBcmdprio\fR options) and the highest 3 bits indicating the IO priority class
+(\fBprioclass\fR and \fBcmdprio_class\fR options).
 .P
 Fio defaults to logging every individual I/O but when windowed logging is set
 through \fBlog_avg_msec\fR, either the average (by default) or the maximum

diff --git a/init.c b/init.c
index 871fb5ad1ff7e9daa902d40217fb75f000dbd736..ec1a2cace948394a6bc48c2b56a501965bac92fb 100644 (file)
--- a/init.c
+++ b/init.c
@@ -1583,6 +1583,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num,
                        .hist_coarseness = o->log_hist_coarseness,
                        .log_type = IO_LOG_TYPE_LAT,
                        .log_offset = o->log_offset,
+                       .log_prio = o->log_prio,
                        .log_gz = o->log_gz,
                        .log_gz_store = o->log_gz_store,
                };
@@ -1616,6 +1617,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num,
                        .hist_coarseness = o->log_hist_coarseness,
                        .log_type = IO_LOG_TYPE_HIST,
                        .log_offset = o->log_offset,
+                       .log_prio = o->log_prio,
                        .log_gz = o->log_gz,
                        .log_gz_store = o->log_gz_store,
                };
@@ -1647,6 +1649,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num,
                        .hist_coarseness = o->log_hist_coarseness,
                        .log_type = IO_LOG_TYPE_BW,
                        .log_offset = o->log_offset,
+                       .log_prio = o->log_prio,
                        .log_gz = o->log_gz,
                        .log_gz_store = o->log_gz_store,
                };
@@ -1678,6 +1681,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num,
                        .hist_coarseness = o->log_hist_coarseness,
                        .log_type = IO_LOG_TYPE_IOPS,
                        .log_offset = o->log_offset,
+                       .log_prio = o->log_prio,
                        .log_gz = o->log_gz,
                        .log_gz_store = o->log_gz_store,
                };

diff --git a/io_u.c b/io_u.c
index 696d25cd7770e05d15e70f9ce95b5fb2cd321e6b..5289b5d1d9c6aa18ccd54f01748f95e9f14a4991 100644 (file)
--- a/io_u.c
+++ b/io_u.c
@@ -1595,7 +1595,7 @@ again:
                assert(io_u->flags & IO_U_F_FREE);
                io_u_clear(td, io_u, IO_U_F_FREE | IO_U_F_NO_FILE_PUT |
                                 IO_U_F_TRIMMED | IO_U_F_BARRIER |
-                                IO_U_F_VER_LIST | IO_U_F_PRIORITY);
+                                IO_U_F_VER_LIST | IO_U_F_HIGH_PRIO);
 
                io_u->error = 0;
                io_u->acct_ddir = -1;
@@ -1799,6 +1799,10 @@ struct io_u *get_io_u(struct thread_data *td)
        io_u->xfer_buf = io_u->buf;
        io_u->xfer_buflen = io_u->buflen;
 
+       /*
+        * Remember the issuing context priority. The IO engine may change this.
+        */
+       io_u->ioprio = td->ioprio;
 out:
        assert(io_u->file);
        if (!td_io_prep(td, io_u)) {
@@ -1884,7 +1888,8 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u,
                unsigned long long tnsec;
 
                tnsec = ntime_since(&io_u->start_time, &icd->time);
-               add_lat_sample(td, idx, tnsec, bytes, io_u->offset, io_u_is_prio(io_u));
+               add_lat_sample(td, idx, tnsec, bytes, io_u->offset,
+                              io_u->ioprio, io_u_is_high_prio(io_u));
 
                if (td->flags & TD_F_PROFILE_OPS) {
                        struct prof_io_ops *ops = &td->prof_io_ops;
@@ -1905,7 +1910,8 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u,
 
        if (ddir_rw(idx)) {
                if (!td->o.disable_clat) {
-                       add_clat_sample(td, idx, llnsec, bytes, io_u->offset, io_u_is_prio(io_u));
+                       add_clat_sample(td, idx, llnsec, bytes, io_u->offset,
+                                       io_u->ioprio, io_u_is_high_prio(io_u));
                        io_u_mark_latency(td, llnsec);
                }
 
@@ -2162,7 +2168,7 @@ void io_u_queued(struct thread_data *td, struct io_u *io_u)
                        td = td->parent;
 
                add_slat_sample(td, io_u->ddir, slat_time, io_u->xfer_buflen,
-                               io_u->offset, io_u_is_prio(io_u));
+                               io_u->offset, io_u->ioprio);
        }
 }
 

diff --git a/io_u.h b/io_u.h
index d4c5be4303b3dc4bb7ed9a3cb905b86852231401..bdbac52577afeff573e6895621dccc903acc1d0a 100644 (file)
--- a/io_u.h
+++ b/io_u.h
@@ -21,7 +21,7 @@ enum {
        IO_U_F_TRIMMED          = 1 << 5,
        IO_U_F_BARRIER          = 1 << 6,
        IO_U_F_VER_LIST         = 1 << 7,
-       IO_U_F_PRIORITY         = 1 << 8,
+       IO_U_F_HIGH_PRIO        = 1 << 8,
 };
 
 /*
@@ -46,6 +46,11 @@ struct io_u {
         */
        unsigned short numberio;
 
+       /*
+        * IO priority.
+        */
+       unsigned short ioprio;
+
        /*
         * Allocated/set buffer and length
         */
@@ -188,7 +193,6 @@ static inline enum fio_ddir acct_ddir(struct io_u *io_u)
        td_flags_clear((td), &(io_u->flags), (val))
 #define io_u_set(td, io_u, val)                \
        td_flags_set((td), &(io_u)->flags, (val))
-#define io_u_is_prio(io_u)     \
-       (io_u->flags & (unsigned int) IO_U_F_PRIORITY) != 0
+#define io_u_is_high_prio(io_u)        (io_u->flags & IO_U_F_HIGH_PRIO)
 
 #endif

diff --git a/iolog.c b/iolog.c
index 26501b4ae55517afecc83dab21e29a595a04b2ab..1aeb7a76b2b6ca21ab8bd3d5c18e5ec200669fcb 100644 (file)
--- a/iolog.c
+++ b/iolog.c
@@ -737,6 +737,7 @@ void setup_log(struct io_log **log, struct log_params *p,
        INIT_FLIST_HEAD(&l->io_logs);
        l->log_type = p->log_type;
        l->log_offset = p->log_offset;
+       l->log_prio = p->log_prio;
        l->log_gz = p->log_gz;
        l->log_gz_store = p->log_gz_store;
        l->avg_msec = p->avg_msec;
@@ -769,6 +770,8 @@ void setup_log(struct io_log **log, struct log_params *p,
 
        if (l->log_offset)
                l->log_ddir_mask = LOG_OFFSET_SAMPLE_BIT;
+       if (l->log_prio)
+               l->log_ddir_mask |= LOG_PRIO_SAMPLE_BIT;
 
        INIT_FLIST_HEAD(&l->chunk_list);
 
@@ -895,33 +898,55 @@ static void flush_hist_samples(FILE *f, int hist_coarseness, void *samples,
 void flush_samples(FILE *f, void *samples, uint64_t sample_size)
 {
        struct io_sample *s;
-       int log_offset;
+       int log_offset, log_prio;
        uint64_t i, nr_samples;
+       unsigned int prio_val;
+       const char *fmt;
 
        if (!sample_size)
                return;
 
        s = __get_sample(samples, 0, 0);
        log_offset = (s->__ddir & LOG_OFFSET_SAMPLE_BIT) != 0;
+       log_prio = (s->__ddir & LOG_PRIO_SAMPLE_BIT) != 0;
+
+       if (log_offset) {
+               if (log_prio)
+                       fmt = "%lu, %" PRId64 ", %u, %llu, %llu, 0x%04x\n";
+               else
+                       fmt = "%lu, %" PRId64 ", %u, %llu, %llu, %u\n";
+       } else {
+               if (log_prio)
+                       fmt = "%lu, %" PRId64 ", %u, %llu, 0x%04x\n";
+               else
+                       fmt = "%lu, %" PRId64 ", %u, %llu, %u\n";
+       }
 
        nr_samples = sample_size / __log_entry_sz(log_offset);
 
        for (i = 0; i < nr_samples; i++) {
                s = __get_sample(samples, log_offset, i);
 
+               if (log_prio)
+                       prio_val = s->priority;
+               else
+                       prio_val = ioprio_value_is_class_rt(s->priority);
+
                if (!log_offset) {
-                       fprintf(f, "%lu, %" PRId64 ", %u, %llu, %u\n",
-                                       (unsigned long) s->time,
-                                       s->data.val,
-                                       io_sample_ddir(s), (unsigned long long) s->bs, s->priority_bit);
+                       fprintf(f, fmt,
+                               (unsigned long) s->time,
+                               s->data.val,
+                               io_sample_ddir(s), (unsigned long long) s->bs,
+                               prio_val);
                } else {
                        struct io_sample_offset *so = (void *) s;
 
-                       fprintf(f, "%lu, %" PRId64 ", %u, %llu, %llu, %u\n",
-                                       (unsigned long) s->time,
-                                       s->data.val,
-                                       io_sample_ddir(s), (unsigned long long) s->bs,
-                                       (unsigned long long) so->offset, s->priority_bit);
+                       fprintf(f, fmt,
+                               (unsigned long) s->time,
+                               s->data.val,
+                               io_sample_ddir(s), (unsigned long long) s->bs,
+                               (unsigned long long) so->offset,
+                               prio_val);
                }
        }
 }

diff --git a/iolog.h b/iolog.h
index 9e382cc0211748254f97dc8071176319d874a591..7d66b7c42fb6e4c0d455165d0ebf56467a7b7e0b 100644 (file)
--- a/iolog.h
+++ b/iolog.h
@@ -42,7 +42,7 @@ struct io_sample {
        uint64_t time;
        union io_sample_data data;
        uint32_t __ddir;
-       uint8_t priority_bit;
+       uint16_t priority;
        uint64_t bs;
 };
 
@@ -104,6 +104,11 @@ struct io_log {
         */
        unsigned int log_offset;
 
+       /*
+        * Log I/O priorities
+        */
+       unsigned int log_prio;
+
        /*
         * Max size of log entries before a chunk is compressed
         */
@@ -145,7 +150,13 @@ struct io_log {
  * If the upper bit is set, then we have the offset as well
  */
 #define LOG_OFFSET_SAMPLE_BIT  0x80000000U
-#define io_sample_ddir(io)     ((io)->__ddir & ~LOG_OFFSET_SAMPLE_BIT)
+/*
+ * If the bit following the upper bit is set, then we have the priority
+ */
+#define LOG_PRIO_SAMPLE_BIT    0x40000000U
+
+#define LOG_SAMPLE_BITS                (LOG_OFFSET_SAMPLE_BIT | LOG_PRIO_SAMPLE_BIT)
+#define io_sample_ddir(io)     ((io)->__ddir & ~LOG_SAMPLE_BITS)
 
 static inline void io_sample_set_ddir(struct io_log *log,
                                      struct io_sample *io,
@@ -262,6 +273,7 @@ struct log_params {
        int hist_coarseness;
        int log_type;
        int log_offset;
+       int log_prio;
        int log_gz;
        int log_gz_store;
        int log_compress;

diff --git a/options.c b/options.c
index 708f3703accacd3149668dca2b83aa25722bee3d..74ac1f3f6f2d13d1539a157dc22d693ed3141fb8 100644 (file)
--- a/options.c
+++ b/options.c
@@ -4292,6 +4292,16 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
                .category = FIO_OPT_C_LOG,
                .group  = FIO_OPT_G_INVALID,
        },
+       {
+               .name   = "log_prio",
+               .lname  = "Log priority of IO",
+               .type   = FIO_OPT_BOOL,
+               .off1   = offsetof(struct thread_options, log_prio),
+               .help   = "Include priority value of IO for each log entry",
+               .def    = "0",
+               .category = FIO_OPT_C_LOG,
+               .group  = FIO_OPT_G_INVALID,
+       },
 #ifdef CONFIG_ZLIB
        {
                .name   = "log_compression",

diff --git a/os/os-android.h b/os/os-android.h
index f013172fd8aa690a6d289e7b2bda13094c8e459d..18eb39ce052884b58dedc2f8f7a256281ab8aa95 100644 (file)
--- a/os/os-android.h
+++ b/os/os-android.h
@@ -184,6 +184,11 @@ static inline int ioprio_value(int ioprio_class, int ioprio)
        return (ioprio_class << IOPRIO_CLASS_SHIFT) | ioprio;
 }
 
+static inline bool ioprio_value_is_class_rt(unsigned int priority)
+{
+       return (priority >> IOPRIO_CLASS_SHIFT) == IOPRIO_CLASS_RT;
+}
+
 static inline int ioprio_set(int which, int who, int ioprio_class, int ioprio)
 {
        return syscall(__NR_ioprio_set, which, who,

diff --git a/os/os-linux.h b/os/os-linux.h
index 12886037dd40ddf5dc3dafbd99be943253e4b78d..808f1d022779c8ba7ca0fda0470bfa5783262fd0 100644 (file)
--- a/os/os-linux.h
+++ b/os/os-linux.h
@@ -129,6 +129,11 @@ static inline int ioprio_value(int ioprio_class, int ioprio)
        return (ioprio_class << IOPRIO_CLASS_SHIFT) | ioprio;
 }
 
+static inline bool ioprio_value_is_class_rt(unsigned int priority)
+{
+       return (priority >> IOPRIO_CLASS_SHIFT) == IOPRIO_CLASS_RT;
+}
+
 static inline int ioprio_set(int which, int who, int ioprio_class, int ioprio)
 {
        return syscall(__NR_ioprio_set, which, who,

diff --git a/os/os.h b/os/os.h
index f2257a7c615f517cd89759cd44eeb1d53fdec35e..827b61e90d745211edb0635a028342ca7da42361 100644 (file)
--- a/os/os.h
+++ b/os/os.h
@@ -117,6 +117,9 @@ static inline int fio_cpus_split(os_cpu_mask_t *mask, unsigned int cpu_index)
 extern int fio_cpus_split(os_cpu_mask_t *mask, unsigned int cpu);
 #endif
 
+#ifndef FIO_HAVE_IOPRIO_CLASS
+#define ioprio_value_is_class_rt(prio) (false)
+#endif
 #ifndef FIO_HAVE_IOPRIO
 #define ioprio_value(prioclass, prio)  (0)
 #define ioprio_set(which, who, prioclass, prio)        (0)

diff --git a/server.h b/server.h
index daed057acb855472805971d0620e57fbe79b7d40..3ff32d9a5318b03e2da68ecf20de0287d0a9fbe2 100644 (file)
--- a/server.h
+++ b/server.h
@@ -48,7 +48,7 @@ struct fio_net_cmd_reply {
 };
 
 enum {
-       FIO_SERVER_VER                  = 92,
+       FIO_SERVER_VER                  = 93,
 
        FIO_SERVER_MAX_FRAGMENT_PDU     = 1024,
        FIO_SERVER_MAX_CMD_MB           = 2048,
@@ -193,6 +193,7 @@ struct cmd_iolog_pdu {
        uint32_t log_type;
        uint32_t compressed;
        uint32_t log_offset;
+       uint32_t log_prio;
        uint32_t log_hist_coarseness;
        uint8_t name[FIO_NET_NAME_MAX];
        struct io_sample samples[0];

diff --git a/stat.c b/stat.c
index a8a96c85a4120b0d70dee939c9102fe340565aae..99275620b69d7d5fcac471fbe0426b65d96534c2 100644 (file)
--- a/stat.c
+++ b/stat.c
@@ -2860,7 +2860,8 @@ static struct io_logs *get_cur_log(struct io_log *iolog)
 
 static void __add_log_sample(struct io_log *iolog, union io_sample_data data,
                             enum fio_ddir ddir, unsigned long long bs,
-                            unsigned long t, uint64_t offset, uint8_t priority_bit)
+                            unsigned long t, uint64_t offset,
+                            unsigned int priority)
 {
        struct io_logs *cur_log;
 
@@ -2879,7 +2880,7 @@ static void __add_log_sample(struct io_log *iolog, union io_sample_data data,
                s->time = t + (iolog->td ? iolog->td->unix_epoch : 0);
                io_sample_set_ddir(iolog, s, ddir);
                s->bs = bs;
-               s->priority_bit = priority_bit;
+               s->priority = priority;
 
                if (iolog->log_offset) {
                        struct io_sample_offset *so = (void *) s;
@@ -2956,7 +2957,7 @@ void reset_io_stats(struct thread_data *td)
 }
 
 static void __add_stat_to_log(struct io_log *iolog, enum fio_ddir ddir,
-                             unsigned long elapsed, bool log_max, uint8_t priority_bit)
+                             unsigned long elapsed, bool log_max)
 {
        /*
         * Note an entry in the log. Use the mean from the logged samples,
@@ -2971,26 +2972,26 @@ static void __add_stat_to_log(struct io_log *iolog, enum fio_ddir ddir,
                else
                        data.val = iolog->avg_window[ddir].mean.u.f + 0.50;
 
-               __add_log_sample(iolog, data, ddir, 0, elapsed, 0, priority_bit);
+               __add_log_sample(iolog, data, ddir, 0, elapsed, 0, 0);
        }
 
        reset_io_stat(&iolog->avg_window[ddir]);
 }
 
 static void _add_stat_to_log(struct io_log *iolog, unsigned long elapsed,
-                            bool log_max, uint8_t priority_bit)
+                            bool log_max)
 {
        int ddir;
 
        for (ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++)
-               __add_stat_to_log(iolog, ddir, elapsed, log_max, priority_bit);
+               __add_stat_to_log(iolog, ddir, elapsed, log_max);
 }
 
 static unsigned long add_log_sample(struct thread_data *td,
                                    struct io_log *iolog,
                                    union io_sample_data data,
                                    enum fio_ddir ddir, unsigned long long bs,
-                                   uint64_t offset, uint8_t priority_bit)
+                                   uint64_t offset, unsigned int ioprio)
 {
        unsigned long elapsed, this_window;
 
@@ -3003,7 +3004,8 @@ static unsigned long add_log_sample(struct thread_data *td,
         * If no time averaging, just add the log sample.
         */
        if (!iolog->avg_msec) {
-               __add_log_sample(iolog, data, ddir, bs, elapsed, offset, priority_bit);
+               __add_log_sample(iolog, data, ddir, bs, elapsed, offset,
+                                ioprio);
                return 0;
        }
 
@@ -3027,7 +3029,7 @@ static unsigned long add_log_sample(struct thread_data *td,
                        return diff;
        }
 
-       __add_stat_to_log(iolog, ddir, elapsed, td->o.log_max != 0, priority_bit);
+       __add_stat_to_log(iolog, ddir, elapsed, td->o.log_max != 0);
 
        iolog->avg_last[ddir] = elapsed - (elapsed % iolog->avg_msec);
 
@@ -3041,19 +3043,19 @@ void finalize_logs(struct thread_data *td, bool unit_logs)
        elapsed = mtime_since_now(&td->epoch);
 
        if (td->clat_log && unit_logs)
-               _add_stat_to_log(td->clat_log, elapsed, td->o.log_max != 0, 0);
+               _add_stat_to_log(td->clat_log, elapsed, td->o.log_max != 0);
        if (td->slat_log && unit_logs)
-               _add_stat_to_log(td->slat_log, elapsed, td->o.log_max != 0, 0);
+               _add_stat_to_log(td->slat_log, elapsed, td->o.log_max != 0);
        if (td->lat_log && unit_logs)
-               _add_stat_to_log(td->lat_log, elapsed, td->o.log_max != 0, 0);
+               _add_stat_to_log(td->lat_log, elapsed, td->o.log_max != 0);
        if (td->bw_log && (unit_logs == per_unit_log(td->bw_log)))
-               _add_stat_to_log(td->bw_log, elapsed, td->o.log_max != 0, 0);
+               _add_stat_to_log(td->bw_log, elapsed, td->o.log_max != 0);
        if (td->iops_log && (unit_logs == per_unit_log(td->iops_log)))
-               _add_stat_to_log(td->iops_log, elapsed, td->o.log_max != 0, 0);
+               _add_stat_to_log(td->iops_log, elapsed, td->o.log_max != 0);
 }
 
-void add_agg_sample(union io_sample_data data, enum fio_ddir ddir, unsigned long long bs,
-                                       uint8_t priority_bit)
+void add_agg_sample(union io_sample_data data, enum fio_ddir ddir,
+                   unsigned long long bs)
 {
        struct io_log *iolog;
 
@@ -3061,7 +3063,7 @@ void add_agg_sample(union io_sample_data data, enum fio_ddir ddir, unsigned long
                return;
 
        iolog = agg_io_log[ddir];
-       __add_log_sample(iolog, data, ddir, bs, mtime_since_genesis(), 0, priority_bit);
+       __add_log_sample(iolog, data, ddir, bs, mtime_since_genesis(), 0, 0);
 }
 
 void add_sync_clat_sample(struct thread_stat *ts, unsigned long long nsec)
@@ -3083,14 +3085,14 @@ static void add_lat_percentile_sample_noprio(struct thread_stat *ts,
 }
 
 static void add_lat_percentile_sample(struct thread_stat *ts,
-                               unsigned long long nsec, enum fio_ddir ddir, uint8_t priority_bit,
-                               enum fio_lat lat)
+                               unsigned long long nsec, enum fio_ddir ddir,
+                               bool high_prio, enum fio_lat lat)
 {
        unsigned int idx = plat_val_to_idx(nsec);
 
        add_lat_percentile_sample_noprio(ts, nsec, ddir, lat);
 
-       if (!priority_bit)
+       if (!high_prio)
                ts->io_u_plat_low_prio[ddir][idx]++;
        else
                ts->io_u_plat_high_prio[ddir][idx]++;
@@ -3098,7 +3100,7 @@ static void add_lat_percentile_sample(struct thread_stat *ts,
 
 void add_clat_sample(struct thread_data *td, enum fio_ddir ddir,
                     unsigned long long nsec, unsigned long long bs,
-                    uint64_t offset, uint8_t priority_bit)
+                    uint64_t offset, unsigned int ioprio, bool high_prio)
 {
        const bool needs_lock = td_async_processing(td);
        unsigned long elapsed, this_window;
@@ -3111,7 +3113,7 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir,
        add_stat_sample(&ts->clat_stat[ddir], nsec);
 
        if (!ts->lat_percentiles) {
-               if (priority_bit)
+               if (high_prio)
                        add_stat_sample(&ts->clat_high_prio_stat[ddir], nsec);
                else
                        add_stat_sample(&ts->clat_low_prio_stat[ddir], nsec);
@@ -3119,13 +3121,13 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir,
 
        if (td->clat_log)
                add_log_sample(td, td->clat_log, sample_val(nsec), ddir, bs,
-                              offset, priority_bit);
+                              offset, ioprio);
 
        if (ts->clat_percentiles) {
                if (ts->lat_percentiles)
                        add_lat_percentile_sample_noprio(ts, nsec, ddir, FIO_CLAT);
                else
-                       add_lat_percentile_sample(ts, nsec, ddir, priority_bit, FIO_CLAT);
+                       add_lat_percentile_sample(ts, nsec, ddir, high_prio, FIO_CLAT);
        }
 
        if (iolog && iolog->hist_msec) {
@@ -3154,7 +3156,7 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir,
                                FIO_IO_U_PLAT_NR * sizeof(uint64_t));
                        flist_add(&dst->list, &hw->list);
                        __add_log_sample(iolog, sample_plat(dst), ddir, bs,
-                                               elapsed, offset, priority_bit);
+                                        elapsed, offset, ioprio);
 
                        /*
                         * Update the last time we recorded as being now, minus
@@ -3171,8 +3173,8 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir,
 }
 
 void add_slat_sample(struct thread_data *td, enum fio_ddir ddir,
-                       unsigned long long nsec, unsigned long long bs, uint64_t offset,
-                       uint8_t priority_bit)
+                    unsigned long long nsec, unsigned long long bs,
+                    uint64_t offset, unsigned int ioprio)
 {
        const bool needs_lock = td_async_processing(td);
        struct thread_stat *ts = &td->ts;
@@ -3186,8 +3188,8 @@ void add_slat_sample(struct thread_data *td, enum fio_ddir ddir,
        add_stat_sample(&ts->slat_stat[ddir], nsec);
 
        if (td->slat_log)
-               add_log_sample(td, td->slat_log, sample_val(nsec), ddir, bs, offset,
-                       priority_bit);
+               add_log_sample(td, td->slat_log, sample_val(nsec), ddir, bs,
+                              offset, ioprio);
 
        if (ts->slat_percentiles)
                add_lat_percentile_sample_noprio(ts, nsec, ddir, FIO_SLAT);
@@ -3198,7 +3200,7 @@ void add_slat_sample(struct thread_data *td, enum fio_ddir ddir,
 
 void add_lat_sample(struct thread_data *td, enum fio_ddir ddir,
                    unsigned long long nsec, unsigned long long bs,
-                   uint64_t offset, uint8_t priority_bit)
+                   uint64_t offset, unsigned int ioprio, bool high_prio)
 {
        const bool needs_lock = td_async_processing(td);
        struct thread_stat *ts = &td->ts;
@@ -3213,11 +3215,11 @@ void add_lat_sample(struct thread_data *td, enum fio_ddir ddir,
 
        if (td->lat_log)
                add_log_sample(td, td->lat_log, sample_val(nsec), ddir, bs,
-                              offset, priority_bit);
+                              offset, ioprio);
 
        if (ts->lat_percentiles) {
-               add_lat_percentile_sample(ts, nsec, ddir, priority_bit, FIO_LAT);
-               if (priority_bit)
+               add_lat_percentile_sample(ts, nsec, ddir, high_prio, FIO_LAT);
+               if (high_prio)
                        add_stat_sample(&ts->clat_high_prio_stat[ddir], nsec);
                else
                        add_stat_sample(&ts->clat_low_prio_stat[ddir], nsec);
@@ -3246,7 +3248,7 @@ void add_bw_sample(struct thread_data *td, struct io_u *io_u,
 
        if (td->bw_log)
                add_log_sample(td, td->bw_log, sample_val(rate), io_u->ddir,
-                              bytes, io_u->offset, io_u_is_prio(io_u));
+                              bytes, io_u->offset, io_u->ioprio);
 
        td->stat_io_bytes[io_u->ddir] = td->this_io_bytes[io_u->ddir];
 
@@ -3300,7 +3302,8 @@ static int __add_samples(struct thread_data *td, struct timespec *parent_tv,
                        if (td->o.min_bs[ddir] == td->o.max_bs[ddir])
                                bs = td->o.min_bs[ddir];
 
-                       next = add_log_sample(td, log, sample_val(rate), ddir, bs, 0, 0);
+                       next = add_log_sample(td, log, sample_val(rate), ddir,
+                                             bs, 0, 0);
                        next_log = min(next_log, next);
                }
 
@@ -3340,7 +3343,7 @@ void add_iops_sample(struct thread_data *td, struct io_u *io_u,
 
        if (td->iops_log)
                add_log_sample(td, td->iops_log, sample_val(1), io_u->ddir,
-                              bytes, io_u->offset, io_u_is_prio(io_u));
+                              bytes, io_u->offset, io_u->ioprio);
 
        td->stat_io_blocks[io_u->ddir] = td->this_io_blocks[io_u->ddir];
 

diff --git a/stat.h b/stat.h
index d08d4dc09780720eea3f1fa5896d12a11bae5e40..a06237e7a41e9f5f0e287786509df6fd5f578301 100644 (file)
--- a/stat.h
+++ b/stat.h
@@ -341,13 +341,12 @@ extern void update_rusage_stat(struct thread_data *);
 extern void clear_rusage_stat(struct thread_data *);
 
 extern void add_lat_sample(struct thread_data *, enum fio_ddir, unsigned long long,
-                               unsigned long long, uint64_t, uint8_t);
+                          unsigned long long, uint64_t, unsigned int, bool);
 extern void add_clat_sample(struct thread_data *, enum fio_ddir, unsigned long long,
-                               unsigned long long, uint64_t, uint8_t);
+                           unsigned long long, uint64_t, unsigned int, bool);
 extern void add_slat_sample(struct thread_data *, enum fio_ddir, unsigned long long,
-                               unsigned long long, uint64_t, uint8_t);
-extern void add_agg_sample(union io_sample_data, enum fio_ddir, unsigned long long bs,
-                               uint8_t priority_bit);
+                               unsigned long long, uint64_t, unsigned int);
+extern void add_agg_sample(union io_sample_data, enum fio_ddir, unsigned long long);
 extern void add_iops_sample(struct thread_data *, struct io_u *,
                                unsigned int);
 extern void add_bw_sample(struct thread_data *, struct io_u *,

diff --git a/thread_options.h b/thread_options.h
index 7133faf66b355795c0ebc06877b5cf2bf1f1d368..9990ab9b6f721dd2fbb2068c8a1f98e698b32e18 100644 (file)
--- a/thread_options.h
+++ b/thread_options.h
@@ -374,6 +374,8 @@ struct thread_options {
        unsigned int ignore_zone_limits;
        fio_fp64_t zrt;
        fio_fp64_t zrf;
+
+       unsigned int log_prio;
 };
 
 #define FIO_TOP_STR_MAX                256
@@ -677,6 +679,8 @@ struct thread_options_pack {
        uint32_t zone_mode;
        int32_t max_open_zones;
        uint32_t ignore_zone_limits;
+
+       uint32_t log_prio;
 } __attribute__((packed));
 
 extern void convert_thread_options_to_cpu(struct thread_options *o, struct thread_options_pack *top);