2 * IO priority handling helper functions common to the libaio and io_uring
9 * Temporary array used during parsing. Will be freed after the corresponding
10 * struct bsprio_desc has been generated and saved in cmdprio->bsprio_desc.
12 struct cmdprio_parse_result {
13 struct split_prio *entries;
18 * Temporary array used during init. Will be freed after the corresponding
19 * struct clat_prio_stat array has been saved in td->ts.clat_prio and the
20 * matching clat_prio_indexes have been saved in each struct cmdprio_prio.
22 struct cmdprio_values {
27 static int find_clat_prio_index(unsigned int *all_prios, int nr_prios,
32 for (i = 0; i < nr_prios; i++) {
33 if (all_prios[i] == prio)
41 * assign_clat_prio_index - In order to avoid stat.c the need to loop through
42 * all possible priorities each time add_clat_sample() / add_lat_sample() is
43 * called, save which index to use in each cmdprio_prio. This will later be
44 * propagated to the io_u, if the specific io_u was determined to use a cmdprio
47 static void assign_clat_prio_index(struct cmdprio_prio *prio,
48 struct cmdprio_values *values)
50 int clat_prio_index = find_clat_prio_index(values->prios,
53 if (clat_prio_index == -1) {
54 clat_prio_index = values->nr_prios;
55 values->prios[clat_prio_index] = prio->prio;
58 prio->clat_prio_index = clat_prio_index;
62 * init_cmdprio_values - Allocate a temporary array that can hold all unique
63 * priorities (per ddir), so that we can assign_clat_prio_index() for each
64 * cmdprio_prio during setup. This temporary array is freed after setup.
66 static int init_cmdprio_values(struct cmdprio_values *values,
67 int max_unique_prios, struct thread_stat *ts)
69 values->prios = calloc(max_unique_prios + 1,
70 sizeof(*values->prios));
74 /* td->ioprio/ts->ioprio is always stored at index 0. */
75 values->prios[0] = ts->ioprio;
82 * init_ts_clat_prio - Allocates and fills a clat_prio_stat array which holds
83 * all unique priorities (per ddir).
85 static int init_ts_clat_prio(struct thread_stat *ts, enum fio_ddir ddir,
86 struct cmdprio_values *values)
90 if (alloc_clat_prio_stat_ddir(ts, ddir, values->nr_prios))
93 for (i = 0; i < values->nr_prios; i++)
94 ts->clat_prio[ddir][i].ioprio = values->prios[i];
99 static int fio_cmdprio_fill_bsprio(struct cmdprio_bsprio *bsprio,
100 struct split_prio *entries,
101 struct cmdprio_values *values,
102 int implicit_cmdprio, int start, int end)
104 struct cmdprio_prio *prio;
105 int i = end - start + 1;
107 bsprio->prios = calloc(i, sizeof(*bsprio->prios));
111 bsprio->bs = entries[start].bs;
112 bsprio->nr_prios = 0;
113 for (i = start; i <= end; i++) {
114 prio = &bsprio->prios[bsprio->nr_prios];
115 prio->perc = entries[i].perc;
116 if (entries[i].prio == -1)
117 prio->prio = implicit_cmdprio;
119 prio->prio = entries[i].prio;
120 assign_clat_prio_index(prio, values);
121 bsprio->tot_perc += entries[i].perc;
122 if (bsprio->tot_perc > 100) {
123 log_err("fio: cmdprio_bssplit total percentage "
124 "for bs: %"PRIu64" exceeds 100\n",
136 fio_cmdprio_generate_bsprio_desc(struct cmdprio_bsprio_desc *bsprio_desc,
137 struct cmdprio_parse_result *parse_res,
138 struct cmdprio_values *values,
139 int implicit_cmdprio)
141 struct split_prio *entries = parse_res->entries;
142 int nr_entries = parse_res->nr_entries;
143 struct cmdprio_bsprio *bsprio;
144 int i, start, count = 0;
147 * The parsed result is sorted by blocksize, so count only the number
148 * of different blocksizes, to know how many cmdprio_bsprio we need.
150 for (i = 0; i < nr_entries; i++) {
151 while (i + 1 < nr_entries && entries[i].bs == entries[i + 1].bs)
157 * This allocation is not freed on error. Instead, the calling function
158 * is responsible for calling fio_cmdprio_cleanup() on error.
160 bsprio_desc->bsprios = calloc(count, sizeof(*bsprio_desc->bsprios));
161 if (!bsprio_desc->bsprios)
165 bsprio_desc->nr_bsprios = 0;
166 for (i = 0; i < nr_entries; i++) {
167 while (i + 1 < nr_entries && entries[i].bs == entries[i + 1].bs)
169 bsprio = &bsprio_desc->bsprios[bsprio_desc->nr_bsprios];
171 * All parsed entries with the same blocksize get saved in the
172 * same cmdprio_bsprio, to expedite the search in the hot path.
174 if (fio_cmdprio_fill_bsprio(bsprio, entries, values,
175 implicit_cmdprio, start, i))
179 bsprio_desc->nr_bsprios++;
185 static int fio_cmdprio_bssplit_ddir(struct thread_options *to, void *cb_arg,
186 enum fio_ddir ddir, char *str, bool data)
188 struct cmdprio_parse_result *parse_res_arr = cb_arg;
189 struct cmdprio_parse_result *parse_res = &parse_res_arr[ddir];
191 if (ddir == DDIR_TRIM)
194 if (split_parse_prio_ddir(to, &parse_res->entries,
195 &parse_res->nr_entries, str))
201 static int fio_cmdprio_bssplit_parse(struct thread_data *td, const char *input,
202 struct cmdprio_parse_result *parse_res)
207 p = str = strdup(input);
209 strip_blank_front(&str);
210 strip_blank_end(str);
212 ret = str_split_parse(td, str, fio_cmdprio_bssplit_ddir, parse_res,
220 * fio_cmdprio_percentage - Returns the percentage of I/Os that should
221 * use a cmdprio priority value (rather than the default context priority).
223 * For CMDPRIO_MODE_BSSPLIT, if the percentage is non-zero, we will also
224 * return the matching bsprio, to avoid the same linear search elsewhere.
225 * For CMDPRIO_MODE_PERC, we will never return a bsprio.
227 static int fio_cmdprio_percentage(struct cmdprio *cmdprio, struct io_u *io_u,
228 struct cmdprio_bsprio **bsprio)
230 struct cmdprio_bsprio *bsprio_entry;
231 enum fio_ddir ddir = io_u->ddir;
234 switch (cmdprio->mode) {
235 case CMDPRIO_MODE_PERC:
237 return cmdprio->perc_entry[ddir].perc;
238 case CMDPRIO_MODE_BSSPLIT:
239 for (i = 0; i < cmdprio->bsprio_desc[ddir].nr_bsprios; i++) {
240 bsprio_entry = &cmdprio->bsprio_desc[ddir].bsprios[i];
241 if (bsprio_entry->bs == io_u->buflen) {
242 *bsprio = bsprio_entry;
243 return bsprio_entry->tot_perc;
249 * An I/O engine should never call this function if cmdprio
256 * This is totally fine, the given blocksize simply does not
257 * have any (non-zero) cmdprio_bssplit entries defined.
264 * fio_cmdprio_set_ioprio - Set an io_u ioprio according to cmdprio options
266 * Generates a random percentage value to determine if an io_u ioprio needs
267 * to be set. If the random percentage value is within the user specified
268 * percentage of I/Os that should use a cmdprio priority value (rather than
269 * the default priority), then this function updates the io_u with an ioprio
270 * value as defined by the cmdprio/cmdprio_hint/cmdprio_class or
271 * cmdprio_bssplit options.
273 * Return true if the io_u ioprio was changed and false otherwise.
275 bool fio_cmdprio_set_ioprio(struct thread_data *td, struct cmdprio *cmdprio,
278 struct cmdprio_bsprio *bsprio;
279 unsigned int p, rand;
283 p = fio_cmdprio_percentage(cmdprio, io_u, &bsprio);
287 rand = rand_between(&td->prio_state, 0, 99);
291 switch (cmdprio->mode) {
292 case CMDPRIO_MODE_PERC:
293 io_u->ioprio = cmdprio->perc_entry[io_u->ddir].prio;
294 io_u->clat_prio_index =
295 cmdprio->perc_entry[io_u->ddir].clat_prio_index;
297 case CMDPRIO_MODE_BSSPLIT:
299 for (i = 0; i < bsprio->nr_prios; i++) {
300 struct cmdprio_prio *prio = &bsprio->prios[i];
304 io_u->ioprio = prio->prio;
305 io_u->clat_prio_index = prio->clat_prio_index;
314 /* When rand < p (total perc), we should always find a cmdprio_prio. */
319 static int fio_cmdprio_gen_perc(struct thread_data *td, struct cmdprio *cmdprio)
321 struct cmdprio_options *options = cmdprio->options;
322 struct cmdprio_prio *prio;
323 struct cmdprio_values values[CMDPRIO_RWDIR_CNT] = {};
324 struct thread_stat *ts = &td->ts;
328 for (ddir = 0; ddir < CMDPRIO_RWDIR_CNT; ddir++) {
330 * Do not allocate a clat_prio array nor set the cmdprio struct
331 * if zero percent of the I/Os (for the ddir) should use a
332 * cmdprio priority value, or when the ddir is not enabled.
334 if (!options->percentage[ddir] ||
335 (ddir == DDIR_READ && !td_read(td)) ||
336 (ddir == DDIR_WRITE && !td_write(td)))
339 ret = init_cmdprio_values(&values[ddir], 1, ts);
343 prio = &cmdprio->perc_entry[ddir];
344 prio->perc = options->percentage[ddir];
345 prio->prio = ioprio_value(options->class[ddir],
346 options->level[ddir],
347 options->hint[ddir]);
348 assign_clat_prio_index(prio, &values[ddir]);
350 ret = init_ts_clat_prio(ts, ddir, &values[ddir]);
354 free(values[ddir].prios);
355 values[ddir].prios = NULL;
356 values[ddir].nr_prios = 0;
362 for (ddir = 0; ddir < CMDPRIO_RWDIR_CNT; ddir++)
363 free(values[ddir].prios);
364 free_clat_prio_stats(ts);
369 static int fio_cmdprio_parse_and_gen_bssplit(struct thread_data *td,
370 struct cmdprio *cmdprio)
372 struct cmdprio_options *options = cmdprio->options;
373 struct cmdprio_parse_result parse_res[CMDPRIO_RWDIR_CNT] = {};
374 struct cmdprio_values values[CMDPRIO_RWDIR_CNT] = {};
375 struct thread_stat *ts = &td->ts;
376 int ret, implicit_cmdprio;
379 ret = fio_cmdprio_bssplit_parse(td, options->bssplit_str,
384 for (ddir = 0; ddir < CMDPRIO_RWDIR_CNT; ddir++) {
386 * Do not allocate a clat_prio array nor set the cmdprio structs
387 * if there are no non-zero entries (for the ddir), or when the
388 * ddir is not enabled.
390 if (!parse_res[ddir].nr_entries ||
391 (ddir == DDIR_READ && !td_read(td)) ||
392 (ddir == DDIR_WRITE && !td_write(td))) {
393 free(parse_res[ddir].entries);
394 parse_res[ddir].entries = NULL;
395 parse_res[ddir].nr_entries = 0;
399 ret = init_cmdprio_values(&values[ddir],
400 parse_res[ddir].nr_entries, ts);
404 implicit_cmdprio = ioprio_value(options->class[ddir],
405 options->level[ddir],
406 options->hint[ddir]);
408 ret = fio_cmdprio_generate_bsprio_desc(&cmdprio->bsprio_desc[ddir],
415 free(parse_res[ddir].entries);
416 parse_res[ddir].entries = NULL;
417 parse_res[ddir].nr_entries = 0;
419 ret = init_ts_clat_prio(ts, ddir, &values[ddir]);
423 free(values[ddir].prios);
424 values[ddir].prios = NULL;
425 values[ddir].nr_prios = 0;
431 for (ddir = 0; ddir < CMDPRIO_RWDIR_CNT; ddir++) {
432 free(parse_res[ddir].entries);
433 free(values[ddir].prios);
435 free_clat_prio_stats(ts);
436 fio_cmdprio_cleanup(cmdprio);
441 static int fio_cmdprio_parse_and_gen(struct thread_data *td,
442 struct cmdprio *cmdprio)
444 struct cmdprio_options *options = cmdprio->options;
448 * If cmdprio_percentage/cmdprio_bssplit is set and cmdprio_class
449 * is not set, default to RT priority class.
451 for (i = 0; i < CMDPRIO_RWDIR_CNT; i++) {
453 * A cmdprio value is only used when fio_cmdprio_percentage()
454 * returns non-zero, so it is safe to set a class even for a
455 * DDIR that will never use it.
457 if (!options->class[i])
458 options->class[i] = IOPRIO_CLASS_RT;
461 switch (cmdprio->mode) {
462 case CMDPRIO_MODE_BSSPLIT:
463 ret = fio_cmdprio_parse_and_gen_bssplit(td, cmdprio);
465 case CMDPRIO_MODE_PERC:
466 ret = fio_cmdprio_gen_perc(td, cmdprio);
476 void fio_cmdprio_cleanup(struct cmdprio *cmdprio)
481 for (ddir = 0; ddir < CMDPRIO_RWDIR_CNT; ddir++) {
482 for (i = 0; i < cmdprio->bsprio_desc[ddir].nr_bsprios; i++)
483 free(cmdprio->bsprio_desc[ddir].bsprios[i].prios);
484 free(cmdprio->bsprio_desc[ddir].bsprios);
485 cmdprio->bsprio_desc[ddir].bsprios = NULL;
486 cmdprio->bsprio_desc[ddir].nr_bsprios = 0;
490 * options points to a cmdprio_options struct that is part of td->eo.
491 * td->eo itself will be freed by free_ioengine().
493 cmdprio->options = NULL;
496 int fio_cmdprio_init(struct thread_data *td, struct cmdprio *cmdprio,
497 struct cmdprio_options *options)
499 struct thread_options *to = &td->o;
500 bool has_cmdprio_percentage = false;
501 bool has_cmdprio_bssplit = false;
504 cmdprio->options = options;
506 if (options->bssplit_str && strlen(options->bssplit_str))
507 has_cmdprio_bssplit = true;
509 for (i = 0; i < CMDPRIO_RWDIR_CNT; i++) {
510 if (options->percentage[i])
511 has_cmdprio_percentage = true;
515 * Check for option conflicts
517 if (has_cmdprio_percentage && has_cmdprio_bssplit) {
518 log_err("%s: cmdprio_percentage and cmdprio_bssplit options "
519 "are mutually exclusive\n",
524 if (has_cmdprio_bssplit)
525 cmdprio->mode = CMDPRIO_MODE_BSSPLIT;
526 else if (has_cmdprio_percentage)
527 cmdprio->mode = CMDPRIO_MODE_PERC;
529 cmdprio->mode = CMDPRIO_MODE_NONE;
531 /* Nothing left to do if cmdprio is not used */
532 if (cmdprio->mode == CMDPRIO_MODE_NONE)
535 return fio_cmdprio_parse_and_gen(td, cmdprio);