Note that size needs to be explicitly provided and only 1 file per
job is supported
+.. option:: dedupe_global=bool
+
+ This controls whether the deduplication buffers will be shared amongst
+ all jobs that have this option set. The buffers are spread evenly between
+ participating jobs.
+
.. option:: invalidate=bool
Invalidate the buffer/page cache parts of the files to be used prior to
setup_log(&agg_io_log[DDIR_TRIM], &p, "agg-trim_bw.log");
}
+ if (init_global_dedupe_working_set_seeds()) {
+ log_err("fio: failed to initialize global dedupe working set\n");
+ return 1;
+ }
+
startup_sem = fio_sem_init(FIO_SEM_LOCKED);
if (!sk_out)
is_local_backend = true;
o->dedupe_percentage = le32_to_cpu(top->dedupe_percentage);
o->dedupe_mode = le32_to_cpu(top->dedupe_mode);
o->dedupe_working_set_percentage = le32_to_cpu(top->dedupe_working_set_percentage);
+ o->dedupe_global = le32_to_cpu(top->dedupe_global);
o->block_error_hist = le32_to_cpu(top->block_error_hist);
o->replay_align = le32_to_cpu(top->replay_align);
o->replay_scale = le32_to_cpu(top->replay_scale);
top->dedupe_percentage = cpu_to_le32(o->dedupe_percentage);
top->dedupe_mode = cpu_to_le32(o->dedupe_mode);
top->dedupe_working_set_percentage = cpu_to_le32(o->dedupe_working_set_percentage);
+ top->dedupe_global = cpu_to_le32(o->dedupe_global);
top->block_error_hist = cpu_to_le32(o->block_error_hist);
top->replay_align = cpu_to_le32(o->replay_align);
top->replay_scale = cpu_to_le32(o->replay_scale);
#include "fio.h"
-int init_dedupe_working_set_seeds(struct thread_data *td)
+/**
+ * initializes the global dedup workset.
+ * this needs to be called after all jobs' seeds
+ * have been initialized
+ */
+int init_global_dedupe_working_set_seeds(void)
{
- unsigned long long i, j, num_seed_advancements;
+ int i;
+ struct thread_data *td;
+
+ for_each_td(td, i) {
+ if (!td->o.dedupe_global)
+ continue;
+
+ if (init_dedupe_working_set_seeds(td, 1))
+ return 1;
+ }
+
+ return 0;
+}
+
+int init_dedupe_working_set_seeds(struct thread_data *td, bool global_dedup)
+{
+ int tindex;
+ struct thread_data *td_seed;
+ unsigned long long i, j, num_seed_advancements, pages_per_seed;
struct frand_state dedupe_working_set_state = {0};
if (!td->o.dedupe_percentage || !(td->o.dedupe_mode == DEDUPE_MODE_WORKING_SET))
return 0;
+ tindex = td->thread_number - 1;
num_seed_advancements = td->o.min_bs[DDIR_WRITE] /
min_not_zero(td->o.min_bs[DDIR_WRITE], (unsigned long long) td->o.compress_chunk);
/*
log_err("fio: could not allocate dedupe working set\n");
return 1;
}
+
frand_copy(&dedupe_working_set_state, &td->buf_state);
- for (i = 0; i < td->num_unique_pages; i++) {
- frand_copy(&td->dedupe_working_set_states[i], &dedupe_working_set_state);
+ frand_copy(&td->dedupe_working_set_states[0], &dedupe_working_set_state);
+ pages_per_seed = max(td->num_unique_pages / thread_number, 1ull);
+ for (i = 1; i < td->num_unique_pages; i++) {
/*
* When compression is used the seed is advanced multiple times to
* generate the buffer. We want to regenerate the same buffer when
*/
for (j = 0; j < num_seed_advancements; j++)
__get_next_seed(&dedupe_working_set_state);
+
+ /*
+ * When global dedup is used, we rotate the seeds to allow
+ * generating same buffers across different jobs. Deduplication buffers
+ * are spread evenly across jobs participating in global dedupe
+ */
+ if (global_dedup && i % pages_per_seed == 0) {
+ td_seed = tnumber_to_td(++tindex % thread_number);
+ frand_copy(&dedupe_working_set_state, &td_seed->buf_state);
+ }
+
+ frand_copy(&td->dedupe_working_set_states[i], &dedupe_working_set_state);
}
return 0;
#ifndef DEDUPE_H
#define DEDUPE_H
-int init_dedupe_working_set_seeds(struct thread_data *td);
+int init_dedupe_working_set_seeds(struct thread_data *td, bool global_dedupe);
+int init_global_dedupe_working_set_seeds(void);
#endif
per job is supported
.RE
.TP
+.BI dedupe_global \fR=\fPbool
+This controls whether the deduplication buffers will be shared amongst
+all jobs that have this option set. The buffers are spread evenly between
+participating jobs.
+.P
+.RS
+Note that \fBdedupe_mode\fR must be set to \fBworking_set\fR for this to work.
+Can be used in combination with compression
+.TP
.BI invalidate \fR=\fPbool
Invalidate the buffer/page cache parts of the files to be used prior to
starting I/O if the platform and file type support it. Defaults to true.
if (fixup_options(td))
goto err;
- if (init_dedupe_working_set_seeds(td))
+ if (!td->o.dedupe_global && init_dedupe_working_set_seeds(td, 0))
goto err;
/*
.category = FIO_OPT_C_IO,
.group = FIO_OPT_G_IO_BUF,
},
+ {
+ .name = "dedupe_global",
+ .lname = "Global deduplication",
+ .type = FIO_OPT_BOOL,
+ .off1 = offsetof(struct thread_options, dedupe_global),
+ .help = "Share deduplication buffers across jobs",
+ .def = "0",
+ .category = FIO_OPT_C_IO,
+ .group = FIO_OPT_G_IO_BUF,
+ },
{
.name = "dedupe_mode",
.lname = "Dedupe mode",
};
enum {
- FIO_SERVER_VER = 96,
+ FIO_SERVER_VER = 97,
FIO_SERVER_MAX_FRAGMENT_PDU = 1024,
FIO_SERVER_MAX_CMD_MB = 2048,
unsigned int dedupe_percentage;
unsigned int dedupe_mode;
unsigned int dedupe_working_set_percentage;
+ unsigned int dedupe_global;
unsigned int time_based;
unsigned int disable_lat;
unsigned int disable_clat;
uint32_t dedupe_percentage;
uint32_t dedupe_mode;
uint32_t dedupe_working_set_percentage;
+ uint32_t dedupe_global;
uint32_t time_based;
uint32_t disable_lat;
uint32_t disable_clat;
uint32_t lat_percentiles;
uint32_t slat_percentiles;
uint32_t percentile_precision;
+ uint32_t pad5;
fio_fp64_t percentile_list[FIO_IO_U_LIST_MAX_LEN];
uint8_t read_iolog_file[FIO_TOP_STR_MAX];