Recent changes (master)
The following changes since commit 5f2d43188c2d65674aaba6280e2a87107e5d7099:
Merge branch 'fix/json/strdup_memory_leak' of https://github.com/dpronin/fio (2022-04-17 16:47:22 -0600)
are available in the Git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 6e594a2fa8388892dffb2ffc9b865689e2d67833:
Merge branch 'global_dedup' of https://github.com/bardavid/fio (2022-04-29 16:30:50 -0600)
----------------------------------------------------------------
Bar David (2):
Introducing support for generation of dedup buffers across jobs. The dedup buffers are spread evenly between the jobs that enabled the dedupe_global option adding an example for dedupe_global usage and DRR testing
Jens Axboe (1):
Merge branch 'global_dedup' of https://github.com/bardavid/fio
HOWTO.rst | 6 +++++ backend.c | 5 ++++ cconv.c | 2 ++ dedupe.c | 46 +++++++++++++++++++++++++++++++++---- dedupe.h | 3 ++- examples/dedupe-global.fio | 57 ++++++++++++++++++++++++++++++++++++++++++++++ fio.1 | 9 ++++++++ init.c | 2 +- options.c | 10 ++++++++ server.h | 2 +- thread_options.h | 3 +++ 11 files changed, 138 insertions(+), 7 deletions(-) create mode 100644 examples/dedupe-global.fio
---
Diff of recent changes:
HOWTO.rst +6 -0
@@ -1749,6 +1749,12 @@ Buffers and memory Note that size needs to be explicitly provided and only 1 file per job is supported +.. option:: dedupe_global=bool + + This controls whether the deduplication buffers will be shared amongst + all jobs that have this option set. The buffers are spread evenly between + participating jobs. + .. option:: invalidate=bool Invalidate the buffer/page cache parts of the files to be used prior to
backend.c +5 -0
@@ -2570,6 +2570,11 @@ int fio_backend(struct sk_out *sk_out)setup_log (&agg_io_log[DDIR_TRIM ], &p,"agg-trim_bw.log" ); } +if (init_global_dedupe_working_set_seeds ()) { +log_err ("fio: failed to initialize global dedupe working set\n" ); +return 1 ; + } + startup_sem =fio_sem_init (FIO_SEM_LOCKED );if (!sk_out) is_local_backend = true;
cconv.c +2 -0
@@ -305,6 +305,7 @@ void convert_thread_options_to_cpu(struct thread_options *o, o->dedupe_percentage =le32_to_cpu (top->dedupe_percentage); o->dedupe_mode =le32_to_cpu (top->dedupe_mode); o->dedupe_working_set_percentage =le32_to_cpu (top->dedupe_working_set_percentage); + o->dedupe_global =le32_to_cpu (top->dedupe_global); o->block_error_hist =le32_to_cpu (top->block_error_hist); o->replay_align =le32_to_cpu (top->replay_align); o->replay_scale =le32_to_cpu (top->replay_scale);
@@ -513,6 +514,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top, top->dedupe_percentage =cpu_to_le32 (o->dedupe_percentage); top->dedupe_mode =cpu_to_le32 (o->dedupe_mode); top->dedupe_working_set_percentage =cpu_to_le32 (o->dedupe_working_set_percentage); + top->dedupe_global =cpu_to_le32 (o->dedupe_global); top->block_error_hist =cpu_to_le32 (o->block_error_hist); top->replay_align =cpu_to_le32 (o->replay_align); top->replay_scale =cpu_to_le32 (o->replay_scale);
dedupe.c +42 -4
70 lines not on this page · raw message
dedupe.h +2 -1
@@ -1,6 +1,7 @@#ifndef DEDUPE_H #define DEDUPE_H -int init_dedupe_working_set_seeds (struct thread_data *td); +int init_dedupe_working_set_seeds (struct thread_data *td,bool global_dedupe); +int init_global_dedupe_working_set_seeds (void );#endif
examples/dedupe-global.fio new file +57 -0
57 lines not on this page · raw message
fio.1 +9 -0
@@ -1553,6 +1553,15 @@ Note that \fBsize\fR needs to be explicitly provided and only 1 file per job is supported .RE .TP +.BI dedupe_global \fR=\fPbool +This controls whether the deduplication buffers will be shared amongst +all jobs that have this option set. The buffers are spread evenly between +participating jobs. +.P +.RS +Note that \fBdedupe_mode\fR must be set to \fBworking_set\fR for this to work. +Can be used in combination with compression +.TP .BI invalidate \fR=\fPbool Invalidate the buffer/page cache parts of the files to be used prior to starting I/O if the platform and file type support it. Defaults to true.
init.c +1 -1
@@ -1541,7 +1541,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num,if (fixup_options (td)) goto err; -if (init_dedupe_working_set_seeds (td)) +if (!td->o.dedupe_global &&init_dedupe_working_set_seeds (td,0 )) goto err;/*
options.c +10 -0
@@ -4665,6 +4665,16 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .category = FIO_OPT_C_IO, .group = FIO_OPT_G_IO_BUF, }, + { + .name = "dedupe_global", + .lname = "Global deduplication", + .type = FIO_OPT_BOOL, + .off1 = offsetof(struct thread_options, dedupe_global), + .help = "Share deduplication buffers across jobs", + .def = "0", + .category = FIO_OPT_C_IO, + .group = FIO_OPT_G_IO_BUF, + }, { .name = "dedupe_mode", .lname = "Dedupe mode",
server.h +1 -1
@@ -51,7 +51,7 @@ struct fio_net_cmd_reply { };enum { -FIO_SERVER_VER =96 , +FIO_SERVER_VER =97 ,FIO_SERVER_MAX_FRAGMENT_PDU =1024 ,FIO_SERVER_MAX_CMD_MB =2048 ,
thread_options.h +3 -0
@@ -263,6 +263,7 @@ struct thread_options {unsigned int dedupe_percentage;unsigned int dedupe_mode;unsigned int dedupe_working_set_percentage; +unsigned int dedupe_global;unsigned int time_based;unsigned int disable_lat;unsigned int disable_clat;
@@ -578,6 +579,7 @@ struct thread_options_pack {uint32_t dedupe_percentage;uint32_t dedupe_mode;uint32_t dedupe_working_set_percentage; +uint32_t dedupe_global;uint32_t time_based;uint32_t disable_lat;uint32_t disable_clat;
@@ -596,6 +598,7 @@ struct thread_options_pack {uint32_t lat_percentiles;uint32_t slat_percentiles;uint32_t percentile_precision; +uint32_t pad5;fio_fp64_t percentile_list[FIO_IO_U_LIST_MAX_LEN ];uint8_t read_iolog_file[FIO_TOP_STR_MAX ];