Recent changes (master)
The following changes since commit 9f51d89c683d70cd8ab23ba09ec6e628a548af5a:
Sync io_uring header with the kernel (2021-11-20 07:31:20 -0700)
are available in the Git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 1d08bfb018e600cc47f122fb78c02bf74b84dee8:
t/dedupe: style fixups (2021-11-21 06:51:11 -0700)
----------------------------------------------------------------
Bar David (2):
Mixed dedup and compression
fio-dedup: adjusted the binary to support compression
Jens Axboe (3):
Merge branch 'dedupe_and_compression' of https://github.com/bardavid/fio
t/io_uring: fix 32-bit compile warnings
t/dedupe: style fixups
DEDUPE-TODO | 3 -- dedupe.c | 12 ++++- io_u.c | 29 ++++++----- t/dedupe.c | 167 +++++++++++++++++++++++++++++++++++++++++++++++------------ t/io_uring.c | 4 +- 5 files changed, 161 insertions(+), 54 deletions(-)
---
Diff of recent changes:
DEDUPE-TODO +0 -3
@@ -1,6 +1,3 @@ -- Mixed buffers of dedupe-able and compressible data. - Major usecase in performance benchmarking of storage subsystems. - - Shifted dedup-able data. Allow for dedup buffer generation to shift contents by random number of sectors (fill the gaps with uncompressible data). Some storage
dedupe.c +10 -2
@@ -2,12 +2,14 @@int init_dedupe_working_set_seeds (struct thread_data *td) { -unsigned long long i; +unsigned long long i, j, num_seed_advancements;struct frand_state dedupe_working_set_state = {0 };if (!td->o.dedupe_percentage || !(td->o.dedupe_mode ==DEDUPE_MODE_WORKING_SET ))return 0 ; + num_seed_advancements = td->o.min_bs[DDIR_WRITE ] / +min_not_zero (td->o.min_bs[DDIR_WRITE ], (unsigned long long ) td->o.compress_chunk);/* * The dedupe working set keeps seeds of unique data (generated by buf_state). * Dedupe-ed pages will be generated using those seeds.
@@ -21,7 +23,13 @@ int init_dedupe_working_set_seeds(struct thread_data *td)frand_copy (&dedupe_working_set_state, &td->buf_state);for (i =0 ; i < td->num_unique_pages; i++) {frand_copy (&td->dedupe_working_set_states[i], &dedupe_working_set_state); -__get_next_seed (&dedupe_working_set_state); +/* +* When compression is used the seed is advanced multiple times to +* generate the buffer. We want to regenerate the same buffer when +* deduping against this page +*/ +for (j =0 ; j < num_seed_advancements; j++) +__get_next_seed (&dedupe_working_set_state); }return 0 ;
io_u.c +16 -13
@@ -2230,27 +2230,30 @@ void fill_io_buffer(struct thread_data *td, void *buf, unsigned long long min_wrif (o->compress_percentage || o->dedupe_percentage) {unsigned int perc = td->o.compress_percentage; -struct frand_state *rs; +struct frand_state *rs =NULL ;unsigned long long left = max_bs;unsigned long long this_write; do { - rs = get_buf_state(td); + /* + * Buffers are either entirely dedupe-able or not. + * If we choose to dedup, the buffer should undergo + * the same manipulation as the original write. Which + * means we should retrack the steps we took for compression + * as well. + */ + if (!rs) + rs = get_buf_state(td); min_write = min(min_write, left); - if (perc) { - this_write = min_not_zero(min_write, - (unsigned long long) td->o.compress_chunk); + this_write = min_not_zero(min_write, + (unsigned long long) td->o.compress_chunk); - fill_random_buf_percentage(rs, buf, perc, - this_write, this_write, - o->buffer_pattern, - o->buffer_pattern_bytes); - } else { - fill_random_buf(rs, buf, min_write); - this_write = min_write; - } + fill_random_buf_percentage(rs, buf, perc, + this_write, this_write, + o->buffer_pattern, + o->buffer_pattern_bytes); buf += this_write; left -= this_write;
t/dedupe.c +133 -34
381 lines not on this page · raw message
t/io_uring.c +2 -2
@@ -192,7 +192,7 @@ unsigned int calc_clat_percentiles(unsigned long *io_u_plat, unsigned long nr,unsigned long *ovals =NULL ;bool is_last; - *minv =-1ULL ; + *minv =-1UL ; *maxv =0 ; ovals =malloc (len *sizeof (*ovals));
@@ -498,7 +498,7 @@ static void init_io(struct submitter *s, unsigned index) sqe->off = offset; sqe->user_data = (unsigned long ) f->fileno;if (stats && stats_running) - sqe->user_data |= ((unsigned long )s->clock_index <<32 ); + sqe->user_data |= ((uint64_t )s->clock_index <<32 ); }static int prep_more_ios_uring (struct submitter *s,int max_ios)