Skip to contentSkip to conversation

Recent changes (master)

PATCH Jens Axboe 1 participant 1 message Nov 22, 2021

lore
Share & tools
lore

Jens Axboe

author
#0 lore raw

The following changes since commit 9f51d89c683d70cd8ab23ba09ec6e628a548af5a:

Sync io_uring header with the kernel (2021-11-20 07:31:20 -0700)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to 1d08bfb018e600cc47f122fb78c02bf74b84dee8:

t/dedupe: style fixups (2021-11-21 06:51:11 -0700)

----------------------------------------------------------------
Bar David (2):
Mixed dedup and compression
fio-dedup: adjusted the binary to support compression

Jens Axboe (3):
Merge branch 'dedupe_and_compression' of https://github.com/bardavid/fio
t/io_uring: fix 32-bit compile warnings
t/dedupe: style fixups

 DEDUPE-TODO  |   3 --
 dedupe.c     |  12 ++++-
 io_u.c       |  29 ++++++-----
 t/dedupe.c   | 167 +++++++++++++++++++++++++++++++++++++++++++++++------------
 t/io_uring.c |   4 +-
 5 files changed, 161 insertions(+), 54 deletions(-)

---

Diff of recent changes:

DEDUPE-TODO +0 -3
index 1f3ee9da..4b0bfd1d 100644
@@ -1,6 +1,3 @@
-- Mixed buffers of dedupe-able and compressible data.
-  Major usecase in performance benchmarking of storage subsystems.
-
 - Shifted dedup-able data.
   Allow for dedup buffer generation to shift contents by random number
   of sectors (fill the gaps with uncompressible data). Some storage
dedupe.c +10 -2
index 043a376c..fd116dfb 100644
@@ -2,12 +2,14 @@
 
 int init_dedupe_working_set_seeds(struct thread_data *td)
 {
-	unsigned long long i;
+	unsigned long long i, j, num_seed_advancements;
 	struct frand_state dedupe_working_set_state = {0};
 
 	if (!td->o.dedupe_percentage || !(td->o.dedupe_mode == DEDUPE_MODE_WORKING_SET))
 		return 0;
 
+	num_seed_advancements = td->o.min_bs[DDIR_WRITE] /
+		min_not_zero(td->o.min_bs[DDIR_WRITE], (unsigned long long) td->o.compress_chunk);
 	/*
 	 * The dedupe working set keeps seeds of unique data (generated by buf_state).
 	 * Dedupe-ed pages will be generated using those seeds.
@@ -21,7 +23,13 @@ int init_dedupe_working_set_seeds(struct thread_data *td)
 	frand_copy(&dedupe_working_set_state, &td->buf_state);
 	for (i = 0; i < td->num_unique_pages; i++) {
 		frand_copy(&td->dedupe_working_set_states[i], &dedupe_working_set_state);
-		__get_next_seed(&dedupe_working_set_state);
+		/*
+		 * When compression is used the seed is advanced multiple times to
+		 * generate the buffer. We want to regenerate the same buffer when
+		 * deduping against this page
+		 */
+		for (j = 0; j < num_seed_advancements; j++)
+			__get_next_seed(&dedupe_working_set_state);
 	}
 
 	return 0;
io_u.c +16 -13
index 586a4bef..3c72d63d 100644
@@ -2230,27 +2230,30 @@ void fill_io_buffer(struct thread_data *td, void *buf, unsigned long long min_wr
 
 	if (o->compress_percentage || o->dedupe_percentage) {
 		unsigned int perc = td->o.compress_percentage;
-		struct frand_state *rs;
+		struct frand_state *rs = NULL;
 		unsigned long long left = max_bs;
 		unsigned long long this_write;
 
 		do {
-			rs = get_buf_state(td);
+			/*
+			 * Buffers are either entirely dedupe-able or not.
+			 * If we choose to dedup, the buffer should undergo
+			 * the same manipulation as the original write. Which
+			 * means we should retrack the steps we took for compression
+			 * as well.
+			 */
+			if (!rs)
+				rs = get_buf_state(td);
 
 			min_write = min(min_write, left);
 
-			if (perc) {
-				this_write = min_not_zero(min_write,
-							(unsigned long long) td->o.compress_chunk);
+			this_write = min_not_zero(min_write,
+						(unsigned long long) td->o.compress_chunk);
 
-				fill_random_buf_percentage(rs, buf, perc,
-					this_write, this_write,
-					o->buffer_pattern,
-					o->buffer_pattern_bytes);
-			} else {
-				fill_random_buf(rs, buf, min_write);
-				this_write = min_write;
-			}
+			fill_random_buf_percentage(rs, buf, perc,
+				this_write, this_write,
+				o->buffer_pattern,
+				o->buffer_pattern_bytes);
 
 			buf += this_write;
 			left -= this_write;
t/dedupe.c +133 -34

381 lines not on this page · raw message

t/io_uring.c +2 -2
index 7bf215c7..a98f78fd 100644
@@ -192,7 +192,7 @@ unsigned int calc_clat_percentiles(unsigned long *io_u_plat, unsigned long nr,
 	unsigned long *ovals = NULL;
 	bool is_last;
 
-	*minv = -1ULL;
+	*minv = -1UL;
 	*maxv = 0;
 
 	ovals = malloc(len * sizeof(*ovals));
@@ -498,7 +498,7 @@ static void init_io(struct submitter *s, unsigned index)
 	sqe->off = offset;
 	sqe->user_data = (unsigned long) f->fileno;
 	if (stats && stats_running)
-		sqe->user_data |= ((unsigned long)s->clock_index << 32);
+		sqe->user_data |= ((uint64_t)s->clock_index << 32);
 }
 
 static int prep_more_ios_uring(struct submitter *s, int max_ios)