Recent changes (master)
The following changes since commit 8fc46b4d03bc61c4e72b69c3bc59bd50c4943e03:
btrace2fio: expand rate options, bug fixes (2014-09-18 17:23:55 +0200)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to c0b69b92fb155424946b19228da9be0924e9e96c:
dedupe: if percentage is 100, don't go through random + math (2014-09-22 14:20:05 -0600)
----------------------------------------------------------------
Jens Axboe (2):
Basic support for dedupe
dedupe: if percentage is 100, don't go through random + math
HOWTO | 20 ++++++++++++++------ cconv.c | 2 ++ fio.1 | 7 +++++++ fio.h | 3 +++ init.c | 7 ++++++- io_u.c | 41 +++++++++++++++++++++++++++++++++++++---- lib/rand.h | 8 ++++++++ options.c | 22 ++++++++++++++++++++++ server.h | 2 +- thread_options.h | 6 ++++-- 10 files changed, 104 insertions(+), 14 deletions(-)
---
Diff of recent changes:
HOWTO +14 -6
@@ -565,12 +565,20 @@ buffer_compress_chunk=int See buffer_compress_percentage. This alternate random and zeroed data throughout the IO buffer. -buffer_pattern=str If set, fio will fill the io buffers with this pattern. - If not set, the contents of io buffers is defined by the other - options related to buffer contents. The setting can be any - pattern of bytes, and can be prefixed with 0x for hex values. - It may also be a string, where the string must then be - wrapped with "". +buffer_pattern=str If set, fio will fill the io buffers with this + pattern. If not set, the contents of io buffers is defined by + the other options related to buffer contents. The setting can + be any pattern of bytes, and can be prefixed with 0x for hex + values. It may also be a string, where the string must then + be wrapped with "". + +dedupe_percentage=int If set, fio will generate this percentage of + identical buffers when writing. These buffers will be + naturally dedupable. The contents of the buffers depend on + what other buffer compression settings have been set. It's + possible to have the individual buffers either fully + compressible, or not at all. This option only controls the + distribution of unique buffers. nrfiles=int Number of files to use for this job. Defaults to 1.
cconv.c +2 -0
@@ -241,6 +241,7 @@ void convert_thread_options_to_cpu(struct thread_options *o, o->latency_percentile.u.f =fio_uint64_to_double (le64_to_cpu (top->latency_percentile.u.i)); o->compress_percentage =le32_to_cpu (top->compress_percentage); o->compress_chunk =le32_to_cpu (top->compress_chunk); + o->dedupe_percentage =le32_to_cpu (top->dedupe_percentage); o->trim_backlog =le64_to_cpu (top->trim_backlog);
@@ -401,6 +402,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top, top->latency_percentile.u.i =__cpu_to_le64 (fio_double_to_uint64 (o->latency_percentile.u.f)); top->compress_percentage =cpu_to_le32 (o->compress_percentage); top->compress_chunk =cpu_to_le32 (o->compress_chunk); + top->dedupe_percentage =cpu_to_le32 (o->dedupe_percentage);for (i =0 ; i <DDIR_RWDIR_CNT ; i++) { top->bs[i] =cpu_to_le32 (o->bs[i]);
fio.1 +7 -0
@@ -481,6 +481,13 @@ setting can be any pattern of bytes, and can be prefixed with 0x for hex values. It may also be a string, where the string must then be wrapped with "". .TP +.BI dedupe_percentage \fR=\fPint +If set, fio will generate this percentage of identical buffers when writing. +These buffers will be naturally dedupable. The contents of the buffers depend +on what other buffer compression settings have been set. It's possible to have +the individual buffers either fully compressible, or not at all. This option +only controls the distribution of unique buffers. +.TP .BI nrfiles \fR=\fPint Number of files to use for this job. Default: 1. .TP
fio.h +3 -0
@@ -89,6 +89,7 @@ enum {FIO_RAND_SEQ_RAND_WRITE_OFF ,FIO_RAND_SEQ_RAND_TRIM_OFF ,FIO_RAND_START_DELAY , +FIO_DEDUPE_OFF , FIO_RAND_NR_OFFS, };
@@ -177,6 +178,8 @@ struct thread_data { };struct frand_state buf_state; +struct frand_state buf_state_prev; +struct frand_state dedupe_state;unsigned int verify_batch;unsigned int trim_batch;
init.c +6 -1
@@ -836,7 +836,9 @@ static void td_fill_rand_seeds_internal(struct thread_data *td)void td_fill_rand_seeds (struct thread_data *td) {if (td->o.allrand_repeatable) { -for (int i =0 ; i <FIO_RAND_NR_OFFS ; i++) +unsigned int i; + +for (i =0 ; i <FIO_RAND_NR_OFFS ; i++) td->rand_seeds[i] =FIO_RANDSEED * td->thread_number + i; }
@@ -847,6 +849,9 @@ void td_fill_rand_seeds(struct thread_data *td)td_fill_rand_seeds_internal (td);init_rand_seed (&td->buf_state, td->rand_seeds[FIO_RAND_BUF_OFF ]); +frand_copy (&td->buf_state_prev, &td->buf_state); + +init_rand_seed (&td->dedupe_state, td->rand_seeds[FIO_DEDUPE_OFF ]); }/*
io_u.c +37 -4
59 lines not on this page · raw message
lib/rand.h +8 -0
@@ -7,6 +7,14 @@ struct frand_state {unsigned int s1, s2, s3; }; +static inline void frand_copy (struct frand_state *dst, +struct frand_state *src) +{ + dst->s1 = src->s1; + dst->s2 = src->s2; + dst->s3 = src->s3; +} +static inline unsigned int __rand (struct frand_state *state) {#define TAUSWORTHE (s,a,b,c,d) ((s&c)<<d) ^ (((s <<a) ^ s)>>b)
options.c +22 -0
@@ -1050,6 +1050,16 @@ static int str_buffer_compress_cb(void *data, unsigned long long *il)return 0 ; } +static int str_dedupe_cb (void *data,unsigned long long *il) +{ +struct thread_data *td = data; + + td->flags |=TD_F_COMPRESS ; + td->o.dedupe_percentage = *il; + td->o.refill_buffers =1 ; +return 0 ; +} +static int str_verify_pattern_cb (void *data,const char *input) {struct thread_data *td = data;
@@ -3257,6 +3267,18 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .group = FIO_OPT_G_IO_BUF, }, { + .name = "dedupe_percentage", + .lname = "Dedupe percentage", + .type = FIO_OPT_INT, + .cb = str_dedupe_cb, + .maxval = 100, + .minval = 0, + .help = "Percentage of buffers that are dedupable", + .interval = 1, + .category = FIO_OPT_C_IO, + .group = FIO_OPT_G_IO_BUF, + }, + { .name = "clat_percentiles", .lname = "Completion latency percentiles", .type = FIO_OPT_BOOL,
server.h +1 -1
@@ -38,7 +38,7 @@ struct fio_net_cmd_reply { };enum { -FIO_SERVER_VER =35 , +FIO_SERVER_VER =36 ,FIO_SERVER_MAX_FRAGMENT_PDU =1024 ,FIO_SERVER_MAX_CMD_MB =2048 ,
thread_options.h +4 -2
@@ -184,6 +184,7 @@ struct thread_options {unsigned int buffer_pattern_bytes;unsigned int compress_percentage;unsigned int compress_chunk; +unsigned int dedupe_percentage;unsigned int time_based;unsigned int disable_lat;unsigned int disable_clat;
@@ -403,8 +404,9 @@ struct thread_options_pack {uint32_t scramble_buffers;uint8_t buffer_pattern[MAX_PATTERN_SIZE ];uint32_t buffer_pattern_bytes; -unsigned int compress_percentage; -unsigned int compress_chunk; +uint32_t compress_percentage; +uint32_t compress_chunk; +uint32_t dedupe_percentage;uint32_t time_based;uint32_t disable_lat;uint32_t disable_clat;