Recent changes (master)
Subsystems (2)
The following changes since commit 1066358aebafb7221732bedd6fb9fde56b14be7b:
Improve dedupe/compression buffer filling for mixed block sizes (2014-09-26 15:04:58 -0600)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to bc095aab789a2e7d036c6d499f6c124a0c2d5de0:
t/dedupe: fixup bloom entry calculation (2014-09-27 21:29:03 -0600)
----------------------------------------------------------------
Jens Axboe (12):
parse: fix issue with not filling leftover percentages correctly
dedupe: exit gracefully if device/file open fails
Add murmurhash3
crc/test: fix alignment
crc/test: add jhash
Move murmur3 hash to crc/
t/dedupe: print dedupe ratio instead of made-up factor crc/test: if the checksum needs a _final(), do it in the loop Add fnv hash
crc/test: add fnv
bloom: up hashes to 5 by default
t/dedupe: fixup bloom entry calculation
Makefile | 4 +-- crc/fnv.c | 16 ++++++++++ crc/fnv.h | 8 +++++ crc/murmur3.c | 68 +++++++++++++++++++++++++++++++++++++++++ crc/murmur3.h | 8 +++++ crc/test.c | 93 ++++++++++++++++++++++++++++++++++++++++++++------------- lib/bloom.c | 29 ++++++++++++++---- options.c | 11 +++---- t/dedupe.c | 15 ++++++---- 9 files changed, 212 insertions(+), 40 deletions(-) create mode 100644 crc/fnv.c create mode 100644 crc/fnv.h create mode 100644 crc/murmur3.c create mode 100644 crc/murmur3.h
---
Diff of recent changes:
Makefile +2 -2
@@ -192,8 +192,8 @@ endif ifeq ($(CONFIG_TARGET_OS), Linux) T_DEDUPE_OBJS = t/dedupe.o T_DEDUPE_OBJS += lib/rbtree.o t/log.o mutex.o smalloc.o gettime.o crc/md5.o \ - memalign.o lib/bloom.o t/debug.o crc/xxhash.o crc/crc32c.o \ - crc/crc32c-intel.o + memalign.o lib/bloom.o t/debug.o crc/xxhash.o crc/murmur3.o \ + crc/crc32c.o crc/crc32c-intel.o crc/fnv.o T_DEDUPE_PROGS = t/dedupe endif
crc/fnv.c new file +16 -0
@@ -0,0 +1,16 @@ +#include "fnv.h" + +#define FNV_PRIME 0x100000001b3ULL + +uint64_t fnv (const void *buf,uint32_t len,uint64_t hval) +{ +const uint64_t *ptr = buf; +const uint64_t *end = (void *) buf + len; + +while (ptr < end) { + hval *=FNV_PRIME ; + hval ^= (uint64_t ) *ptr++; + } + +return hval; +}
crc/fnv.h new file +8 -0
@@ -0,0 +1,8 @@ +#ifndef FIO_FNV_H +#define FIO_FNV_H + +#include <inttypes.h> + +uint64_t fnv (const void *,uint32_t ,uint64_t ); + +#endif
crc/murmur3.c new file +68 -0
68 lines not on this page · raw message
crc/murmur3.h new file +8 -0
@@ -0,0 +1,8 @@ +#ifndef FIO_MURMUR3_H +#define FIO_MURMUR3_H + +#include <inttypes.h> + +uint32_t murmurhash3 (const void *key,uint32_t len,uint32_t seed); + +#endif
crc/test.c +72 -21
204 lines not on this page · raw message
lib/bloom.c +23 -6
@@ -5,7 +5,9 @@#include "../hash.h" #include "../minmax.h" #include "../crc/xxhash.h" +#include "../crc/murmur3.h" #include "../crc/crc32c.h" +#include "../crc/fnv.h" struct bloom {uint64_t nentries;
@@ -21,27 +23,42 @@ struct bloom_hash {uint32_t (*fn)(const void *,uint32_t ,uint32_t ); }; -static uint32_t b_crc32c (const void *buf,uint32_t len,uint32_t seed) +static uint32_t bloom_crc32c (const void *buf,uint32_t len,uint32_t seed) {return fio_crc32c (buf, len); } +static uint32_t bloom_fnv (const void *buf,uint32_t len,uint32_t seed) +{ +return fnv (buf, len, seed); +} + +#define BLOOM_SEED 0x8989 +struct bloom_hash hashes[] = { { - .seed =0x8989 , + .seed =BLOOM_SEED , .fn = jhash, }, { - .seed =0x8989 , + .seed =BLOOM_SEED , .fn =XXH32 , }, { - .seed =0 , - .fn = b_crc32c, + .seed =BLOOM_SEED , + .fn = murmurhash3, + }, + { + .seed =BLOOM_SEED , + .fn = bloom_crc32c, + }, + { + .seed =BLOOM_SEED , + .fn = bloom_fnv, }, }; -#define N_HASHES 3 +#define N_HASHES 5#define MIN_ENTRIES 1073741824UL
options.c +6 -5
@@ -98,9 +98,9 @@ static int bssplit_ddir(struct thread_options *o, int ddir, char *str)if (perc >100 ) perc =100 ;else if (!perc) - perc =-1 ; + perc =-1U ; }else - perc =-1 ; + perc =-1U ;if (str_to_decimal (fname, &val,1 , o,0 )) {log_err ("fio: bssplit conversion failed\n" );
@@ -127,7 +127,7 @@ static int bssplit_ddir(struct thread_options *o, int ddir, char *str)for (i =0 ; i < o->bssplit_nr[ddir]; i++) {struct bssplit *bsp = &bssplit[i]; -if (bsp->perc == (unsigned char )-1 ) +if (bsp->perc ==-1U ) perc_missing++;else perc += bsp->perc;
@@ -138,17 +138,18 @@ static int bssplit_ddir(struct thread_options *o, int ddir, char *str)free (bssplit);return 1 ; } +/* * If values didn't have a percentage set, divide the remains between * them. */ if (perc_missing) { -if (perc_missing ==1 ) +if (perc_missing ==1 && o->bssplit_nr[ddir] ==1 ) perc =100 ;for (i =0 ; i < o->bssplit_nr[ddir]; i++) {struct bssplit *bsp = &bssplit[i]; -if (bsp->perc == (unsigned char )-1 ) +if (bsp->perc ==-1U ) bsp->perc = (100 - perc) / perc_missing; } }
t/dedupe.c +9 -6
@@ -452,7 +452,7 @@ static int dedupe_check(const char *filename, uint64_t *nextents,if (use_bloom) {uint64_t bloom_entries; - bloom_entries = (3 * dev_size ) / (blocksize *2 ); + bloom_entries =8 * (dev_size / blocksize); bloom =bloom_new (bloom_entries); }
@@ -475,10 +475,11 @@ static void show_chunk(struct chunk *c)static void show_stat (uint64_t nextents,uint64_t nchunks) { -double perc; +double perc, ratio;printf ("Extents=%lu, Unique extents=%lu\n" , (unsigned long ) nextents, (unsigned long ) nchunks); -printf ("De-dupe factor: %3.2f\n" , (double ) nextents / (double ) nchunks); + ratio = (double ) nextents / (double ) nchunks; +printf ("De-dupe ratio: 1:%3.2f\n" , ratio -1.0 ); perc =1.00 - ((double ) nchunks / (double ) nextents); perc *=100.0 ;
@@ -575,10 +576,12 @@ int main(int argc, char *argv[]) ret =dedupe_check (argv[optind], &nextents, &nchunks); -if (!bloom) -iter_rb_tree (&nextents, &nchunks); +if (!ret) { +if (!bloom) +iter_rb_tree (&nextents, &nchunks); -show_stat (nextents, nchunks); +show_stat (nextents, nchunks); + }fio_mutex_remove (rb_lock);if (bloom)