Recent changes (master)
Subsystems (2)
The following changes since commit c0b69b92fb155424946b19228da9be0924e9e96c:
dedupe: if percentage is 100, don't go through random + math (2014-09-22 14:20:05 -0600)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 4877c1ab380b199ba6d9207b689bb1df127b0b4b:
t/dedupe: Linux only for now (2014-09-23 18:36:52 -0600)
----------------------------------------------------------------
Jens Axboe (10):
fifo: use minmax.h instead of rolling its own min/max Fix min/max typeof warnings
memalign: fix off-by-one bug in alignment Checksumming updates
Add small tool to check for dedupable contents in a file/device
dedupe: improve wording in output
dedupe: print progress indicator
Build t/ tools by default
Fix 32-bit compile warnings
t/dedupe: Linux only for now
Makefile | 14 ++ crc/md5.c | 20 +++ crc/md5.h | 1 + crc/sha256.c | 64 +++++--- crc/sha256.h | 8 +- crc/test.c | 4 + fifo.h | 19 +-- lib/axmap.c | 2 +- lib/zipf.c | 4 +- memalign.c | 2 +- minmax.h | 13 +- parse.c | 2 +- t/btrace2fio.c | 2 +- t/dedupe.c | 481 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ t/lfsr-test.c | 11 +- 15 files changed, 595 insertions(+), 52 deletions(-) create mode 100644 t/dedupe.c
---
Diff of recent changes:
Makefile +14 -0
@@ -189,12 +189,20 @@ T_BTRACE_FIO_OBJS += fifo.o lib/flist_sort.o t/log.o lib/linux-dev-lookup.o T_BTRACE_FIO_PROGS = t/btrace2fio endif +ifeq ($(CONFIG_TARGET_OS), Linux) +T_DEDUPE_OBJS = t/dedupe.o +T_DEDUPE_OBJS += lib/rbtree.o t/log.o mutex.o smalloc.o gettime.o crc/md5.o \ + memalign.o +T_DEDUPE_PROGS = t/dedupe +endif + T_OBJS = $(T_SMALLOC_OBJS) T_OBJS += $(T_IEEE_OBJS) T_OBJS += $(T_ZIPF_OBJS) T_OBJS += $(T_AXMAP_OBJS) T_OBJS += $(T_LFSR_TEST_OBJS) T_OBJS += $(T_BTRACE_FIO_OBJS) +T_OBJS += $(T_DEDUPE_OBJS) T_PROGS = $(T_SMALLOC_PROGS) T_PROGS += $(T_IEEE_PROGS)
@@ -202,6 +210,9 @@ T_PROGS += $(T_ZIPF_PROGS) T_PROGS += $(T_AXMAP_PROGS) T_PROGS += $(T_LFSR_TEST_PROGS) T_PROGS += $(T_BTRACE_FIO_PROGS) +T_PROGS += $(T_DEDUPE_PROGS) + +PROGS += $(T_PROGS) ifneq ($(findstring $(MAKEFLAGS),s),s) ifndef V
@@ -303,6 +314,9 @@ t/btrace2fio: $(T_BTRACE_FIO_OBJS) $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_BTRACE_FIO_OBJS) $(LIBS) endif +t/dedupe: $(T_DEDUPE_OBJS) + $(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_DEDUPE_OBJS) $(LIBS) + clean: FORCE -rm -f .depend $(FIO_OBJS) $(GFIO_OBJS) $(OBJS) $(T_OBJS) $(PROGS) $(T_PROGS) core.* core gfio FIO-VERSION-FILE *.d lib/*.d crc/*.d engines/*.d profiles/*.d t/*.d config-host.mak config-host.h
crc/md5.c +20 -0
@@ -125,3 +125,23 @@ void fio_md5_update(struct fio_md5_ctx *mctx, const uint8_t *data,memcpy (mctx->block, data, len); } + +void fio_md5_final (struct fio_md5_ctx *mctx) +{ +const unsigned int offset = mctx->byte_count &0x3f ; +char *p = (char *)mctx->block + offset; +int padding =56 - (offset +1 ); + + *p++ =0x80 ; +if (padding <0 ) { +memset (p,0x00 , padding +sizeof (uint64_t )); +md5_transform (mctx->hash, mctx->block); + p = (char *)mctx->block; + padding =56 ; + } + +memset (p,0 , padding); + mctx->block[14 ] = mctx->byte_count <<3 ; + mctx->block[15 ] = mctx->byte_count >>29 ; +md5_transform (mctx->hash, mctx->block); +}
crc/md5.h +1 -0
@@ -23,6 +23,7 @@ struct fio_md5_ctx { };extern void fio_md5_update (struct fio_md5_ctx *,const uint8_t *,unsigned int ); +extern void fio_md5_final (struct fio_md5_ctx *);extern void fio_md5_init (struct fio_md5_ctx *);#endif
crc/sha256.c +42 -22
79 lines not on this page · raw message
crc/sha256.h +6 -2
@@ -1,13 +1,17 @@#ifndef FIO_SHA256_H #define FIO_SHA256_H +#define SHA256_DIGEST_SIZE 32 +#define SHA256_BLOCK_SIZE 64 +struct fio_sha256_ctx { -uint32_t count[2 ]; -uint32_t state[8 ]; +uint32_t count; +uint32_t state[SHA256_DIGEST_SIZE /4 ];uint8_t *buf; };void fio_sha256_init (struct fio_sha256_ctx *);void fio_sha256_update (struct fio_sha256_ctx *,const uint8_t *,unsigned int ); +void fio_sha256_final (struct fio_sha256_ctx *);#endif
crc/test.c +4 -0
@@ -52,6 +52,8 @@ static void t_md5(void *buf, size_t size)for (i =0 ; i <NR_CHUNKS ; i++)fio_md5_update (&ctx, buf, size); + +fio_md5_final (&ctx); }static void t_crc64 (void *buf,size_t size)
@@ -116,6 +118,8 @@ static void t_sha256(void *buf, size_t size)for (i =0 ; i <NR_CHUNKS ; i++)fio_sha256_update (&ctx, buf, size); + +fio_sha256_final (&ctx); }static void t_sha512 (void *buf,size_t size)
fifo.h +4 -15
@@ -1,3 +1,5 @@ +#ifndef FIO_FIFO_H +#define FIO_FIFO_H /* * A simple FIFO implementation. *
@@ -18,6 +20,8 @@* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. * */ +#include "minmax.h" +struct fifo {unsigned char *buffer;/* the buffer holding the data */ unsigned int size;/* the size of the allocated buffer */
@@ -40,19 +44,4 @@ static inline unsigned int fifo_room(struct fifo *fifo) return fifo->size - fifo->in + fifo->out; } -#ifndef min -#define min (x,y) ({ \ - typeof(x) _x = (x); \ - typeof(y) _y = (y); \ - (void) (&_x == &_y); \ - _x < _y ? _x : _y; }) -#endif - -#ifndef max -#define max (x,y) ({ \ - typeof(x) _x = (x); \ - typeof(y) _y = (y); \ - (void) (&_x == &_y); \ - _x > _y ? _x : _y; }) -#endif
lib/axmap.c +1 -1
@@ -33,7 +33,7 @@#error "Number of arch bits unknown"#endif -#define BLOCKS_PER_UNIT (1UL << UNIT_SHIFT) +#define BLOCKS_PER_UNIT (1U << UNIT_SHIFT)#define BLOCKS_PER_UNIT_MASK (BLOCKS_PER_UNIT - 1)#define firstfree_valid (b) ((b)->first_free != (uint64_t) -1)
lib/zipf.c +2 -2
@@ -11,7 +11,7 @@#include "../minmax.h" #include "../hash.h" -#define ZIPF_MAX_GEN 10000000 +#define ZIPF_MAX_GEN 10000000ULstatic void zipf_update (struct zipf_state *zs) {
@@ -23,7 +23,7 @@ static void zipf_update(struct zipf_state *zs)* 10M max, that should be doable in 1-2s on even slow machines. * Precision will take a slight hit, but nothing major. */ - to_gen =min (zs->nranges,ZIPF_MAX_GEN ); + to_gen =min (zs->nranges, (uint64_t )ZIPF_MAX_GEN );for (i =0 ; i < to_gen; i++) zs->zetan +=pow (1.0 / (double ) (i +1 ), zs->theta);
memalign.c +1 -1
@@ -20,7 +20,7 @@ void *fio_memalign(size_t alignment, size_t size) ptr =malloc (size + alignment + size +sizeof (*f) -1 );if (ptr) { - ret =PTR_ALIGN (ptr, alignment); + ret =PTR_ALIGN (ptr, alignment -1 ); f = ret + size; f->offset = (uintptr_t ) ret - (uintptr_t ) ptr; }
minmax.h +11 -2
@@ -2,10 +2,19 @@#define FIO_MIN_MAX_H #ifndef min -#define min (a, b) ((a) < (b) ? (a) : (b)) +#define min (x,y) ({ \ + typeof(x) _x = (x); \ + typeof(y) _y = (y); \ + (void) (&_x == &_y); \ + _x < _y ? _x : _y; })#endif +#ifndef max -#define max (a, b) ((a) > (b) ? (a) : (b)) +#define max (x,y) ({ \ + typeof(x) _x = (x); \ + typeof(y) _y = (y); \ + (void) (&_x == &_y); \ + _x > _y ? _x : _y; })#endif #endif
parse.c +1 -1
@@ -380,7 +380,7 @@ static int check_int(const char *p, int *val)return 1 ; } -static int opt_len (const char *str) +static size_t opt_len (const char *str) {char *postfix;
t/btrace2fio.c +1 -1
@@ -589,7 +589,7 @@ static void __output_p_ascii(struct btrace_pid *p, unsigned long *ios) perc = ((float ) o->merges[i] *100.0 ) / (float ) total;printf ("\tmerges: %lu (perc=%3.2f%%)\n" , o->merges[i], perc); perc = ((float ) o->seq[i] *100.0 ) / (float ) o->ios[i]; -printf ("\tseq: %lu (perc=%3.2f%%)\n" , o->seq[i], perc); +printf ("\tseq: %lu (perc=%3.2f%%)\n" , (unsigned long ) o->seq[i], perc);printf ("\trate: %lu KB/sec\n" ,o_to_kb_rate (o, i));for (j =0 ; j < o->nr_bs[i]; j++) {
t/dedupe.c new file +481 -0
481 lines not on this page · raw message
t/lfsr-test.c +6 -5
@@ -65,11 +65,11 @@ int main(int argc, char *argv[])printf ("LFSR specs\n" );printf ("==========================\n" );printf ("Size is %u\n" ,64 -__builtin_clzl (fl->cached_bit)); -printf ("Max val is %lu\n" , fl->max_val); -printf ("XOR-mask is 0x%lX\n" , fl->xormask); -printf ("Seed is %lu\n" , fl->last_val); +printf ("Max val is %lu\n" , (unsigned long ) fl->max_val); +printf ("XOR-mask is 0x%lX\n" , (unsigned long ) fl->xormask); +printf ("Seed is %lu\n" , (unsigned long ) fl->last_val);printf ("Spin is %u\n" , fl->spin); -printf ("Cycle length is %lu\n" , fl->cycle_length); +printf ("Cycle length is %lu\n" , (unsigned long ) fl->cycle_length);/* Create verification table */ if (verify) {
@@ -102,7 +102,8 @@ int main(int argc, char *argv[])for (i =0 ; i < numbers; i++) {if (*(uint8_t *)(v + i) !=1 ) {fprintf (stderr,"failed (%lu = %d).\n" , - i, *(uint8_t *)(v + i)); + (unsigned long ) i, + *(uint8_t *)(v + i)); r =1 ;break ; }