Skip to contentSkip to conversation

Recent changes (master)

PATCH Jens Axboe 1 participant 1 message Sep 24, 2014

Subsystems (2)
lore
Share & tools
lore

Jens Axboe

author
#0 lore raw

The following changes since commit c0b69b92fb155424946b19228da9be0924e9e96c:

dedupe: if percentage is 100, don't go through random + math (2014-09-22 14:20:05 -0600)

are available in the git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to 4877c1ab380b199ba6d9207b689bb1df127b0b4b:

t/dedupe: Linux only for now (2014-09-23 18:36:52 -0600)

----------------------------------------------------------------
Jens Axboe (10):
fifo: use minmax.h instead of rolling its own min/max Fix min/max typeof warnings
memalign: fix off-by-one bug in alignment Checksumming updates
Add small tool to check for dedupable contents in a file/device
dedupe: improve wording in output
dedupe: print progress indicator
Build t/ tools by default
Fix 32-bit compile warnings
t/dedupe: Linux only for now

 Makefile       |   14 ++
 crc/md5.c      |   20 +++
 crc/md5.h      |    1 +
 crc/sha256.c   |   64 +++++---
 crc/sha256.h   |    8 +-
 crc/test.c     |    4 +
 fifo.h         |   19 +--
 lib/axmap.c    |    2 +-
 lib/zipf.c     |    4 +-
 memalign.c     |    2 +-
 minmax.h       |   13 +-
 parse.c        |    2 +-
 t/btrace2fio.c |    2 +-
 t/dedupe.c     |  481 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 t/lfsr-test.c  |   11 +-
 15 files changed, 595 insertions(+), 52 deletions(-)
 create mode 100644 t/dedupe.c

---

Diff of recent changes:

Makefile +14 -0
index 4350e5a..fe439c1 100644
@@ -189,12 +189,20 @@ T_BTRACE_FIO_OBJS += fifo.o lib/flist_sort.o t/log.o lib/linux-dev-lookup.o
 T_BTRACE_FIO_PROGS = t/btrace2fio
 endif
 
+ifeq ($(CONFIG_TARGET_OS), Linux)
+T_DEDUPE_OBJS = t/dedupe.o
+T_DEDUPE_OBJS += lib/rbtree.o t/log.o mutex.o smalloc.o gettime.o crc/md5.o \
+		memalign.o
+T_DEDUPE_PROGS = t/dedupe
+endif
+
 T_OBJS = $(T_SMALLOC_OBJS)
 T_OBJS += $(T_IEEE_OBJS)
 T_OBJS += $(T_ZIPF_OBJS)
 T_OBJS += $(T_AXMAP_OBJS)
 T_OBJS += $(T_LFSR_TEST_OBJS)
 T_OBJS += $(T_BTRACE_FIO_OBJS)
+T_OBJS += $(T_DEDUPE_OBJS)
 
 T_PROGS = $(T_SMALLOC_PROGS)
 T_PROGS += $(T_IEEE_PROGS)
@@ -202,6 +210,9 @@ T_PROGS += $(T_ZIPF_PROGS)
 T_PROGS += $(T_AXMAP_PROGS)
 T_PROGS += $(T_LFSR_TEST_PROGS)
 T_PROGS += $(T_BTRACE_FIO_PROGS)
+T_PROGS += $(T_DEDUPE_PROGS)
+
+PROGS += $(T_PROGS)
 
 ifneq ($(findstring $(MAKEFLAGS),s),s)
 ifndef V
@@ -303,6 +314,9 @@ t/btrace2fio: $(T_BTRACE_FIO_OBJS)
 	$(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_BTRACE_FIO_OBJS) $(LIBS)
 endif
 
+t/dedupe: $(T_DEDUPE_OBJS)
+	$(QUIET_LINK)$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $(T_DEDUPE_OBJS) $(LIBS)
+
 clean: FORCE
 	-rm -f .depend $(FIO_OBJS) $(GFIO_OBJS) $(OBJS) $(T_OBJS) $(PROGS) $(T_PROGS) core.* core gfio FIO-VERSION-FILE *.d lib/*.d crc/*.d engines/*.d profiles/*.d t/*.d config-host.mak config-host.h
 
crc/md5.c +20 -0
index 0da85e4..64fe48a 100644
@@ -125,3 +125,23 @@ void fio_md5_update(struct fio_md5_ctx *mctx, const uint8_t *data,
 
 	memcpy(mctx->block, data, len);
 }
+
+void fio_md5_final(struct fio_md5_ctx *mctx)
+{
+	const unsigned int offset = mctx->byte_count & 0x3f;
+	char *p = (char *)mctx->block + offset;
+	int padding = 56 - (offset + 1);
+
+	*p++ = 0x80;
+	if (padding < 0) {
+		memset(p, 0x00, padding + sizeof (uint64_t));
+		md5_transform(mctx->hash, mctx->block);
+		p = (char *)mctx->block;
+		padding = 56;
+	}
+
+	memset(p, 0, padding);
+	mctx->block[14] = mctx->byte_count << 3;
+	mctx->block[15] = mctx->byte_count >> 29;
+	md5_transform(mctx->hash, mctx->block);
+}
crc/md5.h +1 -0
index 668f0e9..54e350c 100644
@@ -23,6 +23,7 @@ struct fio_md5_ctx {
 };
 
 extern void fio_md5_update(struct fio_md5_ctx *, const uint8_t *, unsigned int);
+extern void fio_md5_final(struct fio_md5_ctx *);
 extern void fio_md5_init(struct fio_md5_ctx *);
 
 #endif
crc/sha256.c +42 -22

79 lines not on this page · raw message

crc/sha256.h +6 -2
index c7aa28f..b636033 100644
@@ -1,13 +1,17 @@
 #ifndef FIO_SHA256_H
 #define FIO_SHA256_H
 
+#define SHA256_DIGEST_SIZE	32
+#define SHA256_BLOCK_SIZE	64
+
 struct fio_sha256_ctx {
-	uint32_t count[2];
-	uint32_t state[8];
+	uint32_t count;
+	uint32_t state[SHA256_DIGEST_SIZE / 4];
 	uint8_t *buf;
 };
 
 void fio_sha256_init(struct fio_sha256_ctx *);
 void fio_sha256_update(struct fio_sha256_ctx *, const uint8_t *, unsigned int);
+void fio_sha256_final(struct fio_sha256_ctx *);
 
 #endif
crc/test.c +4 -0
index 0c3b2da..36054e6 100644
@@ -52,6 +52,8 @@ static void t_md5(void *buf, size_t size)
 
 	for (i = 0; i < NR_CHUNKS; i++)
 		fio_md5_update(&ctx, buf, size);
+
+	fio_md5_final(&ctx);
 }
 
 static void t_crc64(void *buf, size_t size)
@@ -116,6 +118,8 @@ static void t_sha256(void *buf, size_t size)
 
 	for (i = 0; i < NR_CHUNKS; i++)
 		fio_sha256_update(&ctx, buf, size);
+
+	fio_sha256_final(&ctx);
 }
 
 static void t_sha512(void *buf, size_t size)
fifo.h +4 -15
index 7491365..4b775b0 100644
@@ -1,3 +1,5 @@
+#ifndef FIO_FIFO_H
+#define FIO_FIFO_H
 /*
  * A simple FIFO implementation.
  *
@@ -18,6 +20,8 @@
  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  *
  */
+#include "minmax.h"
+
 struct fifo {
 	unsigned char *buffer;	/* the buffer holding the data */
 	unsigned int size;	/* the size of the allocated buffer */
@@ -40,19 +44,4 @@ static inline unsigned int fifo_room(struct fifo *fifo)
 	return fifo->size - fifo->in + fifo->out;
 }
 
-#ifndef min
-#define min(x,y) ({ \
-	typeof(x) _x = (x);	\
-	typeof(y) _y = (y);	\
-	(void) (&_x == &_y);		\
-	_x < _y ? _x : _y; })
-#endif
-
-#ifndef max
-#define max(x,y) ({ \
-	typeof(x) _x = (x);	\
-	typeof(y) _y = (y);	\
-	(void) (&_x == &_y);		\
-	_x > _y ? _x : _y; })
-
 #endif
lib/axmap.c +1 -1
index 15cd635..5b8cb30 100644
@@ -33,7 +33,7 @@
 #error "Number of arch bits unknown"
 #endif
 
-#define BLOCKS_PER_UNIT		(1UL << UNIT_SHIFT)
+#define BLOCKS_PER_UNIT		(1U << UNIT_SHIFT)
 #define BLOCKS_PER_UNIT_MASK	(BLOCKS_PER_UNIT - 1)
 
 #define firstfree_valid(b)	((b)->first_free != (uint64_t) -1)
lib/zipf.c +2 -2
index 9b6ce63..c691bc5 100644
@@ -11,7 +11,7 @@
 #include "../minmax.h"
 #include "../hash.h"
 
-#define ZIPF_MAX_GEN	10000000
+#define ZIPF_MAX_GEN	10000000UL
 
 static void zipf_update(struct zipf_state *zs)
 {
@@ -23,7 +23,7 @@ static void zipf_update(struct zipf_state *zs)
 	 * 10M max, that should be doable in 1-2s on even slow machines.
 	 * Precision will take a slight hit, but nothing major.
 	 */
-	to_gen = min(zs->nranges, ZIPF_MAX_GEN);
+	to_gen = min(zs->nranges, (uint64_t) ZIPF_MAX_GEN);
 
 	for (i = 0; i < to_gen; i++)
 		zs->zetan += pow(1.0 / (double) (i + 1), zs->theta);
memalign.c +1 -1
index 7a04ffd..cfd6e46 100644
@@ -20,7 +20,7 @@ void *fio_memalign(size_t alignment, size_t size)
 
 	ptr = malloc(size + alignment + size + sizeof(*f) - 1);
 	if (ptr) {
-		ret = PTR_ALIGN(ptr, alignment);
+		ret = PTR_ALIGN(ptr, alignment - 1);
 		f = ret + size;
 		f->offset = (uintptr_t) ret - (uintptr_t) ptr;
 	}
minmax.h +11 -2
index e5c2f58..97957c8 100644
@@ -2,10 +2,19 @@
 #define FIO_MIN_MAX_H
 
 #ifndef min
-#define min(a, b)	((a) < (b) ? (a) : (b))
+#define min(x,y) ({ \
+	typeof(x) _x = (x);	\
+	typeof(y) _y = (y);	\
+	(void) (&_x == &_y);		\
+	_x < _y ? _x : _y; })
 #endif
+
 #ifndef max
-#define max(a, b)	((a) > (b) ? (a) : (b))
+#define max(x,y) ({ \
+	typeof(x) _x = (x);	\
+	typeof(y) _y = (y);	\
+	(void) (&_x == &_y);		\
+	_x > _y ? _x : _y; })
 #endif
 
 #endif
parse.c +1 -1
index e6d9406..40cd465 100644
@@ -380,7 +380,7 @@ static int check_int(const char *p, int *val)
 	return 1;
 }
 
-static int opt_len(const char *str)
+static size_t opt_len(const char *str)
 {
 	char *postfix;
 
t/btrace2fio.c +1 -1
index e4e05ca..5666a56 100644
@@ -589,7 +589,7 @@ static void __output_p_ascii(struct btrace_pid *p, unsigned long *ios)
 		perc = ((float) o->merges[i] * 100.0) / (float) total;
 		printf("\tmerges: %lu (perc=%3.2f%%)\n", o->merges[i], perc);
 		perc = ((float) o->seq[i] * 100.0) / (float) o->ios[i];
-		printf("\tseq:    %lu (perc=%3.2f%%)\n", o->seq[i], perc);
+		printf("\tseq:    %lu (perc=%3.2f%%)\n", (unsigned long) o->seq[i], perc);
 		printf("\trate:   %lu KB/sec\n", o_to_kb_rate(o, i));
 
 		for (j = 0; j < o->nr_bs[i]; j++) {
t/dedupe.c new file +481 -0

481 lines not on this page · raw message

t/lfsr-test.c +6 -5
index d371087..481f37e 100644
@@ -65,11 +65,11 @@ int main(int argc, char *argv[])
 	printf("LFSR specs\n");
 	printf("==========================\n");
 	printf("Size is         %u\n", 64 - __builtin_clzl(fl->cached_bit));
-	printf("Max val is      %lu\n", fl->max_val);
-	printf("XOR-mask is     0x%lX\n", fl->xormask);
-	printf("Seed is         %lu\n", fl->last_val);
+	printf("Max val is      %lu\n", (unsigned long) fl->max_val);
+	printf("XOR-mask is     0x%lX\n", (unsigned long) fl->xormask);
+	printf("Seed is         %lu\n", (unsigned long) fl->last_val);
 	printf("Spin is         %u\n", fl->spin);
-	printf("Cycle length is %lu\n", fl->cycle_length);
+	printf("Cycle length is %lu\n", (unsigned long) fl->cycle_length);
 
 	/* Create verification table */
 	if (verify) {
@@ -102,7 +102,8 @@ int main(int argc, char *argv[])
 		for (i = 0; i < numbers; i++) {
 			if (*(uint8_t *)(v + i) != 1) {
 				fprintf(stderr, "failed (%lu = %d).\n",
-						i, *(uint8_t *)(v + i));
+						(unsigned long) i,
+						*(uint8_t *)(v + i));
 				r = 1;
 				break;
 			}