Recent changes (master)
Subsystems (2)
The following changes since commit b5b571a3f01c17ddb39fd0306cb425a11e216f3d:
Fix compile for FIO_INC_DEBUG not set (2014-09-24 09:54:24 -0600)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 1066358aebafb7221732bedd6fb9fde56b14be7b:
Improve dedupe/compression buffer filling for mixed block sizes (2014-09-26 15:04:58 -0600)
----------------------------------------------------------------
Andrey Kuzmin (2):
Add ability to use an include file in a fio job file Add HOWTO section on include files
Jens Axboe (10):
Add bloom filter
dedupe: default to using a bloom filter to save memory Add debug helper stub for t/ programs
t/lfsr-test: fixup time
dedupe: fix warning and segfault on -B0
dedupe: read in larger chunks at the time
bloom: always use a larger minimum size for bloom filter Clarify that include files may not contain job sections
bloom: use independent hashes
Improve dedupe/compression buffer filling for mixed block sizes
HOWTO | 35 +++++++++++++ Makefile | 9 ++-- crc/xxhash.c | 2 +- crc/xxhash.h | 2 +- fio_time.h | 1 + init.c | 162 +++++++++++++++++++++++++++++++++++++++------------------ io_u.c | 32 +++++++----- lib/bloom.c | 107 +++++++++++++++++++++++++++++++++++++ lib/bloom.h | 13 +++++ t/debug.c | 14 +++++ t/debug.h | 6 +++ t/dedupe.c | 150 ++++++++++++++++++++++++++++++++++++---------------- t/lfsr-test.c | 13 ++--- t/stest.c | 12 +---- 14 files changed, 430 insertions(+), 128 deletions(-) create mode 100644 lib/bloom.c create mode 100644 lib/bloom.h create mode 100644 t/debug.c create mode 100644 t/debug.h
---
Diff of recent changes:
HOWTO +35 -0
@@ -159,6 +159,41 @@ specify: $ fio --name=random-writers --ioengine=libaio --iodepth=4 --rw=randwrite --bs=32k --direct=0 --size=64m --numjobs=4 +When fio is utilized as a basis of any reasonably large test suite, it might be +desirable to share a set of standardized settings across multiple job files. +Instead of copy/pasting such settings, any section may pull in an external +.fio file with 'include filename' directive, as in the following example: + +; -- start job file including.fio -- +[global] +filename=/tmp/test +filesize=1m +include glob-include.fio + +[test] +rw=randread +bs=4k +time_based=1 +runtime=10 +include test-include.fio +; -- end job file including.fio -- + +; -- start job file glob-include.fio -- +thread=1 +group_reporting=1 +; -- end job file glob-include.fio -- + +; -- start job file test-include.fio -- +ioengine=libaio +iodepth=4 +; -- end job file test-include.fio -- + +Settings pulled into a section apply to that section only (except global +section). Include directives may be nested in that any included file may +contain further include directive(s). Include files may not contain [] +sections. + + 4.1 Environment variables -------------------------
Makefile +5 -4
@@ -36,7 +36,7 @@ SOURCE := gettime.c ioengines.c init.c stat.c log.c time.c filesetup.c \ lib/lfsr.c gettime-thread.c helpers.c lib/flist_sort.c \ lib/hweight.c lib/getrusage.c idletime.c td_error.c \ profiles/tiobench.c profiles/act.c io_u_queue.c filelock.c \ - lib/tp.c + lib/tp.c lib/bloom.c ifdef CONFIG_LIBHDFS HDFSFLAGS= -I $(JAVA_HOME)/include -I $(JAVA_HOME)/include/linux -I $(FIO_LIBHDFS_INCLUDE)
@@ -164,7 +164,7 @@ GFIO_OBJS = $(OBJS) gfio.o graph.o tickmarks.o ghelpers.o goptions.o gerror.o \ -include $(OBJS:.o=.d) T_SMALLOC_OBJS = t/stest.o -T_SMALLOC_OBJS += gettime.o mutex.o smalloc.o t/log.o +T_SMALLOC_OBJS += gettime.o mutex.o smalloc.o t/log.o t/debug.o T_SMALLOC_PROGS = t/stest T_IEEE_OBJS = t/ieee754.o
@@ -180,7 +180,7 @@ T_AXMAP_OBJS += lib/lfsr.o lib/axmap.o T_AXMAP_PROGS = t/axmap T_LFSR_TEST_OBJS = t/lfsr-test.o -T_LFSR_TEST_OBJS += lib/lfsr.o +T_LFSR_TEST_OBJS += lib/lfsr.o gettime.o t/log.o t/debug.o T_LFSR_TEST_PROGS = t/lfsr-test ifeq ($(CONFIG_TARGET_OS), Linux)
@@ -192,7 +192,8 @@ endif ifeq ($(CONFIG_TARGET_OS), Linux) T_DEDUPE_OBJS = t/dedupe.o T_DEDUPE_OBJS += lib/rbtree.o t/log.o mutex.o smalloc.o gettime.o crc/md5.o \ - memalign.o + memalign.o lib/bloom.o t/debug.o crc/xxhash.o crc/crc32c.o \ + crc/crc32c-intel.o T_DEDUPE_PROGS = t/dedupe endif
crc/xxhash.c +1 -1
@@ -221,7 +221,7 @@ static uint32_t XXH32_endian_align(const void* input, int len, uint32_t seed, XX } -uint32_t XXH32 (const void * input,int len,uint32_t seed) +uint32_t XXH32 (const void * input,uint32_t len,uint32_t seed) {#if 0 // Simple version, good for code maintenance, but unfortunately slow for small inputs
crc/xxhash.h +1 -1
@@ -88,7 +88,7 @@ typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;// Simple Hash Functions //**************************** -unsigned int XXH32 (const void * input,int len,unsigned int seed); +uint32_t XXH32 (const void * input,uint32_t len,uint32_t seed);/* XXH32() :
fio_time.h +1 -0
@@ -1,6 +1,7 @@#ifndef FIO_TIME_H #define FIO_TIME_H +struct thread_data ;extern uint64_t utime_since (struct timeval *,struct timeval *);extern uint64_t utime_since_now (struct timeval *);extern uint64_t mtime_since (struct timeval *,struct timeval *);
init.c +112 -50
231 lines not on this page · raw message
io_u.c +20 -12
@@ -1487,7 +1487,8 @@ struct io_u *get_io_u(struct thread_data *td)if (io_u->ddir ==DDIR_WRITE ) {if (td->flags &TD_F_REFILL_BUFFERS ) {io_u_fill_buffer (td, io_u, - io_u->xfer_buflen, io_u->xfer_buflen); + td->o.min_bs[DDIR_WRITE ], + io_u->xfer_buflen); }else if ((td->flags &TD_F_SCRAMBLE_BUFFERS ) && !(td->flags &TD_F_COMPRESS )) do_scramble =1 ;
@@ -1864,22 +1865,29 @@ void fill_io_buffer(struct thread_data *td, void *buf, unsigned int min_write,else if (!td->o.zero_buffers) {unsigned int perc = td->o.compress_percentage;struct frand_state *rs; +unsigned int left = max_bs; - rs =get_buf_state (td); +do { + rs =get_buf_state (td); -if (perc) { -unsigned int seg = min_write; + min_write =min (min_write, left); - seg =min (min_write, td->o.compress_chunk); -if (!seg) - seg = min_write; +if (perc) { +unsigned int seg = min_write; -fill_random_buf_percentage (rs, buf, perc, seg,max_bs); -save_buf_state (td, rs); - }else { -fill_random_buf (rs, buf, max_bs); + seg =min (min_write, td->o.compress_chunk); +if (!seg) + seg = min_write; + +fill_random_buf_percentage (rs, buf, perc, seg, + min_write); + }else +fill_random_buf (rs, buf, min_write); + + buf += min_write; + left -= min_write;save_buf_state (td, rs); - } + }while (left); }else memset (buf,0 , max_bs); }
lib/bloom.c new file +107 -0
107 lines not on this page · raw message
lib/bloom.h new file +13 -0
@@ -0,0 +1,13 @@ +#ifndef FIO_BLOOM_H +#define FIO_BLOOM_H + +#include <inttypes.h> + +struct bloom ; + +struct bloom *bloom_new (uint64_t entries); +void bloom_free (struct bloom *b); +int bloom_check (struct bloom *b,uint32_t *data,unsigned int nwords); +int bloom_set (struct bloom *b,uint32_t *data,unsigned int nwords); + +#endif
t/debug.c new file +14 -0
@@ -0,0 +1,14 @@ +#include <stdio.h> + +FILE *f_err; +struct timeval *fio_tv =NULL ; +unsigned int fio_debug =0 ; + +void __dprint (int type,const char *str, ...) +{ +} + +void debug_init (void ) +{ + f_err = stderr; +}
t/debug.h new file +6 -0
@@ -0,0 +1,6 @@ +#ifndef FIO_DEBUG_INC_H +#define FIO_DEBUG_INC_H + +extern void debug_init (void ); + +#endif
t/dedupe.c +106 -44
326 lines not on this page · raw message
t/lfsr-test.c +7 -6
@@ -8,6 +8,8 @@#include <sys/stat.h> #include "../lib/lfsr.h" +#include "../gettime.h" +#include "../fio_time.h" void usage () {
@@ -25,7 +27,7 @@ void usage()int main (int argc,char *argv[]) {int r; -struct timespec start, end; +struct timeval start, end;struct fio_lfsr *fl;int verify =0 ;unsigned int spin =0 ;
@@ -86,12 +88,12 @@ int main(int argc, char *argv[])* negligible overhead. */ fprintf (stderr,"\nTest initiated... " ); -clock_gettime (CLOCK_PROCESS_CPUTIME_ID , &start); +fio_gettime (&start,NULL );while (!lfsr_next (fl, &i, fl->max_val)) {if (verify) *(uint8_t *)(v + i) +=1 ; } -clock_gettime (CLOCK_PROCESS_CPUTIME_ID , &end); +fio_gettime (&end,NULL );fprintf (stderr,"finished.\n" );
@@ -113,8 +115,7 @@ int main(int argc, char *argv[]) }/* Calculate elapsed time and mean time per number */ - total = (end.tv_sec - start.tv_sec) *pow (10 ,9 ) + - end.tv_nsec - start.tv_nsec; + total =utime_since (&start, &end); mean = total / fl->num_vals;printf ("\nTime results " );
@@ -122,7 +123,7 @@ int main(int argc, char *argv[])printf ("(slower due to verification)" );printf ("\n==============================\n" );printf ("Elapsed: %lf s\n" , total /pow (10 ,9 )); -printf ("Mean: %lf ns\n" , mean); +printf ("Mean: %lf us\n" , mean);free (v_start);free (fl);
t/stest.c +2 -10
@@ -4,10 +4,7 @@#include "../smalloc.h" #include "../flist.h" - -FILE *f_err; -struct timeval *fio_tv =NULL ; -unsigned int fio_debug =0 ; +#include "debug.h" #define MAGIC1 0xa9b1c8d2#define MAGIC2 0xf0a1e9b3
@@ -72,9 +69,8 @@ static int do_specific_alloc(unsigned long size)int main (int argc,char *argv[]) { - f_err = stderr; -sinit (); +debug_init ();do_rand_allocs ();
@@ -84,7 +80,3 @@ int main(int argc, char *argv[])scleanup ();return 0 ; } - -void __dprint (int type,const char *str, ...) -{ -}