Recent changes (master)
The following changes since commit 604577f1329b617d724d6712868d344a5adf5251:
libfio: clear iops/bw sample times on stats reset (2016-05-05 10:55:47 -0600)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to d5bdff69e877a3f65928278df9d252d8881ff864:
Makefile: fix path to tools/fiologparser.py (2016-05-06 17:10:33 -0600)
----------------------------------------------------------------
Brian Boylston (4):
add pmemblk engine
add an example job file for pmemblk
pmemblk: remove comments about an external engine
pmemblk: don't use #defines for the pmemblk_* functions
Jens Axboe (13):
Improve logging accuracy
stat: remove debug statement
Makefile: add tools/fiologpaser.py
Merge branch 'libpmemblk' of https://github.com/bgbhpe/fio
engines/pmeblk: fixup coding style engines/pmemblk: get rid of CACHE_LOCK/UNLOCK defines Wire up pmemblk
Merge branch 'logging'
helper_thread: split into separate file os/os-mac: kill unused code
diskutil: adapt to new helper_thread functions Fix typo in tools/fiologparser.py
Makefile: fix path to tools/fiologparser.py
Mark Nelson (1):
added fio log parser tool.
HOWTO | 11 +- Makefile | 7 +- backend.c | 87 +-------- configure | 11 ++ diskutil.c | 3 +- diskutil.h | 5 +- engines/pmemblk.c | 523 ++++++++++++++++++++++++++++++++++++++++++++++++++ examples/pmemblk.fio | 71 +++++++ fio.1 | 13 +- fio.h | 2 - fio_time.h | 1 + helper_thread.c | 167 ++++++++++++++++ helper_thread.h | 11 ++ init.c | 10 + io_u.c | 18 +- iolog.c | 89 +++++++-- iolog.h | 10 +- libfio.c | 2 + options.c | 6 + os/os-mac.h | 69 ------- stat.c | 152 ++++++++++++--- stat.h | 9 +- time.c | 9 + tools/fiologparser.py | 152 +++++++++++++++ workqueue.c | 5 +- 25 files changed, 1213 insertions(+), 230 deletions(-) create mode 100644 engines/pmemblk.c create mode 100644 examples/pmemblk.fio create mode 100644 helper_thread.c create mode 100644 helper_thread.h create mode 100755 tools/fiologparser.py
---
Diff of recent changes:
HOWTO +9 -2
@@ -798,6 +798,9 @@ ioengine=str Defines how the job issues io to the file. The following overwriting. The writetrim mode works well for this constraint. + pmemblk Read and write through the NVML libpmemblk + interface. + external Prefix to specify loading an external IO engine object file. Append the engine filename, eg ioengine=external:/tmp/foo.o
@@ -1263,10 +1266,14 @@ exitall_on_error When one job finishes in error, terminate the rest. The default is to wait for each job to finish. bwavgtime=int Average the calculated bandwidth over the given time. Value - is specified in milliseconds. + is specified in milliseconds. If the job also does bandwidth + logging through 'write_bw_log', then the minimum of this option + and 'log_avg_msec' will be used. Default: 500ms. iopsavgtime=int Average the calculated IOPS over the given time. Value - is specified in milliseconds. + is specified in milliseconds. If the job also does IOPS logging + through 'write_iops_log', then the minimum of this option and + 'log_avg_msec' will be used. Default: 500ms. create_serialize=bool If true, serialize the file creating for the jobs. This may be handy to avoid interleaving of data
Makefile +5 -2
@@ -26,7 +26,7 @@ OPTFLAGS= -g -ffast-math CFLAGS = -std=gnu99 -Wwrite-strings -Wall -Wdeclaration-after-statement $(OPTFLAGS) $(EXTFLAGS) $(BUILD_CFLAGS) -I. -I$(SRCDIR) LIBS += -lm $(EXTLIBS) PROGS = fio -SCRIPTS = $(addprefix $(SRCDIR)/,tools/fio_generate_plots tools/plot/fio2gnuplot tools/genfio) +SCRIPTS = $(addprefix $(SRCDIR)/,tools/fio_generate_plots tools/plot/fio2gnuplot tools/genfio tools/fiologparser.py) ifndef CONFIG_FIO_NO_OPT CFLAGS += -O3
@@ -45,7 +45,7 @@ SOURCE := $(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/crc/*.c)) \ server.c client.c iolog.c backend.c libfio.c flow.c cconv.c \ gettime-thread.c helpers.c json.c idletime.c td_error.c \ profiles/tiobench.c profiles/act.c io_u_queue.c filelock.c \ - workqueue.c rate-submit.c optgroup.c + workqueue.c rate-submit.c optgroup.c helper_thread.c ifdef CONFIG_LIBHDFS HDFSFLAGS= -I $(JAVA_HOME)/include -I $(JAVA_HOME)/include/linux -I $(FIO_LIBHDFS_INCLUDE)
@@ -124,6 +124,9 @@ ifdef CONFIG_MTD SOURCE += oslib/libmtd.c SOURCE += oslib/libmtd_legacy.c endif +ifdef CONFIG_PMEMBLK + SOURCE += engines/pmemblk.c +endif ifeq ($(CONFIG_TARGET_OS), Linux) SOURCE += diskutil.c fifo.c blktrace.c cgroup.c trim.c engines/sg.c \
backend.c +5 -82
126 lines not on this page · raw message
configure +11 -0
@@ -135,6 +135,7 @@ show_help="no" exit_val=0 gfio_check="no" libhdfs="no" +pmemblk="no" disable_lex="" prefix=/usr/local
@@ -169,6 +170,8 @@ for opt do ;; --enable-libhdfs) libhdfs="yes" ;; + --enable-pmemblk) pmemblk="yes" + ;; --disable-lex) disable_lex="yes" ;; --enable-lex) disable_lex="no"
@@ -199,6 +202,7 @@ if test "$show_help" = "yes" ; then echo "--disable-numa Disable libnuma even if found" echo "--disable-gfapi Disable gfapi" echo "--enable-libhdfs Enable hdfs support" + echo "--enable-pmemblk Enable NVML libpmemblk support" echo "--disable-lex Disable use of lex/yacc for math" echo "--enable-lex Enable use of lex/yacc for math" echo "--disable-shm Disable SHM support"
@@ -1479,6 +1483,10 @@ if compile_prog "" "" "mtd"; then fi echo "MTD $mtd" +########################################## +# Report whether pmemblk engine is enabled +echo "NVML libpmemblk engine $pmemblk" + # Check if we have lex/yacc available yacc="no" yacc_is_bison="no"
@@ -1795,6 +1803,9 @@ if test "$libhdfs" = "yes" ; then if test "$mtd" = "yes" ; then output_sym "CONFIG_MTD" fi +if test "$pmemblk" = "yes" ; then + output_sym "CONFIG_PMEMBLK" +fi if test "$arith" = "yes" ; then output_sym "CONFIG_ARITHMETIC" if test "$yacc_is_bison" = "yes" ; then
diskutil.c +2 -1
@@ -11,6 +11,7 @@#include "fio.h" #include "smalloc.h" #include "diskutil.h" +#include "helper_thread.h" static int last_majdev, last_mindev;static struct disk_util *last_du;
@@ -121,7 +122,7 @@ int update_io_ticks(void)fio_mutex_down (disk_util_mutex); -if (!helper_exit) { +if (!helper_should_exit ()) {flist_for_each (entry, &disk_list) { du = flist_entry(entry, struct disk_util, list);update_io_tick_disk (du);
diskutil.h +2 -3
@@ -4,8 +4,7 @@#define FIO_DU_NAME_SZ 64#include "lib/output_buffer.h" - -extern volatile int helper_exit; +#include "helper_thread.h" struct disk_util_stats {uint64_t ios[2 ];
@@ -129,7 +128,7 @@ static inline void print_disk_util(struct disk_util_stat *du,static inline int update_io_ticks (void ) { -return helper_exit; +return helper_should_exit (); }#endif
engines/pmemblk.c new file +523 -0
523 lines not on this page · raw message
examples/pmemblk.fio new file +71 -0
71 lines not on this page · raw message
fio.1 +9 -4
@@ -700,6 +700,9 @@ treated as erases. Depending on the underlying device type, the I/O may have to go in a certain pattern, e.g., on NAND, writing sequentially to erase blocks and discarding before overwriting. The writetrim mode works well for this constraint. +.TP +.B pmemblk +Read and write through the NVML libpmemblk interface. .RE .P .RE
@@ -1180,12 +1183,14 @@ Terminate all jobs if one job finishes in error. Default: wait for each job to finish. .TP .BI bwavgtime \fR=\fPint -Average bandwidth calculations over the given time in milliseconds. Default: -500ms. +Average bandwidth calculations over the given time in milliseconds. If the job +also does bandwidth logging through \fBwrite_bw_log\fR, then the minimum of +this option and \fBlog_avg_msec\fR will be used. Default: 500ms. .TP .BI iopsavgtime \fR=\fPint -Average IOPS calculations over the given time in milliseconds. Default: -500ms. +Average IOPS calculations over the given time in milliseconds. If the job +also does IOPS logging through \fBwrite_iops_log\fR, then the minimum of +this option and \fBlog_avg_msec\fR will be used. Default: 500ms. .TP .BI create_serialize \fR=\fPbool If true, serialize file creation for the jobs. Default: true.
fio.h +0 -2
@@ -445,8 +445,6 @@ extern int nr_clients;extern int log_syslog;extern int status_interval;extern const char fio_version_string[]; -extern int helper_do_stat; -extern pthread_cond_t helper_cond;extern char *trigger_file;extern char *trigger_cmd;extern char *trigger_remote_cmd;
fio_time.h +1 -0
@@ -17,5 +17,6 @@ extern void set_genesis_time(void);extern int ramp_time_over (struct thread_data *);extern int in_ramp_time (struct thread_data *);extern void fio_time_init (void ); +extern void timeval_add_msec (struct timeval *,unsigned int );#endif
helper_thread.c new file +167 -0
167 lines not on this page · raw message
helper_thread.h new file +11 -0
@@ -0,0 +1,11 @@ +#ifndef FIO_HELPER_THREAD_H +#define FIO_HELPER_THREAD_H + +extern void helper_reset (void ); +extern void helper_do_stat (void ); +extern bool helper_should_exit (void ); +extern void helper_thread_destroy (void ); +extern void helper_thread_exit (void ); +extern int helper_thread_create (struct fio_mutex *,struct sk_out *); + +#endif
init.c +10 -0
@@ -1416,6 +1416,11 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, };const char *suf; +if (fio_option_is_set (o, bw_avg_time)) + p.avg_msec =min (o->log_avg_msec, o->bw_avg_time); +else + o->bw_avg_time = p.avg_msec; +if (p.log_gz_store) suf ="log.fz" ;else
@@ -1436,6 +1441,11 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, };const char *suf; +if (fio_option_is_set (o, iops_avg_time)) + p.avg_msec =min (o->log_avg_msec, o->iops_avg_time); +else + o->iops_avg_time = p.avg_msec; +if (p.log_gz_store) suf ="log.fz" ;else
io_u.c +10 -8
@@ -1710,16 +1710,18 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u, } } -if (!td->o.disable_clat) { -add_clat_sample (td, idx, lusec, bytes, io_u->offset); -io_u_mark_latency (td, lusec); - } +if (ddir_rw (idx)) { +if (!td->o.disable_clat) { +add_clat_sample (td, idx, lusec, bytes, io_u->offset); +io_u_mark_latency (td, lusec); + } -if (!td->o.disable_bw) -add_bw_sample (td, idx, bytes, &icd->time); +if (!td->o.disable_bw &&per_unit_log (td->bw_log)) +add_bw_sample (td, io_u, bytes, lusec); -if (no_reduce) -add_iops_sample (td, idx, bytes, &icd->time); +if (no_reduce &&per_unit_log (td->iops_log)) +add_iops_sample (td, io_u, bytes); + }if (td->ts.nr_block_infos && io_u->ddir ==DDIR_TRIM ) {uint32_t *info =io_u_block_info (td, io_u);
iolog.c +71 -18
158 lines not on this page · raw message
iolog.h +8 -2
@@ -207,12 +207,18 @@ struct log_params {int log_compress; }; -extern void finalize_logs (struct thread_data *td); +static inline bool per_unit_log (struct io_log *log) +{ +return log && !log->avg_msec; +} + +extern void finalize_logs (struct thread_data *td,bool );extern void setup_log (struct io_log **,struct log_params *,const char *);extern void flush_log (struct io_log *,int );extern void flush_samples (FILE *,void *,uint64_t );extern void free_log (struct io_log *); -extern void fio_writeout_logs (struct thread_data *); +extern void fio_writeout_logs (bool ); +extern void td_writeout_logs (struct thread_data *,bool );extern int iolog_flush (struct io_log *,int );static inline void init_ipo (struct io_piece *ipo)
libfio.c +2 -0
@@ -33,6 +33,7 @@#include "smalloc.h" #include "os/os.h" #include "filelock.h" +#include "helper_thread.h" /* * Just expose an empty list, if the OS does not support disk util stats
@@ -151,6 +152,7 @@ void reset_all_stats(struct thread_data *td)lat_target_reset (td);clear_rusage_stat (td); +helper_reset (); }void reset_fio_state (void )
options.c +6 -0
@@ -1569,6 +1569,12 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .help = "Hadoop Distributed Filesystem (HDFS) engine" },#endif +#ifdef CONFIG_PMEMBLK + { .ival = "pmemblk", + .help = "NVML libpmemblk based IO engine", + }, + +#endif { .ival = "external", .help = "Load external engine (append name)", },
os/os-mac.h +0 -69
76 lines not on this page · raw message
stat.c +121 -31
243 lines not on this page · raw message
stat.h +5 -4
@@ -276,11 +276,12 @@ extern void add_clat_sample(struct thread_data *, enum fio_ddir, unsigned long, unsigned int, uint64_t);extern void add_slat_sample (struct thread_data *,enum fio_ddir ,unsigned long ,unsigned int ,uint64_t ); -extern void add_bw_sample (struct thread_data *,enum fio_ddir ,unsigned int , -struct timeval *); -extern void add_iops_sample (struct thread_data *,enum fio_ddir ,unsigned int , -struct timeval *);extern void add_agg_sample (unsigned long ,enum fio_ddir ,unsigned int ); +extern void add_iops_sample (struct thread_data *,struct io_u *, +unsigned int ); +extern void add_bw_sample (struct thread_data *,struct io_u *, +unsigned int ,unsigned long ); +extern int calc_log_samples (void );extern struct io_log *agg_io_log[DDIR_RWDIR_CNT ];extern int write_bw_log;
time.c +9 -0
@@ -6,6 +6,15 @@static struct timeval genesis;static unsigned long ns_granularity; +void timeval_add_msec (struct timeval *tv,unsigned int msec) +{ + tv->tv_usec +=1000 * msec; +if (tv->tv_usec >=1000000 ) { + tv->tv_usec -=1000000 ; + tv->tv_sec++; + } +} +/* * busy looping version for the last few usec */
tools/fiologparser.py new file +152 -0
152 lines not on this page · raw message
workqueue.c +3 -2
@@ -9,6 +9,7 @@#include "fio.h" #include "flist.h" #include "workqueue.h" +#include "smalloc.h" enum {SW_F_IDLE =1 <<0 ,
@@ -263,7 +264,7 @@ void workqueue_exit(struct workqueue *wq) } }while (shutdown && shutdown != wq->max_workers); -free (wq->workers); +sfree (wq->workers); wq->workers =NULL ;pthread_mutex_destroy (&wq->flush_lock);pthread_cond_destroy (&wq->flush_cond);
@@ -317,7 +318,7 @@ int workqueue_init(struct thread_data *td, struct workqueue *wq,pthread_mutex_init (&wq->flush_lock,NULL );pthread_mutex_init (&wq->stat_lock,NULL ); - wq->workers =calloc (wq->max_workers,sizeof (struct submit_worker )); + wq->workers =smalloc (wq->max_workers *sizeof (struct submit_worker ));for (i =0 ; i < wq->max_workers; i++)if (start_worker (wq, i, sk_out))