Recent changes (master)
Subsystems (2)
The following changes since commit 5bb79f69c2d9dc8542c25af96f040d1884230688:
workqueue: remove knowledge of td queue state (2015-12-07 22:35:31 -0700)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to c08f9fe23b0f257f914b2d9e0e4f1117418e5da6:
options: add log_compression_cpus option (2015-12-08 15:45:12 -0700)
----------------------------------------------------------------
Jens Axboe (21):
workqueue: move 'td' private data to the workqueue user
workqueue: move init worker private code to the caller
workqueue: move private exit code to caller
workqueue: move private accounting to caller
workqueue: move last bits of end accounting to caller
backend: move rated submit code to its own file
workqueue: add nice support
iolog: replace tp usage with workqueue
workqueue: ensure that workqueue_enqueue() can't fail workqueue/rate-submit: update comments
iolog: reinstate log compression nice level
rate-submit: clean up init/exit helpers
workqueue: make workqueue_exit() safe for multiple exit calls
iolog: remember to destroy cond/lock when done
iolog: cleanup data wait
iolog: fix build with zlib not being installed
iolog: fix potential duplicate definition of compress init/exit
iolog: cleanup log writing
iolog: cleanup up data unref'ing
options: check for dryrun in cpu mask setting
options: add log_compression_cpus option
HOWTO | 14 ++-- Makefile | 2 +- backend.c | 94 ++------------------- cconv.c | 2 + fio.1 | 12 ++- fio.h | 6 +- iolog.c | 152 +++++++++++++++++++++------------ iolog.h | 2 + lib/tp.c | 119 -------------------------- lib/tp.h | 33 -------- options.c | 27 ++++++ rate-submit.c | 252 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ rate-submit.h | 7 ++ server.h | 2 +- thread_options.h | 2 + workqueue.c | 205 ++++++++++---------------------------------- workqueue.h | 71 ++++++++++++++-- 17 files changed, 534 insertions(+), 468 deletions(-) delete mode 100644 lib/tp.c delete mode 100644 lib/tp.h create mode 100644 rate-submit.c create mode 100644 rate-submit.h
---
Diff of recent changes:
HOWTO +9 -5
@@ -1580,11 +1580,15 @@ log_compression=int If this is set, fio will compress the IO logs as in the specified log file. This feature depends on the availability of zlib. -log_store_compressed=bool If set, and log_compression is also set, - fio will store the log files in a compressed format. They - can be decompressed with fio, using the --inflate-log - command line parameter. The files will be stored with a - .fz suffix. +log_compression_cpus=str Define the set of CPUs that are allowed to + handle online log compression for the IO jobs. This can + provide better isolation between performance sensitive jobs, + and background compression work. + +log_store_compressed=bool If set, fio will store the log files in a + compressed format. They can be decompressed with fio, using + the --inflate-log command line parameter. The files will be + stored with a .fz suffix. block_error_percentiles=bool If set, record errors in trim block-sized units from writes and trims and output a histogram of
Makefile +1 -1
@@ -45,7 +45,7 @@ SOURCE := $(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/crc/*.c)) \ server.c client.c iolog.c backend.c libfio.c flow.c cconv.c \ gettime-thread.c helpers.c json.c idletime.c td_error.c \ profiles/tiobench.c profiles/act.c io_u_queue.c filelock.c \ - workqueue.c + workqueue.c rate-submit.c ifdef CONFIG_LIBHDFS HDFSFLAGS= -I $(JAVA_HOME)/include -I $(JAVA_HOME)/include/linux -I $(FIO_LIBHDFS_INCLUDE)
backend.c +9 -85
130 lines not on this page · raw message
cconv.c +2 -0
@@ -261,6 +261,7 @@ void convert_thread_options_to_cpu(struct thread_options *o,#if 0 uint8_t cpumask[FIO_TOP_STR_MAX ];uint8_t verify_cpumask[FIO_TOP_STR_MAX ]; +uint8_t log_gz_cpumask[FIO_TOP_STR_MAX ];#endif }
@@ -482,6 +483,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,#if 0 uint8_t cpumask[FIO_TOP_STR_MAX ];uint8_t verify_cpumask[FIO_TOP_STR_MAX ]; +uint8_t log_gz_cpumask[FIO_TOP_STR_MAX ];#endif }
fio.1 +8 -4
@@ -1427,11 +1427,15 @@ most of the system memory. So pick your poison. The IO logs are saved normally at the end of a run, by decompressing the chunks and storing them in the specified log file. This feature depends on the availability of zlib. .TP +.BI log_compression_cpus \fR=\fPstr +Define the set of CPUs that are allowed to handle online log compression +for the IO jobs. This can provide better isolation between performance +sensitive jobs, and background compression work. +.TP .BI log_store_compressed \fR=\fPbool -If set, and \fBlog\fR_compression is also set, fio will store the log files in -a compressed format. They can be decompressed with fio, using the -\fB\-\-inflate-log\fR command line parameter. The files will be stored with a -\fB\.fz\fR suffix. +If set, fio will store the log files in a compressed format. They can be +decompressed with fio, using the \fB\-\-inflate-log\fR command line parameter. +The files will be stored with a \fB\.fz\fR suffix. .TP .BI block_error_percentiles \fR=\fPbool If set, record errors in trim block-sized units from writes and trims and output
fio.h +5 -1
@@ -129,7 +129,7 @@ struct thread_data {struct io_log *bw_log;struct io_log *iops_log; -struct tp_data *tp_data; +struct workqueue log_compress_wq;struct thread_data *parent;
@@ -565,6 +565,10 @@ extern int is_blktrace(const char *, int *);extern int load_blktrace (struct thread_data *,const char *,int );#endif +extern int io_queue_event (struct thread_data *td,struct io_u *io_u,int *ret, +enum fio_ddir ddir,uint64_t *bytes_issued,int from_verify, +struct timeval *comp_time); +/* * Latency target helpers */
iolog.c +100 -52
239 lines not on this page · raw message
iolog.h +2 -0
@@ -184,6 +184,8 @@ extern void trim_io_piece(struct thread_data *, const struct io_u *);extern void queue_io_piece (struct thread_data *,struct io_piece *);extern void prune_io_piece_log (struct thread_data *);extern void write_iolog_close (struct thread_data *); +extern int iolog_compress_init (struct thread_data *); +extern void iolog_compress_exit (struct thread_data *);#ifdef CONFIG_ZLIB extern int iolog_file_inflate (const char *);
lib/tp.c deleted +0 -119
119 lines not on this page · raw message
lib/tp.h deleted +0 -33
@@ -1,33 +0,0 @@ -#ifndef FIO_TP_H -#define FIO_TP_H - -#include "../flist.h" - -struct tp_work ; -typedef int (tp_work_fn )(struct tp_work *); - -struct tp_work { -struct flist_head list; -tp_work_fn *fn; -int wait; -int prio; -pthread_cond_t cv; -pthread_mutex_t lock; -volatile int done; -}; - -struct tp_data { -pthread_t thread; -pthread_cond_t cv; -pthread_mutex_t lock; -struct flist_head work; -volatile int thread_exit; -pthread_cond_t sleep_cv; -volatile int sleeping; -}; - -extern void tp_init (struct tp_data **); -extern void tp_exit (struct tp_data **); -extern void tp_queue_work (struct tp_data *,struct tp_work *); - -#endif
options.c +27 -0
@@ -530,8 +530,22 @@ static int str_verify_cpus_allowed_cb(void *data, const char *input) {struct thread_data *td = data; +if (parse_dryrun ()) +return 0 ; +return set_cpus_allowed (td, &td->o.verify_cpumask, input); } + +static int str_log_cpus_allowed_cb (void *data,const char *input) +{ +struct thread_data *td = data; + +if (parse_dryrun ()) +return 0 ; + +return set_cpus_allowed (td, &td->o.log_gz_cpumask, input); +} +#endif #ifdef CONFIG_LIBNUMA
@@ -3226,6 +3240,19 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .category = FIO_OPT_C_LOG, .group = FIO_OPT_G_INVALID, }, +#ifdef FIO_HAVE_CPU_AFFINITY + { + .name = "log_compression_cpus", + .lname = "Log Compression CPUs", + .type = FIO_OPT_STR, + .cb = str_log_cpus_allowed_cb, + .off1 = td_var_offset(log_gz_cpumask), + .parent = "log_compression", + .help = "Limit log compression to these CPUs", + .category = FIO_OPT_C_LOG, + .group = FIO_OPT_G_INVALID, + }, +#endif { .name = "log_store_compressed", .lname = "Log store compressed",
rate-submit.c new file +252 -0
252 lines not on this page · raw message
rate-submit.h new file +7 -0
@@ -0,0 +1,7 @@ +#ifndef FIO_RATE_SUBMIT +#define FIO_RATE_SUBMIT + +int rate_submit_init (struct thread_data *); +void rate_submit_exit (struct thread_data *); + +#endif
server.h +1 -1
@@ -38,7 +38,7 @@ struct fio_net_cmd_reply { };enum { -FIO_SERVER_VER =48 , +FIO_SERVER_VER =49 ,FIO_SERVER_MAX_FRAGMENT_PDU =1024 ,FIO_SERVER_MAX_CMD_MB =2048 ,
thread_options.h +2 -0
@@ -170,6 +170,7 @@ struct thread_options {unsigned int numjobs;os_cpu_mask_t cpumask;os_cpu_mask_t verify_cpumask; +os_cpu_mask_t log_gz_cpumask;unsigned int cpus_allowed_policy;char *numa_cpunodes;unsigned short numa_mem_mode;
@@ -415,6 +416,7 @@ struct thread_options_pack {uint32_t numjobs;uint8_t cpumask[FIO_TOP_STR_MAX ];uint8_t verify_cpumask[FIO_TOP_STR_MAX ]; +uint8_t log_gz_cpumask[FIO_TOP_STR_MAX ];uint32_t cpus_allowed_policy;uint32_t iolog;uint32_t rwmixcycle;
workqueue.c +46 -159
301 lines not on this page · raw message
workqueue.h +63 -8
98 lines not on this page · raw message