Recent changes (master)
The following changes since commit a94a977497636bdcbef7106ce3617c96c8ad66bd:
HOWTO: fix unit type suffix in "Parameter types" section to upper case (2017-08-09 08:14:18 -0600)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 29092211c1f926541db0e2863badc03d7378b31a:
HOWTO: update and clarify description of latencies in normal output (2017-08-14 13:02:49 -0600)
----------------------------------------------------------------
Jens Axboe (3):
Merge branch 'serialize_overlap' of https://github.com/sitsofe/fio
backend: cleanup overlap submission logic
Merge branch 'ci' of https://github.com/sitsofe/fio
Sitsofe Wheeler (6):
Makefile: modify make test to use a filesystem file
ci: make CI builds fail on compilation warnings
fio: add serialize_overlap option
iolog: fix double free when verified I/O overlaps
iolog: remove random layout verification optimisation
iolog: tidy up log_io_piece() conditional
Vincent Fu (2):
stat: change indentation of the lat (nsec/usec/msec) section in the normal output
HOWTO: update and clarify description of latencies in normal output
.travis.yml | 2 ++ HOWTO | 44 ++++++++++++++++++++++++++++++++++---------- Makefile | 2 +- appveyor.yml | 2 +- backend.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- cconv.c | 2 ++ fio.1 | 14 ++++++++++++++ init.c | 17 +++++++++++++++++ iolog.c | 24 ++++++++++-------------- options.c | 11 +++++++++++ stat.c | 2 +- thread_options.h | 3 +++ 12 files changed, 142 insertions(+), 29 deletions(-)
---
Diff of recent changes:
.travis.yml +2 -0
@@ -26,3 +26,5 @@ matrix: before_install: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get -qq update; fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -qq -y libaio-dev libnuma-dev libz-dev; fi +script: + - ./configure --extra-cflags="-Werror" && make && make test
HOWTO +34 -10
80 lines not on this page · raw message
Makefile +1 -1
@@ -471,7 +471,7 @@ doc: tools/plot/fio2gnuplot.1 @man -t tools/hist/fiologparser_hist.py.1 | ps2pdf - fiologparser_hist.pdf test: fio - ./fio --minimal --thread --ioengine=null --runtime=1s --name=nulltest --rw=randrw --iodepth=2 --norandommap --random_generator=tausworthe64 --size=16T --name=verifynulltest --rw=write --verify=crc32c --verify_state_save=0 --size=100M + ./fio --minimal --thread --exitall_on_error --runtime=1s --name=nulltest --ioengine=null --rw=randrw --iodepth=2 --norandommap --random_generator=tausworthe64 --size=16T --name=verifyfstest --filename=fiotestfile.tmp --unlink=1 --rw=write --verify=crc32c --verify_state_save=0 --size=16K install: $(PROGS) $(SCRIPTS) tools/plot/fio2gnuplot.1 FORCE $(INSTALL) -m 755 -d $(DESTDIR)$(bindir)
appveyor.yml +1 -1
@@ -13,7 +13,7 @@ environment: build_script: - SET PATH=%CYG_ROOT%\bin;%PATH% - - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && ./configure ${CONFIGURE_OPTIONS} && make.exe' + - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && ./configure --extra-cflags=\"-Werror\" ${CONFIGURE_OPTIONS} && make.exe' after_build: - cd os\windows && dobuild.cmd %BUILD_ARCH%
backend.c +46 -2
66 lines not on this page · raw message
cconv.c +2 -0
@@ -96,6 +96,7 @@ void convert_thread_options_to_cpu(struct thread_options *o, o->iodepth_batch =le32_to_cpu (top->iodepth_batch); o->iodepth_batch_complete_min =le32_to_cpu (top->iodepth_batch_complete_min); o->iodepth_batch_complete_max =le32_to_cpu (top->iodepth_batch_complete_max); + o->serialize_overlap =le32_to_cpu (top->serialize_overlap); o->size =le64_to_cpu (top->size); o->io_size =le64_to_cpu (top->io_size); o->size_percent =le32_to_cpu (top->size_percent);
@@ -346,6 +347,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top, top->iodepth_batch =cpu_to_le32 (o->iodepth_batch); top->iodepth_batch_complete_min =cpu_to_le32 (o->iodepth_batch_complete_min); top->iodepth_batch_complete_max =cpu_to_le32 (o->iodepth_batch_complete_max); + top->serialize_overlap =cpu_to_le32 (o->serialize_overlap); top->size_percent =cpu_to_le32 (o->size_percent); top->fill_device =cpu_to_le32 (o->fill_device); top->file_append =cpu_to_le32 (o->file_append);
fio.1 +14 -0
@@ -1044,6 +1044,20 @@ we simply do polling. Low watermark indicating when to start filling the queue again. Default: \fBiodepth\fR. .TP +.BI serialize_overlap \fR=\fPbool +Serialize in-flight I/Os that might otherwise cause or suffer from data races. +When two or more I/Os are submitted simultaneously, there is no guarantee that +the I/Os will be processed or completed in the submitted order. Further, if +two or more of those I/Os are writes, any overlapping region between them can +become indeterminate/undefined on certain storage. These issues can cause +verification to fail erratically when at least one of the racing I/Os is +changing data and the overlapping region has a non-zero size. Setting +\fBserialize_overlap\fR tells fio to avoid provoking this behavior by explicitly +serializing in-flight I/Os that have a non-zero overlap. Note that setting +this option can reduce both performance and the \fBiodepth\fR achieved. +Additionally this option does not work when \fBio_submit_mode\fR is set to +offload. Default: false. +.TP .BI io_submit_mode \fR=\fPstr This option controls how fio submits the IO to the IO engine. The default is \fBinline\fR, which means that the fio job threads submit and reap IO directly.
init.c +17 -0
@@ -698,6 +698,23 @@ static int fixup_options(struct thread_data *td)if (o->iodepth_batch_complete_min > o->iodepth_batch_complete_max) o->iodepth_batch_complete_max = o->iodepth_batch_complete_min; +/* +* There's no need to check for in-flight overlapping IOs if the job +* isn't changing data or the maximum iodepth is guaranteed to be 1 +*/ +if (o->serialize_overlap && !(td->flags &TD_F_READ_IOLOG ) && + (!(td_write (td) ||td_trim (td)) || o->iodepth ==1 )) + o->serialize_overlap =0 ; +/* +* Currently can't check for overlaps in offload mode +*/ +if (o->serialize_overlap && o->io_submit_mode ==IO_MODE_OFFLOAD ) { +log_err ("fio: checking for in-flight overlaps when the " +"io_submit_mode is offload is not supported\n" ); + o->serialize_overlap =0 ; + ret = warnings_fatal; + } +if (o->nr_files > td->files_index) o->nr_files = td->files_index;
iolog.c +10 -14
@@ -227,21 +227,16 @@ void log_io_piece(struct thread_data *td, struct io_u *io_u) }/* -* We don't need to sort the entries, if: +* We don't need to sort the entries if we only performed sequential +* writes. In this case, just reading back data in the order we wrote +* it out is the faster but still safe. * -* Sequential writes, or -* Random writes that lay out the file as it goes along -* -* For both these cases, just reading back data in the order we -* wrote it out is the fastest. -* -* One exception is if we don't have a random map AND we are doing -* verifies, in that case we need to check for duplicate blocks and -* drop the old one, which we rely on the rb insert/lookup for -* handling. +* One exception is if we don't have a random map in which case we need +* to check for duplicate blocks and drop the old one, which we rely on +* the rb insert/lookup for handling. */ -if (((!td->o.verifysort) || !td_random (td) || !td->o.overwrite) && - (file_randommap (td, ipo->file) || td->o.verify ==VERIFY_NONE )) { +if (((!td->o.verifysort) || !td_random (td)) && +file_randommap (td, ipo->file)) {INIT_FLIST_HEAD (&ipo->list);flist_add_tail (&ipo->list, &td->io_hist_list); ipo->flags |=IP_F_ONLIST ;
@@ -284,7 +279,8 @@ restart: td->io_hist_len--;rb_erase (parent, &td->io_hist_tree);remove_trim_entry (td, __ipo); -free (__ipo); +if (!(__ipo->flags &IP_F_IN_FLIGHT )) +free (__ipo); goto restart; } }
options.c +11 -0
@@ -1882,6 +1882,17 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .group = FIO_OPT_G_IO_BASIC, }, { + .name = "serialize_overlap", + .lname = "Serialize overlap", + .off1 = offsetof(struct thread_options, serialize_overlap), + .type = FIO_OPT_BOOL, + .help = "Wait for in-flight IOs that collide to complete", + .parent = "iodepth", + .def = "0", + .category = FIO_OPT_C_IO, + .group = FIO_OPT_G_IO_BASIC, + }, + { .name = "io_submit_mode", .lname = "IO submit mode", .type = FIO_OPT_STR,
stat.c +1 -1
@@ -520,7 +520,7 @@ static int show_lat(double *io_u_lat, int nr, const char **ranges,if (new_line) {if (line)log_buf (out,"\n" ); -log_buf (out," lat (%s) : " , msg); +log_buf (out," lat (%s) : " , msg); new_line =0 ; line =0 ; }
thread_options.h +3 -0
@@ -65,6 +65,7 @@ struct thread_options {unsigned int iodepth_batch;unsigned int iodepth_batch_complete_min;unsigned int iodepth_batch_complete_max; +unsigned int serialize_overlap;unsigned int unique_filename;
@@ -340,6 +341,8 @@ struct thread_options_pack {uint32_t iodepth_batch;uint32_t iodepth_batch_complete_min;uint32_t iodepth_batch_complete_max; +uint32_t serialize_overlap; +uint32_t pad3;uint64_t size;uint64_t io_size;