Recent changes (master)
The following changes since commit 695611a9d4cd554d44d8b2ec5da2811061950a2e:
Allow offload with FAKEIO engines (2020-09-11 09:58:15 -0600)
are available in the Git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 7064f8942a3b8070acf60b8e5fabc16f8221ae40:
Merge branch 'msys2' of https://github.com/sitsofe/fio into master (2020-09-14 19:43:39 -0600)
----------------------------------------------------------------
Bart Van Assche (2):
backend: Remove two superfluous casts
backend: Use asprintf() instead of strlen() + sprintf()
Jens Axboe (2):
Merge branch 'backend' of https://github.com/bvanassche/fio into master Merge branch 'msys2' of https://github.com/sitsofe/fio into master
Sitsofe Wheeler (12):
configure: pass non-null pointer to (v)asprintf
net: coerce the result of htonl before printing
windows: fix wrong format strings
windows: fix DWORD format string complaints
configure: be explicit about "XP" Windows API version configure/Makefile: add option to generate pdb symbols Makefile/ci: Don't pass CFLAGS when linking
appveyor: cleanup and add separate install script
Makefile: introduce FIO_CFLAGS
memlock: avoid type confusion in format string
configure: cleanup lex/yacc tests
travis: cleanup build script
.appveyor.yml | 40 ++++++++++++++++-------- Makefile | 77 ++++++++++++++++++++++++++--------------------- backend.c | 15 ++++----- ci/appveyor-install.sh | 41 +++++++++++++++++++++++++ ci/travis-build.sh | 8 +++-- ci/travis-install.sh | 5 ++- configure | 60 ++++++++++++++++++++++++++---------- engines/net.c | 5 +-- engines/windowsaio.c | 8 ++--- os/windows/cpu-affinity.c | 26 ++++++++-------- os/windows/dobuild.cmd | 10 ++++-- os/windows/install.wxs | 8 +++++ os/windows/posix.c | 5 +-- t/memlock.c | 2 +- 14 files changed, 211 insertions(+), 99 deletions(-) create mode 100755 ci/appveyor-install.sh
---
Diff of recent changes:
.appveyor.yml +28 -12
@@ -5,33 +5,49 @@ image: environment: CYG_MIRROR: http://cygwin.mirror.constant.com - CYG_ROOT: C:\cygwin64 - MAKEFLAGS: -j 2 matrix: - - platform: x64 - PACKAGE_ARCH: x86_64 + - ARCHITECTURE: x64 + CC: clang + CONFIGURE_OPTIONS: --enable-pdb + DISTRO: msys2 +# Skip 32 bit clang build +# - ARCHITECTURE: x86 +# CC: clang +# CONFIGURE_OPTIONS: --enable-pdb +# DISTRO: msys2 + - ARCHITECTURE: x64 CONFIGURE_OPTIONS: - - platform: x86 - PACKAGE_ARCH: i686 + DISTRO: cygwin + - ARCHITECTURE: x86 CONFIGURE_OPTIONS: --build-32bit-win --target-win-ver=xp + DISTRO: cygwin install: - - '%CYG_ROOT%\setup-x86_64.exe --quiet-mode --no-shortcuts --only-site --site "%CYG_MIRROR%" --packages "mingw64-%PACKAGE_ARCH%-zlib,mingw64-%PACKAGE_ARCH%-CUnit" > NUL' - - SET PATH=C:\Python38-x64;%CYG_ROOT%\bin;%PATH% # NB: Changed env variables persist to later sections + - if %DISTRO%==cygwin ( + SET "PATH=C:\cygwin64\bin;C:\cygwin64;%PATH%" + ) + - if %DISTRO%==msys2 if %ARCHITECTURE%==x86 ( + SET "PATH=C:\msys64\mingw32\bin;C:\msys64\usr\bin;%PATH%" + ) + - if %DISTRO%==msys2 if %ARCHITECTURE%==x64 ( + SET "PATH=C:\msys64\mingw64\bin;C:\msys64\usr\bin;%PATH%" + ) + - SET PATH=C:\Python38-x64;%PATH% # NB: Changed env variables persist to later sections - SET PYTHONUNBUFFERED=TRUE - - python.exe -m pip install scipy six + - bash.exe ci\appveyor-install.sh build_script: - - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && ./configure --disable-native --extra-cflags=\"-Werror\" ${CONFIGURE_OPTIONS} && make.exe' + - bash.exe configure --extra-cflags=-Werror --disable-native %CONFIGURE_OPTIONS% + - make.exe -j2 after_build: - file.exe fio.exe - make.exe test - - 'cd os\windows && dobuild.cmd %PLATFORM% && cd ..' + - 'cd os\windows && dobuild.cmd %ARCHITECTURE% && cd ..' - ps: Get-ChildItem .\os\windows\*.msi | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name -DeploymentName fio.msi } test_script: - - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && [ -f fio.exe ] && python.exe t/run-fio-tests.py --artifact-root test-artifacts --debug' + - python.exe t/run-fio-tests.py --artifact-root test-artifacts --debug on_finish: - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && [ -d test-artifacts ] && 7z a -t7z test-artifacts.7z test-artifacts -xr!foo.0.0 -xr!latency.?.0 -xr!fio_jsonplus_clat2csv.test && appveyor PushArtifact test-artifacts.7z'
Makefile +42 -35
196 lines not on this page · raw message
backend.c +8 -7
@@ -1458,16 +1458,17 @@ static bool keep_running(struct thread_data *td)return false; } -static int exec_string (struct thread_options *o,const char *string,const char *mode) +static int exec_string (struct thread_options *o,const char *string, +const char *mode) { -size_t newlen =strlen (string) +strlen (o->name) +strlen (mode) +13 +1 ;int ret;char *str; - str =malloc (newlen); -sprintf (str,"%s > %s.%s.txt 2>&1" , string, o->name, mode); +if (asprintf (&str,"%s > %s.%s.txt 2>&1" , string, o->name, mode) <0 ) +return -1 ; -log_info ("%s : Saving output of %s in %s.%s.txt\n" ,o->name, mode, o->name, mode); +log_info ("%s : Saving output of %s in %s.%s.txt\n" , o->name, mode, + o->name, mode); ret =system (str);if (ret ==-1 )log_err ("fio: exec of cmd <%s> failed\n" , str);
@@ -1731,7 +1732,7 @@ static void *thread_main(void *data)if (!init_random_map (td)) goto err; -if (o->exec_prerun &&exec_string (o, o->exec_prerun, (const char *)"prerun" )) +if (o->exec_prerun &&exec_string (o, o->exec_prerun,"prerun" )) goto err;if (o->pre_read && !pre_read_files (td))
@@ -1890,7 +1891,7 @@ static void *thread_main(void *data)rate_submit_exit (td);if (o->exec_postrun) -exec_string (o, o->exec_postrun, (const char *)"postrun" ); +exec_string (o, o->exec_postrun,"postrun" );if (exitall_on_terminate || (o->exitall_error && td->error))fio_terminate_threads (td->groupid, td->o.exit_what);
ci/appveyor-install.sh new file +41 -0
41 lines not on this page · raw message
ci/travis-build.sh +5 -3
@@ -1,8 +1,9 @@ #!/bin/bash +set -eu CI_TARGET_ARCH="${BUILD_ARCH:-$TRAVIS_CPU_ARCH}" EXTRA_CFLAGS="-Werror" -PYTHONUNBUFFERED=TRUE +export PYTHONUNBUFFERED=TRUE CONFIGURE_FLAGS=() case "$TRAVIS_OS_NAME" in
@@ -11,6 +12,7 @@ case "$TRAVIS_OS_NAME" in case "$CI_TARGET_ARCH" in "x86") EXTRA_CFLAGS="${EXTRA_CFLAGS} -m32" + export LDFLAGS="-m32" ;; "amd64") CONFIGURE_FLAGS+=(--enable-cuda)
@@ -24,7 +26,7 @@ CONFIGURE_FLAGS+=(--extra-cflags="${EXTRA_CFLAGS}") make && make test && if [[ "$CI_TARGET_ARCH" == "arm64" ]]; then - sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug -p 1010:"--skip 15 16 17 18 19 20" + sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug -p 1010:"--skip 15 16 17 18 19 20" else - sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug + sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug fi
ci/travis-install.sh +2 -3
@@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu CI_TARGET_ARCH="${BUILD_ARCH:-$TRAVIS_CPU_ARCH}" case "$TRAVIS_OS_NAME" in
@@ -51,6 +51,5 @@ case "$TRAVIS_OS_NAME" in ;; esac -echo "Python version: $(/usr/bin/python -V 2>&1)" -echo "Python3 path: $(which python3 2>&1)" +echo "Python3 path: $(type -p python3 2>&1)" echo "Python3 version: $(python3 -V 2>&1)"
configure +44 -16
131 lines not on this page · raw message
engines/net.c +3 -2
@@ -938,8 +938,9 @@ static int fio_netio_udp_recv_open(struct thread_data *td, struct fio_file *f)if (ntohl (msg.magic) !=FIO_LINK_OPEN_CLOSE_MAGIC ||ntohl (msg.cmd) !=FIO_LINK_OPEN ) { -log_err ("fio: bad udp open magic %x/%x\n" ,ntohl (msg.magic), -ntohl (msg.cmd)); +log_err ("fio: bad udp open magic %x/%x\n" , + (unsigned int )ntohl (msg.magic), + (unsigned int )ntohl (msg.cmd));return -1 ; }
engines/windowsaio.c +4 -4
@@ -161,15 +161,15 @@ static int windowsaio_invalidate_cache(struct fio_file *f)if (ihFile !=INVALID_HANDLE_VALUE ) {if (!CloseHandle (ihFile)) { error =GetLastError (); -log_info ("windowsaio: invalidation fd close %s " -"failed: error %d\n" , f->file_name, error); +log_info ("windowsaio: invalidation fd close %s failed: error %lu\n" , + f->file_name, error); rc =1 ; } }else { error =GetLastError ();if (error !=ERROR_FILE_NOT_FOUND ) { -log_info ("windowsaio: cache invalidation of %s failed: " -"error %d\n" , f->file_name, error); +log_info ("windowsaio: cache invalidation of %s failed: error %lu\n" , + f->file_name, error); rc =1 ; } }
os/windows/cpu-affinity.c +14 -12
@@ -14,7 +14,7 @@ int fio_setaffinity(int pid, os_cpu_mask_t cpumask) bSuccess =SetThreadAffinityMask (h, cpumask);if (!bSuccess)log_err ("fio_setaffinity failed: failed to set thread affinity (pid %d, mask %.16llx)\n" , - pid, cpumask); + pid, (long long unsigned ) cpumask);CloseHandle (h); }else {
@@ -83,7 +83,7 @@ unsigned int cpus_online(void)static void print_mask (os_cpu_mask_t *cpumask) {for (int i =0 ; i <FIO_CPU_MASK_ROWS ; i++) -dprint (FD_PROCESS ,"cpumask[%d]=%lu\n" , i, cpumask->row[i]); +dprint (FD_PROCESS ,"cpumask[%d]=%" PRIu64"\n" , i, cpumask->row[i]); }/* Return the index of the least significant set CPU in cpumask or -1 if no
@@ -99,7 +99,7 @@ int first_set_cpu(os_cpu_mask_t *cpumask)int row_first_cpu; row_first_cpu =__builtin_ffsll (cpumask->row[row]) -1 ; -dprint (FD_PROCESS ,"row_first_cpu=%d cpumask->row[%d]=%lu\n" , +dprint (FD_PROCESS ,"row_first_cpu=%d cpumask->row[%d]=%" PRIu64"\n" , row_first_cpu, row, cpumask->row[row]);if (row_first_cpu >-1 ) { mask_first_cpu = cpus_offset + row_first_cpu;
@@ -136,7 +136,7 @@ static int last_set_cpu(os_cpu_mask_t *cpumask) row_last_cpu++; } -dprint (FD_PROCESS ,"row_last_cpu=%d cpumask->row[%d]=%lu\n" , +dprint (FD_PROCESS ,"row_last_cpu=%d cpumask->row[%d]=%" PRIu64"\n" , row_last_cpu, row, cpumask->row[row]);if (row_last_cpu >-1 ) { mask_last_cpu = cpus_offset + row_last_cpu;
@@ -213,13 +213,17 @@ static int mask_to_group_mask(os_cpu_mask_t *cpumask, int *processor_group, uint needed_shift =FIO_CPU_MASK_STRIDE - bit_offset; needed_mask_shift =FIO_CPU_MASK_STRIDE - needed; needed_mask = (uint64_t )-1 >> needed_mask_shift; -dprint (FD_PROCESS ,"bit_offset=%d end=%d needed=%d needed_shift=%d needed_mask=%ld needed_mask_shift=%d\n" , bit_offset, end, needed, needed_shift, needed_mask, needed_mask_shift); +dprint (FD_PROCESS , +"bit_offset=%d end=%d needed=%d needed_shift=%d needed_mask=%" PRIu64"needed_mask_shift=%d\n" , + bit_offset, end, needed, needed_shift, needed_mask, + needed_mask_shift); group_cpumask |= (cpumask->row[row +1 ] & needed_mask) << needed_shift; } group_cpumask &= (uint64_t )-1 >> (FIO_CPU_MASK_STRIDE - group_size);/* Return group and mask */ -dprint (FD_PROCESS ,"Returning group=%d group_mask=%lu\n" , group, group_cpumask); +dprint (FD_PROCESS ,"Returning group=%d group_mask=%" PRIu64"\n" , + group, group_cpumask); *processor_group = group; *affinity_mask = group_cpumask;
@@ -257,10 +261,8 @@ int fio_setaffinity(int pid, os_cpu_mask_t cpumask)if (SetThreadGroupAffinity (handle, &new_group_affinity,NULL ) !=0 ) ret =0 ;else { -log_err ("fio_setaffinity: failed to set thread affinity " -"(pid %d, group %d, mask %" PRIx64", " -"GetLastError=%d)\n" , pid, group, group_mask, -GetLastError ()); +log_err ("fio_setaffinity: failed to set thread affinity (pid %d, group %d, mask %" PRIx64", GetLastError=%lu)\n" , + pid, group, group_mask,GetLastError ()); goto err; }
@@ -319,7 +321,7 @@ int fio_getaffinity(int pid, os_cpu_mask_t *mask) goto err; }if (!GetProcessGroupAffinity (handle, &group_count, current_groups)) { -log_err ("%s: failed to get single group affinity for pid %d (%d)\n" , +log_err ("%s: failed to get single group affinity for pid %d (%lu)\n" , __func__, pid,GetLastError ()); goto err; }
@@ -329,7 +331,7 @@ int fio_getaffinity(int pid, os_cpu_mask_t *mask) goto err; }if (!GetProcessAffinityMask (handle, &process_mask, &system_mask)) { -log_err ("%s: GetProcessAffinityMask() failed for pid\n" , +log_err ("%s: GetProcessAffinityMask() failed for pid %d\n" , __func__, pid); goto err; }
os/windows/dobuild.cmd +8 -2
@@ -34,7 +34,13 @@ if defined SIGN_FIO ( signtool sign /as /n "%SIGNING_CN%" /tr http://timestamp.digicert.com /td sha256 /fd sha256 ..\..\t\*.exe ) -"%WIX%bin\candle" -nologo -arch %FIO_ARCH% -dFioVersionNumbers="%FIO_VERSION_NUMBERS%" install.wxs +if exist ..\..\fio.pdb ( + set FIO_PDB=true +) else ( + set FIO_PDB=false +) + +"%WIX%bin\candle" -nologo -arch %FIO_ARCH% -dFioVersionNumbers="%FIO_VERSION_NUMBERS%" -dFioPDB="%FIO_PDB%" install.wxs @if ERRORLEVEL 1 goto end "%WIX%bin\candle" -nologo -arch %FIO_ARCH% examples.wxs @if ERRORLEVEL 1 goto end
@@ -43,4 +49,4 @@ if defined SIGN_FIO ( if defined SIGN_FIO ( signtool sign /n "%SIGNING_CN%" /tr http://timestamp.digicert.com /td sha256 /fd sha256 %FIO_VERSION%-%FIO_ARCH%.msi -) +)
os/windows/install.wxs +8 -0
@@ -27,6 +27,11 @@ <File Source="..\..\fio.exe"/> <Environment Action="set" Part="last" Id="PATH" Name="PATH" Value="[INSTALLDIR]fio\" System="yes"/> </Component> + <?if $(var.FioPDB) = true?> + <Component> + <File Id="fio.pdb" Name="fio.pdb" Source="..\..\fio.pdb"/> + </Component> + <?endif?> <Component> <File Id="README" Name="README.txt" Source="..\..\README"/> </Component>
@@ -76,6 +81,9 @@ <Feature Id="AlwaysInstall" Absent="disallow" ConfigurableDirectory="INSTALLDIR" Display="hidden" Level="1" Title="Flexible I/O Tester"> <ComponentRef Id="fio.exe"/> + <?if $(var.FioPDB) = true?> + <ComponentRef Id="fio.pdb"/> + <?endif?> <ComponentRef Id="HOWTO"/> <ComponentRef Id="README"/> <ComponentRef Id="REPORTING_BUGS"/>
os/windows/posix.c +3 -2
@@ -168,7 +168,7 @@ int win_to_posix_error(DWORD winerr)case ERROR_FILE_INVALID :return ENXIO ;default : -log_err ("fio: windows error %d not handled\n" , winerr); +log_err ("fio: windows error %lu not handled\n" , winerr);return EIO ; }
@@ -188,7 +188,8 @@ int GetNumLogicalProcessors(void)if (error ==ERROR_INSUFFICIENT_BUFFER ) processor_info =malloc (len);else { -log_err ("Error: GetLogicalProcessorInformation failed: %d\n" , error); +log_err ("Error: GetLogicalProcessorInformation failed: %lu\n" , + error);return -1 ; }
t/memlock.c +1 -1
@@ -22,7 +22,7 @@ static void *worker(void *data)for (index =0 ; index +4096 < size; index +=4096 )memset (&buf[index+512 ],0x89 ,512 );if (first) { -printf ("loop%d: did %lu MiB\n" , i+1 , size/(1024UL *1024UL )); +printf ("loop%d: did %lu MiB\n" , i+1 , td->mib); first =0 ; } }