Skip to contentSkip to conversation

Recent changes (master)

PATCH Jens Axboe 1 participant 1 message Sep 15, 2020 Kbuild

lore
Share & tools
lore

Jens Axboe

author
#0 lore raw

The following changes since commit 695611a9d4cd554d44d8b2ec5da2811061950a2e:

Allow offload with FAKEIO engines (2020-09-11 09:58:15 -0600)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to 7064f8942a3b8070acf60b8e5fabc16f8221ae40:

Merge branch 'msys2' of https://github.com/sitsofe/fio into master (2020-09-14 19:43:39 -0600)

----------------------------------------------------------------
Bart Van Assche (2):
backend: Remove two superfluous casts
backend: Use asprintf() instead of strlen() + sprintf()

Jens Axboe (2):
Merge branch 'backend' of https://github.com/bvanassche/fio into master Merge branch 'msys2' of https://github.com/sitsofe/fio into master

Sitsofe Wheeler (12):
configure: pass non-null pointer to (v)asprintf
net: coerce the result of htonl before printing
windows: fix wrong format strings
windows: fix DWORD format string complaints
configure: be explicit about "XP" Windows API version configure/Makefile: add option to generate pdb symbols Makefile/ci: Don't pass CFLAGS when linking
appveyor: cleanup and add separate install script
Makefile: introduce FIO_CFLAGS
memlock: avoid type confusion in format string
configure: cleanup lex/yacc tests
travis: cleanup build script

 .appveyor.yml             | 40 ++++++++++++++++--------
 Makefile                  | 77 ++++++++++++++++++++++++++---------------------
 backend.c                 | 15 ++++-----
 ci/appveyor-install.sh    | 41 +++++++++++++++++++++++++
 ci/travis-build.sh        |  8 +++--
 ci/travis-install.sh      |  5 ++-
 configure                 | 60 ++++++++++++++++++++++++++----------
 engines/net.c             |  5 +--
 engines/windowsaio.c      |  8 ++---
 os/windows/cpu-affinity.c | 26 ++++++++--------
 os/windows/dobuild.cmd    | 10 ++++--
 os/windows/install.wxs    |  8 +++++
 os/windows/posix.c        |  5 +--
 t/memlock.c               |  2 +-
 14 files changed, 211 insertions(+), 99 deletions(-)
 create mode 100755 ci/appveyor-install.sh

---

Diff of recent changes:

.appveyor.yml +28 -12
index 352caeee..fad16326 100644
@@ -5,33 +5,49 @@ image:
 
 environment:
   CYG_MIRROR: http://cygwin.mirror.constant.com
-  CYG_ROOT: C:\cygwin64
-  MAKEFLAGS: -j 2
   matrix:
-    - platform: x64
-      PACKAGE_ARCH: x86_64
+    - ARCHITECTURE: x64
+      CC: clang
+      CONFIGURE_OPTIONS: --enable-pdb
+      DISTRO: msys2
+# Skip 32 bit clang build
+#    - ARCHITECTURE: x86
+#      CC: clang
+#      CONFIGURE_OPTIONS: --enable-pdb
+#      DISTRO: msys2
+    - ARCHITECTURE: x64
       CONFIGURE_OPTIONS:
-    - platform: x86
-      PACKAGE_ARCH: i686
+      DISTRO: cygwin
+    - ARCHITECTURE: x86
       CONFIGURE_OPTIONS: --build-32bit-win --target-win-ver=xp
+      DISTRO: cygwin
 
 install:
-  - '%CYG_ROOT%\setup-x86_64.exe --quiet-mode --no-shortcuts --only-site --site "%CYG_MIRROR%" --packages "mingw64-%PACKAGE_ARCH%-zlib,mingw64-%PACKAGE_ARCH%-CUnit" > NUL'
-  - SET PATH=C:\Python38-x64;%CYG_ROOT%\bin;%PATH% # NB: Changed env variables persist to later sections
+  - if %DISTRO%==cygwin (
+      SET "PATH=C:\cygwin64\bin;C:\cygwin64;%PATH%"
+    )
+  - if %DISTRO%==msys2 if %ARCHITECTURE%==x86 (
+      SET "PATH=C:\msys64\mingw32\bin;C:\msys64\usr\bin;%PATH%"
+    )
+  - if %DISTRO%==msys2 if %ARCHITECTURE%==x64 (
+      SET "PATH=C:\msys64\mingw64\bin;C:\msys64\usr\bin;%PATH%"
+    )
+  - SET PATH=C:\Python38-x64;%PATH% # NB: Changed env variables persist to later sections
   - SET PYTHONUNBUFFERED=TRUE
-  - python.exe -m pip install scipy six
+  - bash.exe ci\appveyor-install.sh
 
 build_script:
-  - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && ./configure --disable-native --extra-cflags=\"-Werror\" ${CONFIGURE_OPTIONS} && make.exe'
+  - bash.exe configure --extra-cflags=-Werror --disable-native %CONFIGURE_OPTIONS%
+  - make.exe -j2
 
 after_build:
   - file.exe fio.exe
   - make.exe test
-  - 'cd os\windows && dobuild.cmd %PLATFORM% && cd ..'
+  - 'cd os\windows && dobuild.cmd %ARCHITECTURE% && cd ..'
   - ps: Get-ChildItem .\os\windows\*.msi | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name -DeploymentName fio.msi }
 
 test_script:
-  - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && [ -f fio.exe ] && python.exe t/run-fio-tests.py --artifact-root test-artifacts --debug'
+  - python.exe t/run-fio-tests.py --artifact-root test-artifacts --debug
 
 on_finish:
   - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && [ -d test-artifacts ] && 7z a -t7z test-artifacts.7z test-artifacts -xr!foo.0.0 -xr!latency.?.0 -xr!fio_jsonplus_clat2csv.test && appveyor PushArtifact test-artifacts.7z'
Makefile +42 -35

196 lines not on this page · raw message

backend.c +8 -7
index 05453ae2..f91f3caf 100644
@@ -1458,16 +1458,17 @@ static bool keep_running(struct thread_data *td)
 	return false;
 }
 
-static int exec_string(struct thread_options *o, const char *string, const char *mode)
+static int exec_string(struct thread_options *o, const char *string,
+		       const char *mode)
 {
-	size_t newlen = strlen(string) + strlen(o->name) + strlen(mode) + 13 + 1;
 	int ret;
 	char *str;
 
-	str = malloc(newlen);
-	sprintf(str, "%s > %s.%s.txt 2>&1", string, o->name, mode);
+	if (asprintf(&str, "%s > %s.%s.txt 2>&1", string, o->name, mode) < 0)
+		return -1;
 
-	log_info("%s : Saving output of %s in %s.%s.txt\n",o->name, mode, o->name, mode);
+	log_info("%s : Saving output of %s in %s.%s.txt\n", o->name, mode,
+		 o->name, mode);
 	ret = system(str);
 	if (ret == -1)
 		log_err("fio: exec of cmd <%s> failed\n", str);
@@ -1731,7 +1732,7 @@ static void *thread_main(void *data)
 	if (!init_random_map(td))
 		goto err;
 
-	if (o->exec_prerun && exec_string(o, o->exec_prerun, (const char *)"prerun"))
+	if (o->exec_prerun && exec_string(o, o->exec_prerun, "prerun"))
 		goto err;
 
 	if (o->pre_read && !pre_read_files(td))
@@ -1890,7 +1891,7 @@ static void *thread_main(void *data)
 	rate_submit_exit(td);
 
 	if (o->exec_postrun)
-		exec_string(o, o->exec_postrun, (const char *)"postrun");
+		exec_string(o, o->exec_postrun, "postrun");
 
 	if (exitall_on_terminate || (o->exitall_error && td->error))
 		fio_terminate_threads(td->groupid, td->o.exit_what);
ci/appveyor-install.sh new file +41 -0

41 lines not on this page · raw message

ci/travis-build.sh +5 -3
index 231417e2..923d882d 100755
@@ -1,8 +1,9 @@
 #!/bin/bash
+set -eu
 
 CI_TARGET_ARCH="${BUILD_ARCH:-$TRAVIS_CPU_ARCH}"
 EXTRA_CFLAGS="-Werror"
-PYTHONUNBUFFERED=TRUE
+export PYTHONUNBUFFERED=TRUE
 CONFIGURE_FLAGS=()
 
 case "$TRAVIS_OS_NAME" in
@@ -11,6 +12,7 @@ case "$TRAVIS_OS_NAME" in
         case "$CI_TARGET_ARCH" in
             "x86")
                 EXTRA_CFLAGS="${EXTRA_CFLAGS} -m32"
+                export LDFLAGS="-m32"
                 ;;
             "amd64")
                 CONFIGURE_FLAGS+=(--enable-cuda)
@@ -24,7 +26,7 @@ CONFIGURE_FLAGS+=(--extra-cflags="${EXTRA_CFLAGS}")
     make &&
     make test &&
     if [[ "$CI_TARGET_ARCH" == "arm64" ]]; then
-	sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug -p 1010:"--skip 15 16 17 18 19 20"
+        sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug -p 1010:"--skip 15 16 17 18 19 20"
     else
-	sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug
+        sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug
     fi
ci/travis-install.sh +2 -3
index b6895e82..103695dc 100755
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -e
+set -eu
 
 CI_TARGET_ARCH="${BUILD_ARCH:-$TRAVIS_CPU_ARCH}"
 case "$TRAVIS_OS_NAME" in
@@ -51,6 +51,5 @@ case "$TRAVIS_OS_NAME" in
 	;;
 esac
 
-echo "Python version: $(/usr/bin/python -V 2>&1)"
-echo "Python3 path: $(which python3 2>&1)"
+echo "Python3 path: $(type -p python3 2>&1)"
 echo "Python3 version: $(python3 -V 2>&1)"
configure +44 -16

131 lines not on this page · raw message

engines/net.c +3 -2
index 91f25774..c6cec584 100644
@@ -938,8 +938,9 @@ static int fio_netio_udp_recv_open(struct thread_data *td, struct fio_file *f)
 
 	if (ntohl(msg.magic) != FIO_LINK_OPEN_CLOSE_MAGIC ||
 	    ntohl(msg.cmd) != FIO_LINK_OPEN) {
-		log_err("fio: bad udp open magic %x/%x\n", ntohl(msg.magic),
-								ntohl(msg.cmd));
+		log_err("fio: bad udp open magic %x/%x\n",
+			(unsigned int) ntohl(msg.magic),
+			(unsigned int) ntohl(msg.cmd));
 		return -1;
 	}
 
engines/windowsaio.c +4 -4
index 5c7e7964..9868e816 100644
@@ -161,15 +161,15 @@ static int windowsaio_invalidate_cache(struct fio_file *f)
 	if (ihFile != INVALID_HANDLE_VALUE) {
 		if (!CloseHandle(ihFile)) {
 			error = GetLastError();
-			log_info("windowsaio: invalidation fd close %s "
-				 "failed: error %d\n", f->file_name, error);
+			log_info("windowsaio: invalidation fd close %s failed: error %lu\n",
+				 f->file_name, error);
 			rc = 1;
 		}
 	} else {
 		error = GetLastError();
 		if (error != ERROR_FILE_NOT_FOUND) {
-			log_info("windowsaio: cache invalidation of %s failed: "
-					"error %d\n", f->file_name, error);
+			log_info("windowsaio: cache invalidation of %s failed: error %lu\n",
+				 f->file_name, error);
 			rc = 1;
 		}
 	}
os/windows/cpu-affinity.c +14 -12
index 69997b24..46fd048d 100644
@@ -14,7 +14,7 @@ int fio_setaffinity(int pid, os_cpu_mask_t cpumask)
 		bSuccess = SetThreadAffinityMask(h, cpumask);
 		if (!bSuccess)
 			log_err("fio_setaffinity failed: failed to set thread affinity (pid %d, mask %.16llx)\n",
-				pid, cpumask);
+				pid, (long long unsigned) cpumask);
 
 		CloseHandle(h);
 	} else {
@@ -83,7 +83,7 @@ unsigned int cpus_online(void)
 static void print_mask(os_cpu_mask_t *cpumask)
 {
 	for (int i = 0; i < FIO_CPU_MASK_ROWS; i++)
-		dprint(FD_PROCESS, "cpumask[%d]=%lu\n", i, cpumask->row[i]);
+		dprint(FD_PROCESS, "cpumask[%d]=%" PRIu64 "\n", i, cpumask->row[i]);
 }
 
 /* Return the index of the least significant set CPU in cpumask or -1 if no
@@ -99,7 +99,7 @@ int first_set_cpu(os_cpu_mask_t *cpumask)
 		int row_first_cpu;
 
 		row_first_cpu = __builtin_ffsll(cpumask->row[row]) - 1;
-		dprint(FD_PROCESS, "row_first_cpu=%d cpumask->row[%d]=%lu\n",
+		dprint(FD_PROCESS, "row_first_cpu=%d cpumask->row[%d]=%" PRIu64 "\n",
 		       row_first_cpu, row, cpumask->row[row]);
 		if (row_first_cpu > -1) {
 			mask_first_cpu = cpus_offset + row_first_cpu;
@@ -136,7 +136,7 @@ static int last_set_cpu(os_cpu_mask_t *cpumask)
 			    row_last_cpu++;
 		}
 
-		dprint(FD_PROCESS, "row_last_cpu=%d cpumask->row[%d]=%lu\n",
+		dprint(FD_PROCESS, "row_last_cpu=%d cpumask->row[%d]=%" PRIu64 "\n",
 		       row_last_cpu, row, cpumask->row[row]);
 		if (row_last_cpu > -1) {
 			mask_last_cpu = cpus_offset + row_last_cpu;
@@ -213,13 +213,17 @@ static int mask_to_group_mask(os_cpu_mask_t *cpumask, int *processor_group, uint
 		needed_shift = FIO_CPU_MASK_STRIDE - bit_offset;
 		needed_mask_shift = FIO_CPU_MASK_STRIDE - needed;
 		needed_mask = (uint64_t)-1 >> needed_mask_shift;
-		dprint(FD_PROCESS, "bit_offset=%d end=%d needed=%d needed_shift=%d needed_mask=%ld needed_mask_shift=%d\n", bit_offset, end, needed, needed_shift, needed_mask, needed_mask_shift);
+		dprint(FD_PROCESS,
+		       "bit_offset=%d end=%d needed=%d needed_shift=%d needed_mask=%" PRIu64 "needed_mask_shift=%d\n",
+		       bit_offset, end, needed, needed_shift, needed_mask,
+		       needed_mask_shift);
 		group_cpumask |= (cpumask->row[row + 1] & needed_mask) << needed_shift;
 	}
 	group_cpumask &= (uint64_t)-1 >> (FIO_CPU_MASK_STRIDE - group_size);
 
 	/* Return group and mask */
-	dprint(FD_PROCESS, "Returning group=%d group_mask=%lu\n", group, group_cpumask);
+	dprint(FD_PROCESS, "Returning group=%d group_mask=%" PRIu64 "\n",
+	       group, group_cpumask);
 	*processor_group = group;
 	*affinity_mask = group_cpumask;
 
@@ -257,10 +261,8 @@ int fio_setaffinity(int pid, os_cpu_mask_t cpumask)
 	if (SetThreadGroupAffinity(handle, &new_group_affinity, NULL) != 0)
 		ret = 0;
 	else {
-		log_err("fio_setaffinity: failed to set thread affinity "
-			 "(pid %d, group %d, mask %" PRIx64 ", "
-			 "GetLastError=%d)\n", pid, group, group_mask,
-			 GetLastError());
+		log_err("fio_setaffinity: failed to set thread affinity (pid %d, group %d, mask %" PRIx64 ", GetLastError=%lu)\n",
+			pid, group, group_mask, GetLastError());
 		goto err;
 	}
 
@@ -319,7 +321,7 @@ int fio_getaffinity(int pid, os_cpu_mask_t *mask)
 		goto err;
 	}
 	if (!GetProcessGroupAffinity(handle, &group_count, current_groups)) {
-		log_err("%s: failed to get single group affinity for pid %d (%d)\n",
+		log_err("%s: failed to get single group affinity for pid %d (%lu)\n",
 			__func__, pid, GetLastError());
 		goto err;
 	}
@@ -329,7 +331,7 @@ int fio_getaffinity(int pid, os_cpu_mask_t *mask)
 		goto err;
 	}
 	if (!GetProcessAffinityMask(handle, &process_mask, &system_mask)) {
-		log_err("%s: GetProcessAffinityMask() failed for pid\n",
+		log_err("%s: GetProcessAffinityMask() failed for pid %d\n",
 			__func__, pid);
 		goto err;
 	}
os/windows/dobuild.cmd +8 -2
index d06a2afa..08df3e87 100644
@@ -34,7 +34,13 @@ if defined SIGN_FIO (
   signtool sign /as /n "%SIGNING_CN%" /tr http://timestamp.digicert.com /td sha256 /fd sha256 ..\..\t\*.exe
 )
 
-"%WIX%bin\candle" -nologo -arch %FIO_ARCH% -dFioVersionNumbers="%FIO_VERSION_NUMBERS%" install.wxs
+if exist ..\..\fio.pdb (
+  set FIO_PDB=true
+) else (
+  set FIO_PDB=false
+)
+
+"%WIX%bin\candle" -nologo -arch %FIO_ARCH% -dFioVersionNumbers="%FIO_VERSION_NUMBERS%" -dFioPDB="%FIO_PDB%" install.wxs
 @if ERRORLEVEL 1 goto end
 "%WIX%bin\candle" -nologo -arch %FIO_ARCH% examples.wxs
 @if ERRORLEVEL 1 goto end
@@ -43,4 +49,4 @@ if defined SIGN_FIO (
 
 if defined SIGN_FIO (
   signtool sign /n "%SIGNING_CN%" /tr http://timestamp.digicert.com /td sha256 /fd sha256 %FIO_VERSION%-%FIO_ARCH%.msi
-)
\ No newline at end of file
+)
os/windows/install.wxs +8 -0
index dcb8c92c..f73ec5e2 100755
@@ -27,6 +27,11 @@
 							<File Source="..\..\fio.exe"/>
 							<Environment Action="set" Part="last" Id="PATH" Name="PATH" Value="[INSTALLDIR]fio\" System="yes"/>
 						</Component>
+						<?if $(var.FioPDB) = true?>
+						<Component>
+							<File Id="fio.pdb" Name="fio.pdb" Source="..\..\fio.pdb"/>
+						</Component>
+						<?endif?>
 						<Component>
 							<File Id="README" Name="README.txt" Source="..\..\README"/>
 						</Component>
@@ -76,6 +81,9 @@
 
 	<Feature Id="AlwaysInstall" Absent="disallow" ConfigurableDirectory="INSTALLDIR" Display="hidden" Level="1" Title="Flexible I/O Tester">
 		<ComponentRef Id="fio.exe"/>
+		<?if $(var.FioPDB) = true?>
+		<ComponentRef Id="fio.pdb"/>
+		<?endif?>
 		<ComponentRef Id="HOWTO"/>
 		<ComponentRef Id="README"/>
 		<ComponentRef Id="REPORTING_BUGS"/>
os/windows/posix.c +3 -2
index 31271de0..9e9f12ef 100644
@@ -168,7 +168,7 @@ int win_to_posix_error(DWORD winerr)
 	case ERROR_FILE_INVALID:
 		return ENXIO;
 	default:
-		log_err("fio: windows error %d not handled\n", winerr);
+		log_err("fio: windows error %lu not handled\n", winerr);
 		return EIO;
 	}
 
@@ -188,7 +188,8 @@ int GetNumLogicalProcessors(void)
 		if (error == ERROR_INSUFFICIENT_BUFFER)
 			processor_info = malloc(len);
 		else {
-			log_err("Error: GetLogicalProcessorInformation failed: %d\n", error);
+			log_err("Error: GetLogicalProcessorInformation failed: %lu\n",
+				error);
 			return -1;
 		}
 
t/memlock.c +1 -1
index 418dc3c4..9f5a3ea8 100644
@@ -22,7 +22,7 @@ static void *worker(void *data)
 		for (index = 0; index + 4096 < size; index += 4096)
 			memset(&buf[index+512], 0x89, 512);
 		if (first) {
-			printf("loop%d: did %lu MiB\n", i+1, size/(1024UL*1024UL));
+			printf("loop%d: did %lu MiB\n", i+1, td->mib);
 			first = 0;
 		}
 	}