Skip to contentSkip to conversation

Recent changes (master)

PATCH Jens Axboe 1 participant 1 message Jan 7, 2020 Kbuild

lore
Share & tools
lore

Jens Axboe

author
#0 lore raw

The following changes since commit dbab52955aeb0b58cc88c8eff1b1c2239241f0bd:

Merge branch 'memalign1' of https://github.com/kusumi/fio (2020-01-05 10:31:40 -0700)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to 2198a6b5a9f40726b40aced24cf2dcdb3b639898:

Merge branch 'master' of https://github.com/bvanassche/fio (2020-01-06 18:38:02 -0700)

----------------------------------------------------------------
Bart Van Assche (12):
Makefile: Avoid duplicating code
Makefile: Use 'tr' if 'fmt' is not available
configure: Improve the ibverbs test
configure: Improve the getopt_long_only() test
Windows: Fix multiple configure tests
Fix the build in case FIO_HAVE_DISK_UTIL is not defined Change off64_t into uint64_t
Windows: Remove more unused OS dependent code
Include "oslib/asprintf.h" where necessary
Windows: Use snprintf() instead of StringCch*()
Windows: Uninline CPU affinity functions
Windows >= 7: Make fio_getaffinity() error reporting more detailed

Jens Axboe (3):
Merge branch 'json1' of https://github.com/kusumi/fio
Merge branch 'master' of https://github.com/vincentkfu/fio
Merge branch 'master' of https://github.com/bvanassche/fio

Tomohiro Kusumi (1):
json: remove two redundant json_print_array() prototypes

Vincent Fu (2):
t/run-fio-tests: automatically skip t/jobs/t0005 on Windows t/steadystate_tests: relax acceptance criterion

 .appveyor.yml             |   2 +-
 Makefile                  |  37 ++--
 configure                 |  43 ++++-
 diskutil.h                |   2 +-
 helpers.c                 |   3 +-
 helpers.h                 |   4 +-
 init.c                    |   1 +
 io_u.c                    |   2 +-
 json.c                    |   3 +-
 os/os-dragonfly.h         |   1 -
 os/os-freebsd.h           |   2 -
 os/os-mac.h               |   2 -
 os/os-netbsd.h            |   2 -
 os/os-openbsd.h           |   2 -
 os/os-windows-7.h         | 360 -------------------------------------
 os/os-windows-xp.h        |  67 -------
 os/os-windows.h           |  17 +-
 os/windows/cpu-affinity.c | 444 ++++++++++++++++++++++++++++++++++++++++++++++
 os/windows/posix.c        |  23 ++-
 os/windows/posix.h        |   1 -
 stat.c                    |   1 +
 t/run-fio-tests.py        |   8 +-
 t/steadystate_tests.py    |  10 +-
 verify.c                  |   1 +
 zbd.c                     |   2 +
 25 files changed, 542 insertions(+), 498 deletions(-)
 create mode 100644 os/windows/cpu-affinity.c

---

Diff of recent changes:

.appveyor.yml +1 -1
index 4fb0a90d..bf0978ad 100644
@@ -25,7 +25,7 @@ after_build:
 
 test_script:
   - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && file.exe fio.exe && make.exe test'
-  - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && python.exe t/run-fio-tests.py --skip 5 --debug'
+  - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && python.exe t/run-fio-tests.py --debug'
 
 artifacts:
   - path: os\windows\*.msi
Makefile +13 -24
index dd26afca..3c5e0f5b 100644
@@ -208,7 +208,8 @@ ifeq ($(CONFIG_TARGET_OS), Darwin)
   LIBS	 += -lpthread -ldl
 endif
 ifneq (,$(findstring CYGWIN,$(CONFIG_TARGET_OS)))
-  SOURCE += os/windows/posix.c
+  SOURCE += os/windows/cpu-affinity.c os/windows/posix.c
+  WINDOWS_OBJS = os/windows/cpu-affinity.o os/windows/posix.o lib/hweight.o
   LIBS	 += -lpthread -lpsapi -lws2_32 -lssp
   CFLAGS += -DPSAPI_VERSION=1 -Ios/windows/posix/include -Wno-format
 endif
@@ -299,9 +300,9 @@ T_OBJS += $(T_TT_OBJS)
 T_OBJS += $(T_IOU_RING_OBJS)
 
 ifneq (,$(findstring CYGWIN,$(CONFIG_TARGET_OS)))
-    T_DEDUPE_OBJS += os/windows/posix.o lib/hweight.o
-    T_SMALLOC_OBJS += os/windows/posix.o lib/hweight.o
-    T_LFSR_TEST_OBJS += os/windows/posix.o lib/hweight.o
+    T_DEDUPE_OBJS += $(WINDOWS_OBJS)
+    T_SMALLOC_OBJS += $(WINDOWS_OBJS)
+    T_LFSR_TEST_OBJS += $(WINDOWS_OBJS)
 endif
 
 T_TEST_PROGS = $(T_SMALLOC_PROGS)
@@ -387,13 +388,14 @@ override CFLAGS += -DFIO_VERSION='"$(FIO_VERSION)"'
 	@$(CC) -MM $(CFLAGS) $(CPPFLAGS) $(SRCDIR)/$*.c > $*.d
 	@mv -f $*.d $*.d.tmp
 	@sed -e 's|.*:|$*.o:|' < $*.d.tmp > $*.d
-ifeq ($(CONFIG_TARGET_OS), NetBSD)
-	@sed -e 's/.*://' -e 's/\\$$//' < $*.d.tmp | tr -cs "[:graph:]" "\n" | \
-		sed -e 's/^ *//' -e '/^$$/ d' -e 's/$$/:/' >> $*.d
-else
-	@sed -e 's/.*://' -e 's/\\$$//' < $*.d.tmp | fmt -w 1 | \
-		sed -e 's/^ *//' -e 's/$$/:/' >> $*.d
-endif
+	@if type -p fmt >/dev/null 2>&1; then				\
+		sed -e 's/.*://' -e 's/\\$$//' < $*.d.tmp | fmt -w 1 |	\
+		sed -e 's/^ *//' -e 's/$$/:/' >> $*.d;			\
+	else								\
+		sed -e 's/.*://' -e 's/\\$$//' < $*.d.tmp |		\
+		tr -cs "[:graph:]" "\n" |				\
+		sed -e 's/^ *//' -e '/^$$/ d' -e 's/$$/:/' >> $*.d;	\
+	fi
 	@rm -f $*.d.tmp
 
 ifdef CONFIG_ARITHMETIC
@@ -426,19 +428,6 @@ parse.o: lex.yy.o y.tab.o
 endif
 
 init.o: init.c FIO-VERSION-FILE
-	@mkdir -p $(dir $@)
-	$(QUIET_CC)$(CC) -o $@ $(CFLAGS) $(CPPFLAGS) -c $<
-	@$(CC) -MM $(CFLAGS) $(CPPFLAGS) $(SRCDIR)/$*.c > $*.d
-	@mv -f $*.d $*.d.tmp
-	@sed -e 's|.*:|$*.o:|' < $*.d.tmp > $*.d
-ifeq ($(CONFIG_TARGET_OS), NetBSD)
-	@sed -e 's/.*://' -e 's/\\$$//' < $*.d.tmp | tr -cs "[:graph:]" "\n" | \
-		sed -e 's/^ *//' -e '/^$$/ d' -e 's/$$/:/' >> $*.d
-else
-	@sed -e 's/.*://' -e 's/\\$$//' < $*.d.tmp | fmt -w 1 | \
-		sed -e 's/^ *//' -e 's/$$/:/' >> $*.d
-endif
-	@rm -f $*.d.tmp
 
 gcompat.o: gcompat.c gcompat.h
 	$(QUIET_CC)$(CC) $(CFLAGS) $(GTK_CFLAGS) $(CPPFLAGS) -c $<
configure +37 -6

141 lines not on this page · raw message

diskutil.h +1 -1
index 15ec681a..f6b09d22 100644
@@ -120,7 +120,7 @@ static inline void print_disk_util(struct disk_util_stat *du,
 				   struct buf_output *out)
 {
 }
-#define show_disk_util(terse, parent, out)
+#define show_disk_util(terse, parent, out) do { } while (0)
 #define disk_util_prune_entries()
 #define init_disk_util(td)
 #define setup_disk_util()
helpers.c +2 -1
index a0ee3704..ab9d706d 100644
@@ -18,7 +18,8 @@ int posix_fallocate(int fd, off_t offset, off_t len)
 #endif
 
 #ifndef CONFIG_SYNC_FILE_RANGE
-int sync_file_range(int fd, off64_t offset, off64_t nbytes, unsigned int flags)
+int sync_file_range(int fd, uint64_t offset, uint64_t nbytes,
+		    unsigned int flags)
 {
 	errno = ENOSYS;
 	return -1;
helpers.h +3 -1
index a0b32858..4ec0f052 100644
@@ -7,8 +7,10 @@
 
 extern int fallocate(int fd, int mode, off_t offset, off_t len);
 extern int posix_fallocate(int fd, off_t offset, off_t len);
-extern int sync_file_range(int fd, off64_t offset, off64_t nbytes,
+#ifndef CONFIG_SYNC_FILE_RANGE
+extern int sync_file_range(int fd, uint64_t offset, uint64_t nbytes,
 					unsigned int flags);
+#endif
 extern int posix_fadvise(int fd, off_t offset, off_t len, int advice);
 
 #endif /* FIO_HELPERS_H_ */
init.c +1 -0
index 60c85761..2f64726c 100644
@@ -32,6 +32,7 @@
 #include "steadystate.h"
 #include "blktrace.h"
 
+#include "oslib/asprintf.h"
 #include "oslib/getopt.h"
 #include "oslib/strcasestr.h"
 
io_u.c +1 -1
index 4a0c725a..03f5c21f 100644
@@ -2182,7 +2182,7 @@ void io_u_fill_buffer(struct thread_data *td, struct io_u *io_u,
 static int do_sync_file_range(const struct thread_data *td,
 			      struct fio_file *f)
 {
-	off64_t offset, nbytes;
+	uint64_t offset, nbytes;
 
 	offset = f->first_write;
 	nbytes = f->last_write - f->first_write;
json.c +1 -2
index 75212c85..e2819a65 100644
@@ -230,7 +230,6 @@ int json_object_add_value_type(struct json_object *obj, const char *name, int ty
 	return 0;
 }
 
-static void json_print_array(struct json_array *array, struct buf_output *);
 int json_array_add_value_type(struct json_array *array, int type, ...)
 {
 	struct json_value *value;
@@ -296,8 +295,8 @@ static void json_print_level(int level, struct buf_output *out)
 }
 
 static void json_print_pair(struct json_pair *pair, struct buf_output *);
-static void json_print_array(struct json_array *array, struct buf_output *);
 static void json_print_value(struct json_value *value, struct buf_output *);
+
 void json_print_object(struct json_object *obj, struct buf_output *out)
 {
 	int i;
os/os-dragonfly.h +0 -1
index 3c460ae2..44bfcd5d 100644
@@ -47,7 +47,6 @@
 /* This is supposed to equal (sizeof(cpumask_t)*8) */
 #define FIO_MAX_CPUS	SMP_MAXCPU
 
-typedef off_t off64_t;
 typedef cpumask_t os_cpu_mask_t;
 
 /*
os/os-freebsd.h +0 -2
index 789da178..b3addf98 100644
@@ -30,8 +30,6 @@
 #define fio_swap32(x)	bswap32(x)
 #define fio_swap64(x)	bswap64(x)
 
-typedef off_t off64_t;
-
 typedef cpuset_t os_cpu_mask_t;
 
 #define fio_cpu_clear(mask, cpu)        (void) CPU_CLR((cpu), (mask))
os/os-mac.h +0 -2
index 0d97f6b9..2852ac67 100644
@@ -33,8 +33,6 @@
  */
 #define FIO_MAX_JOBS		128
 
-typedef off_t off64_t;
-
 #ifndef CONFIG_CLOCKID_T
 typedef unsigned int clockid_t;
 #endif
os/os-netbsd.h +0 -2
index 88fb3ef1..abc1d3cb 100644
@@ -35,8 +35,6 @@
 #define fio_swap32(x)	bswap32(x)
 #define fio_swap64(x)	bswap64(x)
 
-typedef off_t off64_t;
-
 static inline int blockdev_size(struct fio_file *f, unsigned long long *bytes)
 {
 	struct disklabel dl;
os/os-openbsd.h +0 -2
index 43a649d4..085a6f2b 100644
@@ -35,8 +35,6 @@
 #define fio_swap32(x)	bswap32(x)
 #define fio_swap64(x)	bswap64(x)
 
-typedef off_t off64_t;
-
 static inline int blockdev_size(struct fio_file *f, unsigned long long *bytes)
 {
 	struct disklabel dl;
os/os-windows-7.h +0 -360

363 lines not on this page · raw message

os/os-windows-xp.h +0 -67

70 lines not on this page · raw message

os/os-windows.h +12 -5
index 6061d8c7..6d48ffe8 100644
@@ -22,11 +22,6 @@
 
 #include "windows/posix.h"
 
-/* MinGW won't declare rand_r unless _POSIX is defined */
-#if defined(WIN32) && !defined(rand_r)
-int rand_r(unsigned *);
-#endif
-
 #ifndef PTHREAD_STACK_MIN
 #define PTHREAD_STACK_MIN 65535
 #endif
@@ -219,7 +214,19 @@ static inline int fio_mkdir(const char *path, mode_t mode) {
 #ifdef CONFIG_WINDOWS_XP
 #include "os-windows-xp.h"
 #else
+#define FIO_HAVE_CPU_ONLINE_SYSCONF
+unsigned int cpus_online(void);
 #include "os-windows-7.h"
 #endif
 
+int first_set_cpu(os_cpu_mask_t *cpumask);
+int fio_setaffinity(int pid, os_cpu_mask_t cpumask);
+int fio_cpuset_init(os_cpu_mask_t *mask);
+int fio_getaffinity(int pid, os_cpu_mask_t *mask);
+void fio_cpu_clear(os_cpu_mask_t *mask, int cpu);
+void fio_cpu_set(os_cpu_mask_t *mask, int cpu);
+int fio_cpu_isset(os_cpu_mask_t *mask, int cpu);
+int fio_cpu_count(os_cpu_mask_t *mask);
+int fio_cpuset_exit(os_cpu_mask_t *mask);
+
 #endif /* FIO_OS_WINDOWS_H */
os/windows/cpu-affinity.c new file +444 -0

444 lines not on this page · raw message

os/windows/posix.c +10 -13
index fd1d5582..e36453e9 100644
@@ -28,10 +28,6 @@
 extern unsigned long mtime_since_now(struct timespec *);
 extern void fio_gettime(struct timespec *, void *);
 
-/* These aren't defined in the MinGW headers */
-HRESULT WINAPI StringCchCopyA(char *pszDest, size_t cchDest, const char *pszSrc);
-HRESULT WINAPI StringCchPrintfA(char *pszDest, size_t cchDest, const char *pszFormat, ...);
-
 int win_to_posix_error(DWORD winerr)
 {
 	switch (winerr) {
@@ -312,11 +308,11 @@ char *ctime_r(const time_t *t, char *buf)
 	 * We don't know how long `buf` is, but assume it's rounded up from
 	 * the minimum of 25 to 32
 	 */
-	StringCchPrintfA(buf, 31, "%s %s %d %02d:%02d:%02d %04d\n",
-				dayOfWeek[systime.wDayOfWeek % 7],
-				monthOfYear[(systime.wMonth - 1) % 12],
-				systime.wDay, systime.wHour, systime.wMinute,
-				systime.wSecond, systime.wYear);
+	snprintf(buf, 32, "%s %s %d %02d:%02d:%02d %04d\n",
+		 dayOfWeek[systime.wDayOfWeek % 7],
+		 monthOfYear[(systime.wMonth - 1) % 12],
+		 systime.wDay, systime.wHour, systime.wMinute,
+		 systime.wSecond, systime.wYear);
 	return buf;
 }
 
@@ -958,8 +954,8 @@ DIR *opendir(const char *dirname)
 				OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL);
 	if (file != INVALID_HANDLE_VALUE) {
 		CloseHandle(file);
-		dc = (struct dirent_ctx*)malloc(sizeof(struct dirent_ctx));
-		StringCchCopyA(dc->dirname, MAX_PATH, dirname);
+		dc = malloc(sizeof(struct dirent_ctx));
+		snprintf(dc->dirname, sizeof(dc->dirname), "%s", dirname);
 		dc->find_handle = INVALID_HANDLE_VALUE;
 	} else {
 		DWORD error = GetLastError();
@@ -999,7 +995,8 @@ struct dirent *readdir(DIR *dirp)
 	if (dirp->find_handle == INVALID_HANDLE_VALUE) {
 		char search_pattern[MAX_PATH];
 
-		StringCchPrintfA(search_pattern, MAX_PATH-1, "%s\\*", dirp->dirname);
+		snprintf(search_pattern, sizeof(search_pattern), "%s\\*",
+			 dirp->dirname);
 		dirp->find_handle = FindFirstFileA(search_pattern, &find_data);
 		if (dirp->find_handle == INVALID_HANDLE_VALUE)
 			return NULL;
@@ -1008,7 +1005,7 @@ struct dirent *readdir(DIR *dirp)
 			return NULL;
 	}
 
-	StringCchCopyA(de.d_name, MAX_PATH, find_data.cFileName);
+	snprintf(de.d_name, sizeof(de.d_name), find_data.cFileName);
 	de.d_ino = 0;
 
 	return &de;
os/windows/posix.h +0 -1
index 85640a21..02a9075b 100644
@@ -1,7 +1,6 @@
 #ifndef FIO_WINDOWS_POSIX_H
 #define FIO_WINDOWS_POSIX_H
 
-typedef off_t off64_t;
 typedef int clockid_t;
 
 extern int clock_gettime(clockid_t clock_id, struct timespec *tp);
stat.c +1 -0
index e2bc8ddb..2b303494 100644
@@ -15,6 +15,7 @@
 #include "helper_thread.h"
 #include "smalloc.h"
 #include "zbd.h"
+#include "oslib/asprintf.h"
 
 #define LOG_MSEC_SLACK	1
 
t/run-fio-tests.py +7 -1
index a0a1e8fa..3d236e37 100755
@@ -428,11 +428,13 @@ class Requirements(object):
     _root = False
     _zoned_nullb = False
     _not_macos = False
+    _not_windows = False
     _unittests = False
     _cpucount4 = False
 
     def __init__(self, fio_root):
         Requirements._not_macos = platform.system() != "Darwin"
+        Requirements._not_windows = platform.system() != "Windows"
         Requirements._linux = platform.system() == "Linux"
 
         if Requirements._linux:
@@ -470,6 +472,7 @@ class Requirements(object):
                     Requirements.root,
                     Requirements.zoned_nullb,
                     Requirements.not_macos,
+                    Requirements.not_windows,
                     Requirements.unittests,
                     Requirements.cpucount4]
         for req in req_list:
@@ -494,6 +497,9 @@ class Requirements(object):
     def not_macos():
         return Requirements._not_macos, "platform other than macOS required"
 
+    def not_windows():
+        return Requirements._not_windows, "platform other than Windows required"
+
     def unittests():
         return Requirements._unittests, "Unittests support required"
 
@@ -561,7 +567,7 @@ TEST_LIST = [
             'pre_job':          None,
             'pre_success':      None,
             'output_format':    'json',
-            'requirements':     [],
+            'requirements':     [Requirements.not_windows],
         },
         {
             'test_id':          6,
t/steadystate_tests.py +5 -5
index 9122a60f..b55a67ac 100755
@@ -187,7 +187,7 @@ if __name__ == '__main__':
                     # check runtime, confirm criterion calculation, and confirm that criterion was not met
                     expected = job['timeout'] * 1000
                     actual = jsonjob['read']['runtime']
-                    if abs(expected - actual) > 10:
+                    if abs(expected - actual) > 50:
                         line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
                     else:
                         line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
@@ -215,12 +215,12 @@ if __name__ == '__main__':
             else:
                 expected = job['timeout'] * 1000
                 actual = jsonjob['read']['runtime']
-                if abs(expected - actual) < 10:
-                    result = 'PASSED '
-                    passed = passed + 1
-                else:
+                if abs(expected - actual) > 50:
                     result = 'FAILED '
                     failed = failed + 1
+                else:
+                    result = 'PASSED '
+                    passed = passed + 1
                 line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
             print(line)
             if 'steadystate' in jsonjob:
verify.c +1 -0
index 37d2be8d..a2c0d41d 100644
@@ -14,6 +14,7 @@
 #include "lib/rand.h"
 #include "lib/hweight.h"
 #include "lib/pattern.h"
+#include "oslib/asprintf.h"
 
 #include "crc/md5.h"
 #include "crc/crc64.h"
zbd.c +2 -0
index 99310c49..ee8bcb30 100644
@@ -13,10 +13,12 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <linux/blkzoned.h>
+
 #include "file.h"
 #include "fio.h"
 #include "lib/pow2.h"
 #include "log.h"
+#include "oslib/asprintf.h"
 #include "smalloc.h"
 #include "verify.h"
 #include "zbd.h"