Recent changes (master)

PATCH Jens Axboe 1 participant 1 message Oct 1, 2015

Subsystems (2)

Kbuild Library code

lore

Share & tools

lore

Jens Axboe

author

2015-10-01 08:33 UTC #0 lore raw

The following changes since commit 194fffd042d374d5e13af61a14fe16734c396d8c:

Fix integer overflow in rate_iops (2015-09-24 20:35:44 -0600)

are available in the git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to 2099539d0f49098148b101d2e3d3f533777c700e:

server: bump protocol version (2015-10-01 08:55:18 +0200)

----------------------------------------------------------------
Jens Axboe (2):
sg: fix short reads
server: bump protocol version

Kris Davis (2):
sg: 16-byte cdb support and lots of fixes
Use line buffering on stdout

Roman Pen (2):
fio.1,HOWTO: keep 'iodepth_batch' option in sync Introduce new option: iodepth_batch_complete_max

 HOWTO            |  29 +++-
 Makefile         |   3 +
 backend.c        |  14 +-
 cconv.c          |   6 +-
 configure        |  22 +++
 engines/libaio.c |   2 +-
 engines/sg.c     | 493 +++++++++++++++++++++++++++++++++++++++++++++++++------
 fio.1            |  41 ++++-
 fio.c            |   6 +
 init.c           |   7 +
 io_u.c           |  11 +-
 ioengine.h       |   3 +-
 lib/strlcat.c    |  23 +++
 lib/strlcat.h    |   6 +
 options.c        |  22 ++-
 os/os.h          |   4 +
 server.h         |   2 +-
 thread_options.h |   7 +-
 18 files changed, 629 insertions(+), 72 deletions(-)
 create mode 100644 lib/strlcat.c
 create mode 100644 lib/strlcat.h

---

Diff of recent changes:

HOWTO +28 -1

index d3ead19..40233bd 100644
@@ -812,8 +812,10 @@ iodepth_batch_submit=int
 iodepth_batch=int This defines how many pieces of IO to submit at once.
 		It defaults to 1 which means that we submit each IO
 		as soon as it is available, but can be raised to submit
-		bigger batches of IO at the time.
+		bigger batches of IO at the time. If it is set to 0 the iodepth
+		value will be used.
 
+iodepth_batch_complete_min=int
 iodepth_batch_complete=int This defines how many pieces of IO to retrieve
 		at once. It defaults to 1 which means that we'll ask
 		for a minimum of 1 IO in the retrieval process from

@@ -823,6 +825,31 @@ iodepth_batch_complete=int This defines how many pieces of IO to retrieve
 		events before queuing more IO. This helps reduce
 		IO latency, at the cost of more retrieval system calls.
 
+iodepth_batch_complete_max=int This defines maximum pieces of IO to
+		retrieve at once. This variable should be used along with
+		iodepth_batch_complete_min=int variable, specifying the range
+		of min and max amount of IO which should be retrieved. By default
+		it is equal to iodepth_batch_complete_min value.
+
+		Example #1:
+
+		iodepth_batch_complete_min=1
+		iodepth_batch_complete_max=<iodepth>
+
+		which means that we will retrieve at leat 1 IO and up to the
+		whole submitted queue depth. If none of IO has been completed
+		yet, we will wait.
+
+		Example #2:
+
+		iodepth_batch_complete_min=0
+		iodepth_batch_complete_max=<iodepth>
+
+		which means that we can retrieve up to the whole submitted
+		queue depth, but if none of IO has been completed yet, we will
+		NOT wait and immediately exit the system call. In this example
+		we simply do polling.
+
 iodepth_low=int	The low water mark indicating when to start filling
 		the queue again. Defaults to the same as iodepth, meaning
 		that fio will attempt to keep the queue full at all times.

Makefile +3 -0

index 296655b..d287126 100644
@@ -104,6 +104,9 @@ endif
 ifndef CONFIG_STRCASESTR
   SOURCE += lib/strcasestr.c
 endif
+ifndef CONFIG_STRLCAT
+  SOURCE += lib/strlcat.c
+endif
 ifndef CONFIG_GETOPT_LONG_ONLY
   SOURCE += lib/getopt_long.c
 endif

backend.c +10 -4

index dec0d55..b1477df 100644
@@ -446,8 +446,8 @@ static int wait_for_completions(struct thread_data *td, struct timeval *time)
 	/*
 	 * if the queue is full, we MUST reap at least 1 event
 	 */
-	min_evts = min(td->o.iodepth_batch_complete, td->cur_depth);
-    if ((full && !min_evts) || !td->o.iodepth_batch_complete)
+	min_evts = min(td->o.iodepth_batch_complete_min, td->cur_depth);
+    if ((full && !min_evts) || !td->o.iodepth_batch_complete_min)
 		min_evts = 1;
 
 	if (time && (__should_check_rate(td, DDIR_READ) ||

@@ -551,6 +551,12 @@ sync_done:
 	return 0;
 }
 
+static inline int io_in_polling(struct thread_data *td)
+{
+	return !td->o.iodepth_batch_complete_min &&
+		   !td->o.iodepth_batch_complete_max;
+}
+
 /*
  * The main verify engine. Runs over the writes we previously submitted,
  * reads the blocks back in, and checks the crc/md5 of the data.

@@ -684,7 +690,7 @@ static void do_verify(struct thread_data *td, uint64_t verify_bytes)
 		 */
 reap:
 		full = queue_full(td) || (ret == FIO_Q_BUSY && td->cur_depth);
-		if (full || !td->o.iodepth_batch_complete)
+		if (full || io_in_polling(td))
 			ret = wait_for_completions(td, NULL);
 
 		if (ret < 0)

@@ -932,7 +938,7 @@ static uint64_t do_io(struct thread_data *td)
 reap:
 			full = queue_full(td) ||
 				(ret == FIO_Q_BUSY && td->cur_depth);
-			if (full || !td->o.iodepth_batch_complete)
+			if (full || io_in_polling(td))
 				ret = wait_for_completions(td, &comp_time);
 		}
 		if (ret < 0)

cconv.c +4 -2

index 44f17da..fde8c6d 100644
@@ -83,7 +83,8 @@ void convert_thread_options_to_cpu(struct thread_options *o,
 	o->iodepth = le32_to_cpu(top->iodepth);
 	o->iodepth_low = le32_to_cpu(top->iodepth_low);
 	o->iodepth_batch = le32_to_cpu(top->iodepth_batch);
-	o->iodepth_batch_complete = le32_to_cpu(top->iodepth_batch_complete);
+	o->iodepth_batch_complete_min = le32_to_cpu(top->iodepth_batch_complete_min);
+	o->iodepth_batch_complete_max = le32_to_cpu(top->iodepth_batch_complete_max);
 	o->size = le64_to_cpu(top->size);
 	o->io_limit = le64_to_cpu(top->io_limit);
 	o->size_percent = le32_to_cpu(top->size_percent);

@@ -300,7 +301,8 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
 	top->iodepth = cpu_to_le32(o->iodepth);
 	top->iodepth_low = cpu_to_le32(o->iodepth_low);
 	top->iodepth_batch = cpu_to_le32(o->iodepth_batch);
-	top->iodepth_batch_complete = cpu_to_le32(o->iodepth_batch_complete);
+	top->iodepth_batch_complete_min = cpu_to_le32(o->iodepth_batch_complete_min);
+	top->iodepth_batch_complete_max = cpu_to_le32(o->iodepth_batch_complete_max);
 	top->size_percent = cpu_to_le32(o->size_percent);
 	top->fill_device = cpu_to_le32(o->fill_device);
 	top->file_append = cpu_to_le32(o->file_append);

configure +22 -0

index ee096b3..7489775 100755
@@ -965,6 +965,25 @@ fi
 echo "strcasestr                    $strcasestr"
 
 ##########################################
+# strlcat() probe
+strlcat="no"
+cat > $TMPC << EOF
+#include <string.h>
+int main(int argc, char **argv)
+{
+  static char dst[64];
+  static char *string = "This is a string";
+  memset(dst, 0, sizeof(dst));
+  strlcat(dst, string, sizeof(dst));
+  return 0;
+}
+EOF
+if compile_prog "" "" "strlcat"; then
+  strlcat="yes"
+fi
+echo "strlcat                       $strlcat"
+
+##########################################
 # getopt_long_only() probe
 getopt_long_only="no"
 cat > $TMPC << EOF

@@ -1612,6 +1631,9 @@ fi
 if test "$strcasestr" = "yes" ; then
   output_sym "CONFIG_STRCASESTR"
 fi
+if test "$strlcat" = "yes" ; then
+  output_sym "CONFIG_STRLCAT"
+fi
 if test "$getopt_long_only" = "yes" ; then
   output_sym "CONFIG_GETOPT_LONG_ONLY"
 fi

engines/libaio.c +1 -1

index 9685c99..60dc49d 100644
@@ -146,7 +146,7 @@ static int fio_libaio_getevents(struct thread_data *td, unsigned int min,
 {
 	struct libaio_data *ld = td->io_ops->data;
 	struct libaio_options *o = td->eo;
-	unsigned actual_min = td->o.iodepth_batch_complete == 0 ? 0 : min;
+	unsigned actual_min = td->o.iodepth_batch_complete_min == 0 ? 0 : min;
 	struct timespec __lt, *lt = NULL;
 	int r, events = 0;

engines/sg.c +441 ~~-52~~

653 lines not on this page · raw message

fio.1 +38 -3

54 lines not on this page · raw message

fio.c +6 -0

index ee6eae2..bafbd48 100644
@@ -42,6 +42,12 @@ int main(int argc, char *argv[], char *envp[])
 	if (parse_options(argc, argv))
 		goto done;
 
+	/*
+	 * line buffer stdout to avoid output lines from multiple
+	 * threads getting mixed
+	 */
+	setvbuf(stdout, NULL, _IOLBF, 0);
+
 	fio_time_init();
 
 	if (nr_clients) {

init.c +7 -0

index 684cd60..3f72b36 100644
@@ -630,6 +630,13 @@ static int fixup_options(struct thread_data *td)
 	if (o->iodepth_batch > o->iodepth || !o->iodepth_batch)
 		o->iodepth_batch = o->iodepth;
 
+	/*
+	 * If max batch complete number isn't set or set incorrectly,
+	 * default to the same as iodepth_batch_complete_min
+	 */
+	if (o->iodepth_batch_complete_min > o->iodepth_batch_complete_max)
+		o->iodepth_batch_complete_max = o->iodepth_batch_complete_min;
+
 	if (o->nr_files > td->files_index)
 		o->nr_files = td->files_index;

io_u.c +10 -1

index 9f10206..6dda579 100644
@@ -1578,6 +1578,13 @@ static void __io_u_log_error(struct thread_data *td, struct io_u *io_u)
 		io_ddir_name(io_u->ddir),
 		io_u->offset, io_u->xfer_buflen);
 
+	if (td->io_ops->errdetails) {
+		char *err = td->io_ops->errdetails(io_u);
+
+		log_err("fio: %s\n", err);
+		free(err);
+	}
+
 	if (!td->error)
 		td_verror(td, io_u->error, "io_u error");
 }

@@ -1829,7 +1836,9 @@ int io_u_queued_complete(struct thread_data *td, int min_evts)
 	else if (min_evts > td->cur_depth)
 		min_evts = td->cur_depth;
 
-	ret = td_io_getevents(td, min_evts, td->o.iodepth_batch_complete, tvp);
+	/* No worries, td_io_getevents fixes min and max if they are
+	 * set incorrectly */
+	ret = td_io_getevents(td, min_evts, td->o.iodepth_batch_complete_max, tvp);
 	if (ret < 0) {
 		td_verror(td, -ret, "td_io_getevents");
 		return ret;

ioengine.h +2 -1

index 3d49993..f7f3ec3 100644
@@ -15,7 +15,7 @@
 #include <guasi.h>
 #endif
 
-#define FIO_IOOPS_VERSION	21
+#define FIO_IOOPS_VERSION	22
 
 enum {
 	IO_U_F_FREE		= 1 << 0,

@@ -144,6 +144,7 @@ struct ioengine_ops {
 	int (*commit)(struct thread_data *);
 	int (*getevents)(struct thread_data *, unsigned int, unsigned int, const struct timespec *);
 	struct io_u *(*event)(struct thread_data *, int);
+	char *(*errdetails)(struct io_u *);
 	int (*cancel)(struct thread_data *, struct io_u *);
 	void (*cleanup)(struct thread_data *);
 	int (*open_file)(struct thread_data *, struct fio_file *);

lib/strlcat.c new file +23 -0

new file mode 100644
index 0000000..643d496
@@ -0,0 +1,23 @@
+#include <string.h>
+
+size_t strlcat(char *dst, const char *src, size_t size)
+{
+	size_t dstlen;
+	size_t srclen;
+
+	dstlen = strlen(dst);
+	size -= dstlen + 1;
+
+	/* return if no room */
+	if (!size)
+		return dstlen;
+
+	srclen = strlen(src);
+	if (srclen > size)
+		srclen = size;
+
+	memcpy(dst + dstlen, src, srclen);
+	dst[dstlen + srclen] = '\0';
+
+	return dstlen + srclen;
+}

lib/strlcat.h new file +6 -0

new file mode 100644
index 0000000..34b668e
@@ -0,0 +1,6 @@
+#ifndef FIO_STRLCAT_H
+#define FIO_STRLCAT_h
+
+size_t strlcat(char *dst, const char *src, size_t size);
+
+#endif

options.c +18 -4

index 1868dfd..0169ca2 100644
@@ -1504,11 +1504,12 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.group	= FIO_OPT_G_IO_BASIC,
 	},
 	{
-		.name	= "iodepth_batch_complete",
-		.lname	= "IO Depth batch complete",
+		.name	= "iodepth_batch_complete_min",
+		.lname	= "Min IO depth batch complete",
+		.alias	= "iodepth_batch_complete",
 		.type	= FIO_OPT_INT,
-		.off1	= td_var_offset(iodepth_batch_complete),
-		.help	= "Number of IO buffers to retrieve in one go",
+		.off1	= td_var_offset(iodepth_batch_complete_min),
+		.help	= "Min number of IO buffers to retrieve in one go",
 		.parent	= "iodepth",
 		.hide	= 1,
 		.minval	= 0,

@@ -1518,6 +1519,19 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.group	= FIO_OPT_G_IO_BASIC,
 	},
 	{
+		.name	= "iodepth_batch_complete_max",
+		.lname	= "Max IO depth batch complete",
+		.type	= FIO_OPT_INT,
+		.off1	= td_var_offset(iodepth_batch_complete_max),
+		.help	= "Max number of IO buffers to retrieve in one go",
+		.parent	= "iodepth",
+		.hide	= 1,
+		.minval	= 0,
+		.interval = 1,
+		.category = FIO_OPT_C_IO,
+		.group	= FIO_OPT_G_IO_BASIC,
+	},
+	{
 		.name	= "iodepth_low",
 		.lname	= "IO Depth batch low",
 		.type	= FIO_OPT_INT,

os/os.h +4 -0

index f809a36..8e0b8e8 100644
@@ -68,6 +68,10 @@ typedef struct aiocb os_aiocb_t;
 #include "../lib/strsep.h"
 #endif
 
+#ifndef CONFIG_STRLCAT
+#include "../lib/strlcat.h"
+#endif
+
 #ifdef MSG_DONTWAIT
 #define OS_MSG_DONTWAIT	MSG_DONTWAIT
 #endif

server.h +1 -1

index e4da882..18a689c 100644
@@ -38,7 +38,7 @@ struct fio_net_cmd_reply {
 };
 
 enum {
-	FIO_SERVER_VER			= 46,
+	FIO_SERVER_VER			= 47,
 
 	FIO_SERVER_MAX_FRAGMENT_PDU	= 1024,
 	FIO_SERVER_MAX_CMD_MB		= 2048,

thread_options.h +5 -2

index 38936e9..5ef560e 100644
@@ -54,7 +54,8 @@ struct thread_options {
 	unsigned int iodepth;
 	unsigned int iodepth_low;
 	unsigned int iodepth_batch;
-	unsigned int iodepth_batch_complete;
+	unsigned int iodepth_batch_complete_min;
+	unsigned int iodepth_batch_complete_max;
 
 	unsigned long long size;
 	unsigned long long io_limit;

@@ -299,7 +300,9 @@ struct thread_options_pack {
 	uint32_t iodepth;
 	uint32_t iodepth_low;
 	uint32_t iodepth_batch;
-	uint32_t iodepth_batch_complete;
+	uint32_t iodepth_batch_complete_min;
+	uint32_t iodepth_batch_complete_max;
+	uint32_t __proper_alignment_for_64b;
 
 	uint64_t size;
 	uint64_t io_limit;