Skip to contentSkip to conversation

Recent changes (master)

PATCH Jens Axboe 1 participant 1 message Feb 14, 2024

lore
Share & tools
lore

Jens Axboe

author
#0 lore raw

The following changes since commit 097af663e93f86106f32580bfa59e68fae007035:

Merge branch 'vsock' of https://github.com/MPinna/fio (2024-02-12 11:56:33 -0700)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to a99bd37f690ab246caa9b9d65adfa65a25967190:

examples: add PI example with xnvme ioengine (2024-02-13 14:24:59 -0500)

----------------------------------------------------------------
Ankit Kumar (5):
engines/xnvme: allocate iovecs only if vectored I/O is enabled engines/xnvme: add support for metadata
engines:xnvme: add support for end to end data protection engines/xnvme: add checks for verify, block size and metadata size
examples: add PI example with xnvme ioengine

Vincent Fu (4):
logging: record timestamp for each thread
helper_thread: do not send A_EXIT message when exit is called
logging: expand runstates eligible for logging
docs: explain duplicate logging timestamps

 HOWTO.rst             |  25 ++--
 configure             |   2 +-
 engines/xnvme.c       | 331 +++++++++++++++++++++++++++++++++++++++++++++++---
 examples/xnvme-pi.fio |  53 ++++++++
 fio.1                 |  24 ++--
 helper_thread.c       |   1 -
 stat.c                |  24 +++-
 7 files changed, 414 insertions(+), 46 deletions(-)
 create mode 100644 examples/xnvme-pi.fio

---

Diff of recent changes:

HOWTO.rst +14 -11
index 53b03021..5bc1713c 100644
@@ -2491,11 +2491,11 @@ with the caveat that when used on the command line, they must come after the
         want fio to use placement identifier only at indices 0, 2 and 5 specify
         ``fdp_pli=0,2,5``.
 
-.. option:: md_per_io_size=int : [io_uring_cmd]
+.. option:: md_per_io_size=int : [io_uring_cmd] [xnvme]
 
 	Size in bytes for separate metadata buffer per IO. Default: 0.
 
-.. option:: pi_act=int : [io_uring_cmd]
+.. option:: pi_act=int : [io_uring_cmd] [xnvme]
 
 	Action to take when nvme namespace is formatted with protection
 	information. If this is set to 1 and namespace is formatted with
@@ -2511,7 +2511,7 @@ with the caveat that when used on the command line, they must come after the
 	it will use the default slower generator.
 	(see: https://github.com/intel/isa-l)
 
-.. option:: pi_chk=str[,str][,str] : [io_uring_cmd]
+.. option:: pi_chk=str[,str][,str] : [io_uring_cmd] [xnvme]
 
 	Controls the protection information check. This can take one or more
 	of these values. Default: none.
@@ -2524,12 +2524,12 @@ with the caveat that when used on the command line, they must come after the
 	**APPTAG**
 		Enables protection information checking of application tag field.
 
-.. option:: apptag=int : [io_uring_cmd]
+.. option:: apptag=int : [io_uring_cmd] [xnvme]
 
 	Specifies logical block application tag value, if namespace is
 	formatted to use end to end protection information. Default: 0x1234.
 
-.. option:: apptag_mask=int : [io_uring_cmd]
+.. option:: apptag_mask=int : [io_uring_cmd] [xnvme]
 
 	Specifies logical block application tag mask value, if namespace is
 	formatted to use end to end protection information. Default: 0xffff.
@@ -4066,12 +4066,15 @@ Measurements and reporting
 
 .. option:: log_avg_msec=int
 
-	By default, fio will log an entry in the iops, latency, or bw log for every
-	I/O that completes. When writing to the disk log, that can quickly grow to a
-	very large size. Setting this option makes fio average the each log entry
-	over the specified period of time, reducing the resolution of the log.  See
-	:option:`log_window_value` as well. Defaults to 0, logging all entries.
-	Also see `Log File Formats`_.
+        By default, fio will log an entry in the iops, latency, or bw log for
+        every I/O that completes. When writing to the disk log, that can
+        quickly grow to a very large size. Setting this option directs fio to
+        instead record an average over the specified duration for each log
+        entry, reducing the resolution of the log. When the job completes, fio
+        will flush any accumulated latency log data, so the final log interval
+        may not match the value specified by this option and there may even be
+        duplicate timestamps. See :option:`log_window_value` as well. Defaults
+        to 0, logging entries for each I/O. Also see `Log File Formats`_.
 
 .. option:: log_hist_msec=int
 
configure +1 -1
index becb193e..3eef022b 100755
@@ -2697,7 +2697,7 @@ fi
 ##########################################
 # Check if we have xnvme
 if test "$xnvme" != "no" ; then
-  if check_min_lib_version xnvme 0.7.0; then
+  if check_min_lib_version xnvme 0.7.4; then
     xnvme="yes"
     xnvme_cflags=$(pkg-config --cflags xnvme)
     xnvme_libs=$(pkg-config --libs xnvme)
engines/xnvme.c +314 -17

462 lines not on this page · raw message

examples/xnvme-pi.fio new file +53 -0

53 lines not on this page · raw message

fio.1 +13 -11
index 227fcb47..7ec5c745 100644
@@ -2251,10 +2251,10 @@ By default, the job will cycle through all available Placement IDs, so use this
 to isolate these identifiers to specific jobs. If you want fio to use placement
 identifier only at indices 0, 2 and 5 specify, you would set `fdp_pli=0,2,5`.
 .TP
-.BI (io_uring_cmd)md_per_io_size \fR=\fPint
+.BI (io_uring_cmd,xnvme)md_per_io_size \fR=\fPint
 Size in bytes for separate metadata buffer per IO. Default: 0.
 .TP
-.BI (io_uring_cmd)pi_act \fR=\fPint
+.BI (io_uring_cmd,xnvme)pi_act \fR=\fPint
 Action to take when nvme namespace is formatted with protection information.
 If this is set to 1 and namespace is formatted with metadata size equal to
 protection information size, fio won't use separate metadata buffer or extended
@@ -2268,7 +2268,7 @@ For 16 bit CRC generation fio will use isa-l if available otherwise it will
 use the default slower generator.
 (see: https://github.com/intel/isa-l)
 .TP
-.BI (io_uring_cmd)pi_chk \fR=\fPstr[,str][,str]
+.BI (io_uring_cmd,xnvme)pi_chk \fR=\fPstr[,str][,str]
 Controls the protection information check. This can take one or more of these
 values. Default: none.
 .RS
@@ -2285,11 +2285,11 @@ Enables protection information checking of application tag field.
 .RE
 .RE
 .TP
-.BI (io_uring_cmd)apptag \fR=\fPint
+.BI (io_uring_cmd,xnvme)apptag \fR=\fPint
 Specifies logical block application tag value, if namespace is formatted to use
 end to end protection information. Default: 0x1234.
 .TP
-.BI (io_uring_cmd)apptag_mask \fR=\fPint
+.BI (io_uring_cmd,xnvme)apptag_mask \fR=\fPint
 Specifies logical block application tag mask value, if namespace is formatted
 to use end to end protection information. Default: 0xffff.
 .TP
@@ -3765,12 +3765,14 @@ resulting in more precise time-related I/O statistics.
 Also see \fBlog_avg_msec\fR as well. Defaults to 1024.
 .TP
 .BI log_avg_msec \fR=\fPint
-By default, fio will log an entry in the iops, latency, or bw log for every
-I/O that completes. When writing to the disk log, that can quickly grow to a
-very large size. Setting this option makes fio average the each log entry
-over the specified period of time, reducing the resolution of the log. See
-\fBlog_window_value\fR as well. Defaults to 0, logging all entries.
-Also see \fBLOG FILE FORMATS\fR section.
+By default, fio will log an entry in the iops, latency, or bw log for every I/O
+that completes. When writing to the disk log, that can quickly grow to a very
+large size. Setting this option directs fio to instead record an average over
+the specified duration for each log entry, reducing the resolution of the log.
+When the job completes, fio will flush any accumulated latency log data, so the
+final log interval may not match the value specified by this option and there
+may even be duplicate timestamps. See \fBlog_window_value\fR as well. Defaults
+to 0, logging entries for each I/O. Also see \fBLOG FILE FORMATS\fR section.
 .TP
 .BI log_hist_msec \fR=\fPint
 Same as \fBlog_avg_msec\fR, but logs entries for completion latency
helper_thread.c +0 -1
index 2a9dabf5..332ccb53 100644
@@ -161,7 +161,6 @@ void helper_thread_exit(void)
 		return;
 
 	helper_data->exit = 1;
-	submit_action(A_EXIT);
 	pthread_join(helper_data->thread, NULL);
 }
 
stat.c +19 -5
index 11b58626..b98e8b27 100644
@@ -3576,6 +3576,22 @@ static int add_iops_samples(struct thread_data *td, struct timespec *t)
 				td->ts.iops_stat, td->iops_log, false);
 }
 
+static bool td_in_logging_state(struct thread_data *td)
+{
+	if (in_ramp_time(td))
+		return false;
+
+	switch(td->runstate) {
+	case TD_RUNNING:
+	case TD_VERIFYING:
+	case TD_FINISHING:
+	case TD_EXITED:
+		return true;
+	default:
+		return false;
+	}
+}
+
 /*
  * Returns msecs to next event
  */
@@ -3585,15 +3601,13 @@ int calc_log_samples(void)
 	struct timespec now;
 	long elapsed_time = 0;
 
-	fio_gettime(&now, NULL);
-
 	for_each_td(td) {
-		elapsed_time = mtime_since_now(&td->epoch);
+		fio_gettime(&now, NULL);
+		elapsed_time = mtime_since(&td->epoch, &now);
 
 		if (!td->o.stats)
 			continue;
-		if (in_ramp_time(td) ||
-		    !(td->runstate == TD_RUNNING || td->runstate == TD_VERIFYING)) {
+		if (!td_in_logging_state(td)) {
 			next = min(td->o.iops_avg_time, td->o.bw_avg_time);
 			continue;
 		}