Skip to contentSkip to conversation

Recent changes (master)

PATCH Jens Axboe 1 participant 1 message Sep 27, 2024

lore
Share & tools
lore

Jens Axboe

author
#0 lore raw

The following changes since commit bcd46be2adaa4afc32b836ad6137798544a3d80a:

Merge branch 'atomic-writes' (2024-09-16 20:23:06 -0600)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to e4e8520b374e7e021d289d400ade70c41681d635:

engines/io_uring: Fix negative errno in io_u->error (2024-09-27 04:18:26 -0600)

----------------------------------------------------------------
HuangShumin (1):
Fix configure with --build-static which enables static RDMA in #1801

Jens Axboe (1):
engines/io_uring: don't use abs() on an unsigned value

Minwoo Im (4):
io_uring: Add IO_U_F_DEVICE_ERROR to identify error types
io_uring: Support Compare command for verification
HOWTO: update 'verify_mode' for io_uring_cmd engines/io_uring: Fix negative errno in io_u->error

Vincent Fu (6):
ci: remove unneeded packages
ci: only remove libunwind-14-dev if installed
ci: handle sudo in actions-install.sh
ci: build and run a QEMU guest VM for testing ci/cifuzz: update to upload-artifacts@v4
Merge branch 'master' of https://github.com/HuangShumin/fio

 .github/actions/build-qemu/action.yml         |  31 ++++++++
 .github/actions/create-guest-image/action.yml |  45 +++++++++++
 .github/actions/start-vm/action.yml           |  54 +++++++++++++
 .github/workflows/cifuzz.yml                  |   2 +-
 .github/workflows/qemu.yml                    | 104 ++++++++++++++++++++++++++
 HOWTO.rst                                     |   9 +++
 ci/actions-install.sh                         |  39 +++++-----
 configure                                     |   6 +-
 engines/io_uring.c                            |  63 ++++++++++++++--
 engines/nvme.c                                |   5 +-
 engines/nvme.h                                |   4 +-
 fio.1                                         |  14 ++++
 io_u.c                                        |   9 ++-
 io_u.h                                        |   2 +
 verify.c                                      |   7 ++
 15 files changed, 359 insertions(+), 35 deletions(-)
 create mode 100644 .github/actions/build-qemu/action.yml
 create mode 100644 .github/actions/create-guest-image/action.yml
 create mode 100644 .github/actions/start-vm/action.yml
 create mode 100644 .github/workflows/qemu.yml

---

Diff of recent changes:

.github/actions/build-qemu/action.yml new file +31 -0
new file mode 100644
index 00000000..06804b8b
@@ -0,0 +1,31 @@
+name: 'Build and Install QEMU on Ubuntu'
+desription: 'Build QEMU and Install on Ubuntu'
+
+inputs:
+  version: # What QEMU version to build/install
+    description: 'QEMU version to build and install'
+    required: false
+    default: '9.1.0'
+
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install QEMU build dependencies
+      run: sudo apt-get -qq install libglib2.0-dev libfdt-dev libpixman-1-dev ninja-build flex bison libsdl2-dev libaio-dev python3-tomli libslirp-dev
+      shell: bash
+
+    - name: Build and install QEMU
+      run: |
+        wget -nv https://download.qemu.org/qemu-$INPUT_VER.tar.xz
+        tar xJf qemu-$INPUT_VER.tar.xz
+        rm qemu-$INPUT_VER.tar.xz
+        cd qemu-$INPUT_VER
+        ./configure --enable-kvm --target-list=x86_64-softmmu
+        make -j $(nproc)
+        sudo make install
+        cd ..
+        rm -rf qemu-$INPUT_VER
+      shell: bash
+      env:
+        INPUT_VER: ${{ inputs.version }}
.github/actions/create-guest-image/action.yml new file +45 -0

45 lines not on this page · raw message

.github/actions/start-vm/action.yml new file +54 -0

54 lines not on this page · raw message

.github/workflows/cifuzz.yml +1 -1
index acc8d482..9227735e 100644
@@ -17,7 +17,7 @@ jobs:
         fuzz-seconds: 600
         dry-run: false
     - name: Upload Crash
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v4
       if: failure() && steps.build.outcome == 'success'
       with:
         name: artifacts
.github/workflows/qemu.yml new file +104 -0

104 lines not on this page · raw message

HOWTO.rst +9 -0
index 4f071484..539bce19 100644
@@ -2883,6 +2883,15 @@ with the caveat that when used on the command line, they must come after the
                 **verify**
                         Use Verify commands for write operations
 
+.. option:: verify_mode=str : [io_uring_cmd]
+
+        Specifies the type of command to be used in the verification phase.  Defaults to 'read'.
+
+                **read**
+                        Use Read commands for data verification
+                **compare**
+                        Use Compare commands for data verification
+
 .. option:: sg_write_mode=str : [sg]
 
 	Specify the type of write commands to issue. This option can take ten values:
ci/actions-install.sh +20 -19
index 7a87fbe3..5a32ec91 100755
@@ -6,16 +6,18 @@ SCRIPT_DIR=$(dirname "$0")
 # shellcheck disable=SC1091
 . "${SCRIPT_DIR}/common.sh"
 
+_sudo() {
+    if type -P sudo >/dev/null; then
+        sudo "$@"
+    else
+        "$@"
+    fi
+}
+
 install_ubuntu() {
     local pkgs
 
-    if [ "${GITHUB_JOB}" == "build-containers" ]; then
-        # containers run as root and do not have sudo
-        apt update
-        apt -y install sudo
-    fi
-
-    cat <<DPKGCFG | sudo tee /etc/dpkg/dpkg.cfg.d/dpkg-speedup > /dev/null
+    cat <<DPKGCFG | _sudo tee /etc/dpkg/dpkg.cfg.d/dpkg-speedup > /dev/null
 # Skip fsync
 force-unsafe-io
 # Don't install documentation
@@ -36,7 +38,7 @@ DPKGCFG
     )
     case "${CI_TARGET_ARCH}" in
         "i686")
-            sudo dpkg --add-architecture i386
+            _sudo dpkg --add-architecture i386
             pkgs=("${pkgs[@]/%/:i386}")
             pkgs+=(
                 gcc-multilib
@@ -60,8 +62,10 @@ DPKGCFG
                 librdmacm-dev
 	        pkg-config
             )
-	    echo "Removing libunwind-14-dev because of conflicts with libunwind-dev"
-	    sudo apt remove -y libunwind-14-dev
+	    if apt list --installed | grep -c "libunwind-14-dev"; then
+		    echo "Removing libunwind-14-dev because of conflicts with libunwind-dev"
+		    _sudo apt remove -y libunwind-14-dev
+	    fi
 	    if [ "${CI_TARGET_OS}" == "linux" ] || [ "${CI_TARGET_OS}" == "ubuntu" ]; then
 	        # Only for Ubuntu
 		pkgs+=(
@@ -77,23 +81,22 @@ DPKGCFG
         python3-scipy
 	python3-sphinx
 	python3-statsmodels
+	sudo
+	${EXTRA_PKGS:-}
     )
-    if [ "${GITHUB_JOB}" == "build-containers" ]; then
+    if [ "${GITHUB_JOB}" == "build-containers" ] || [ "${GITHUB_JOB}" == "qemu-guest" ]; then
         pkgs+=(
             bison
             build-essential
-            cmake
             flex
-            unzip
-            wget
             zlib1g-dev
         )
     fi
 
     echo "Updating APT..."
-    sudo apt-get -qq update
+    _sudo apt-get -qq update
     echo "Installing packages... ${pkgs[@]}"
-    sudo apt-get install -o APT::Immediate-Configure=false --no-install-recommends -qq -y "${pkgs[@]}"
+    _sudo apt-get install -o APT::Immediate-Configure=false --no-install-recommends -qq -y "${pkgs[@]}"
 }
 
 # Fedora and related distributions
@@ -101,7 +104,6 @@ install_fedora() {
     pkgs=(
         bison-devel
         git
-        cmake
         flex-devel
         gperftools
         isa-l-devel
@@ -119,9 +121,8 @@ install_fedora() {
         python3-scipy
         python3-sphinx
         sudo
-        unzip
         valgrind-devel
-        wget
+	${EXTRA_PKGS:-}
     )
 
     case "${CI_TARGET_OS}" in
configure +4 -2
index eb92cde3..715f0602 100755
@@ -1005,8 +1005,9 @@ int main(int argc, char **argv)
   return pd != NULL;
 }
 EOF
-if test "$disable_rdma" != "yes" && compile_prog "" "-libverbs" "libverbs" ; then
+if test "$disable_rdma" != "yes" && compile_prog "" "-libverbs -lnl-3 -lnl-route-3" "libverbs" ; then
     libverbs="yes"
+    LIBS="-libverbs -lnl-3 -lnl-route-3 $LIBS"
 fi
 print_config "libverbs" "$libverbs"
 
@@ -1024,8 +1025,9 @@ int main(int argc, char **argv)
   return 0;
 }
 EOF
-if test "$disable_rdma" != "yes" && compile_prog "" "-lrdmacm" "rdma"; then
+if test "$disable_rdma" != "yes" && compile_prog "" "-lrdmacm -lnl-3 -lnl-route-3" "rdma"; then
     rdmacm="yes"
+    LIBS="-libverbs -lnl-3 -lnl-route-3 $LIBS"
 fi
 print_config "rdmacm" "$rdmacm"
 
engines/io_uring.c +56 -7

113 lines not on this page · raw message

engines/nvme.c +3 -2
index 33d87477..18010c0b 100644
@@ -363,7 +363,8 @@ void fio_nvme_uring_cmd_trim_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 
 int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 			    struct iovec *iov, struct nvme_dsm *dsm,
-			    uint8_t write_opcode, unsigned int cdw12_flags)
+			    uint8_t read_opcode, uint8_t write_opcode,
+			    unsigned int cdw12_flags)
 {
 	struct nvme_data *data = FILE_ENG_DATA(io_u->file);
 	__u64 slba;
@@ -373,7 +374,7 @@ int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 
 	switch (io_u->ddir) {
 	case DDIR_READ:
-		cmd->opcode = nvme_cmd_read;
+		cmd->opcode = read_opcode;
 		break;
 	case DDIR_WRITE:
 		cmd->opcode = write_opcode;
engines/nvme.h +3 -1
index b5fef2fb..60b38d7f 100644
@@ -77,6 +77,7 @@ enum nvme_io_opcode {
 	nvme_cmd_write			= 0x01,
 	nvme_cmd_read			= 0x02,
 	nvme_cmd_write_uncor		= 0x04,
+	nvme_cmd_compare		= 0x05,
 	nvme_cmd_write_zeroes		= 0x08,
 	nvme_cmd_dsm			= 0x09,
 	nvme_cmd_verify			= 0x0c,
@@ -431,7 +432,8 @@ int fio_nvme_get_info(struct fio_file *f, __u64 *nlba, __u32 pi_act,
 
 int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 			    struct iovec *iov, struct nvme_dsm *dsm,
-			    uint8_t write_opcode, unsigned int cdw12_flags);
+			    uint8_t read_opcode, uint8_t write_opcode,
+			    unsigned int cdw12_flags);
 
 void fio_nvme_pi_fill(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 		      struct nvme_cmd_ext_io_opts *opts);
fio.1 +14 -0
index 0fd0fb25..356980b5 100644
@@ -2675,6 +2675,20 @@ Use Verify commands for write operations
 .RE
 .RE
 .TP
+.BI (io_uring_cmd)verify_mode \fR=\fPstr
+Specifies the type of command to be used in the verification phase. Defaults to 'read'.
+.RS
+.RS
+.TP
+.B read
+Use Read commands for data verification
+.TP
+.B compare
+Use Compare commands for data verification
+.TP
+.RE
+.RE
+.TP
 .BI (sg)sg_write_mode \fR=\fPstr
 Specify the type of write commands to issue. This option can take multiple
 values:
io_u.c +6 -3
index c49cd4df..b699169d 100644
@@ -1956,7 +1956,8 @@ static void __io_u_log_error(struct thread_data *td, struct io_u *io_u)
 	log_err("fio: io_u error%s%s: %s: %s offset=%llu, buflen=%llu\n",
 		io_u->file ? " on file " : "",
 		io_u->file ? io_u->file->file_name : "",
-		strerror(io_u->error),
+		(io_u->flags & IO_U_F_DEVICE_ERROR) ?
+			"Device-specific error" : strerror(io_u->error),
 		io_ddir_name(io_u->ddir),
 		io_u->offset, io_u->xfer_buflen);
 
@@ -1965,8 +1966,10 @@ static void __io_u_log_error(struct thread_data *td, struct io_u *io_u)
 	if (td->io_ops->errdetails) {
 		char *err = td->io_ops->errdetails(td, io_u);
 
-		log_err("fio: %s\n", err);
-		free(err);
+		if (err) {
+			log_err("fio: %s\n", err);
+			free(err);
+		}
 	}
 
 	if (!td->error)
io_u.h +2 -0
index ab93d50f..22ae6ed4 100644
@@ -22,6 +22,8 @@ enum {
 	IO_U_F_BARRIER		= 1 << 6,
 	IO_U_F_VER_LIST		= 1 << 7,
 	IO_U_F_PATTERN_DONE	= 1 << 8,
+	IO_U_F_DEVICE_ERROR	= 1 << 9,
+	IO_U_F_VER_IN_DEV	= 1 << 10, /* Verify data in device */
 };
 
 /*
verify.c +7 -0
index f3d228ba..2e113862 100644
@@ -901,6 +901,13 @@ int verify_io_u(struct thread_data *td, struct io_u **io_u_ptr)
 	if (td_ioengine_flagged(td, FIO_FAKEIO))
 		return 0;
 
+	/*
+	 * If data has already been verified from the device, we can skip
+	 * the actual verification phase here.
+	 */
+	if (io_u->flags & IO_U_F_VER_IN_DEV)
+		return 0;
+
 	if (io_u->flags & IO_U_F_TRIMMED) {
 		ret = verify_trimmed_io_u(td, io_u);
 		goto done;