Recent changes (master)
The following changes since commit bcd46be2adaa4afc32b836ad6137798544a3d80a:
Merge branch 'atomic-writes' (2024-09-16 20:23:06 -0600)
are available in the Git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to e4e8520b374e7e021d289d400ade70c41681d635:
engines/io_uring: Fix negative errno in io_u->error (2024-09-27 04:18:26 -0600)
----------------------------------------------------------------
HuangShumin (1):
Fix configure with --build-static which enables static RDMA in #1801
Jens Axboe (1):
engines/io_uring: don't use abs() on an unsigned value
Minwoo Im (4):
io_uring: Add IO_U_F_DEVICE_ERROR to identify error types
io_uring: Support Compare command for verification
HOWTO: update 'verify_mode' for io_uring_cmd engines/io_uring: Fix negative errno in io_u->error
Vincent Fu (6):
ci: remove unneeded packages
ci: only remove libunwind-14-dev if installed
ci: handle sudo in actions-install.sh
ci: build and run a QEMU guest VM for testing ci/cifuzz: update to upload-artifacts@v4
Merge branch 'master' of https://github.com/HuangShumin/fio
.github/actions/build-qemu/action.yml | 31 ++++++++ .github/actions/create-guest-image/action.yml | 45 +++++++++++ .github/actions/start-vm/action.yml | 54 +++++++++++++ .github/workflows/cifuzz.yml | 2 +- .github/workflows/qemu.yml | 104 ++++++++++++++++++++++++++ HOWTO.rst | 9 +++ ci/actions-install.sh | 39 +++++----- configure | 6 +- engines/io_uring.c | 63 ++++++++++++++-- engines/nvme.c | 5 +- engines/nvme.h | 4 +- fio.1 | 14 ++++ io_u.c | 9 ++- io_u.h | 2 + verify.c | 7 ++ 15 files changed, 359 insertions(+), 35 deletions(-) create mode 100644 .github/actions/build-qemu/action.yml create mode 100644 .github/actions/create-guest-image/action.yml create mode 100644 .github/actions/start-vm/action.yml create mode 100644 .github/workflows/qemu.yml
---
Diff of recent changes:
.github/actions/build-qemu/action.yml new file +31 -0
@@ -0,0 +1,31 @@ +name: 'Build and Install QEMU on Ubuntu' +desription: 'Build QEMU and Install on Ubuntu' + +inputs: + version: # What QEMU version to build/install + description: 'QEMU version to build and install' + required: false + default: '9.1.0' + + +runs: + using: "composite" + steps: + - name: Install QEMU build dependencies + run: sudo apt-get -qq install libglib2.0-dev libfdt-dev libpixman-1-dev ninja-build flex bison libsdl2-dev libaio-dev python3-tomli libslirp-dev + shell: bash + + - name: Build and install QEMU + run: | + wget -nv https://download.qemu.org/qemu-$INPUT_VER.tar.xz + tar xJf qemu-$INPUT_VER.tar.xz + rm qemu-$INPUT_VER.tar.xz + cd qemu-$INPUT_VER + ./configure --enable-kvm --target-list=x86_64-softmmu + make -j $(nproc) + sudo make install + cd .. + rm -rf qemu-$INPUT_VER + shell: bash + env: + INPUT_VER: ${{ inputs.version }}
.github/actions/create-guest-image/action.yml new file +45 -0
45 lines not on this page · raw message
.github/actions/start-vm/action.yml new file +54 -0
54 lines not on this page · raw message
.github/workflows/cifuzz.yml +1 -1
@@ -17,7 +17,7 @@ jobs: fuzz-seconds: 600 dry-run: false - name: Upload Crash - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 if: failure() && steps.build.outcome == 'success' with: name: artifacts
.github/workflows/qemu.yml new file +104 -0
104 lines not on this page · raw message
HOWTO.rst +9 -0
@@ -2883,6 +2883,15 @@ with the caveat that when used on the command line, they must come after the **verify** Use Verify commands for write operations +.. option:: verify_mode=str : [io_uring_cmd] + + Specifies the type of command to be used in the verification phase. Defaults to 'read'. + + **read** + Use Read commands for data verification + **compare** + Use Compare commands for data verification + .. option:: sg_write_mode=str : [sg] Specify the type of write commands to issue. This option can take ten values:
ci/actions-install.sh +20 -19
@@ -6,16 +6,18 @@ SCRIPT_DIR=$(dirname "$0") # shellcheck disable=SC1091 . "${SCRIPT_DIR}/common.sh" +_sudo() { + if type -P sudo >/dev/null; then + sudo "$@" + else + "$@" + fi +} + install_ubuntu() { local pkgs - if [ "${GITHUB_JOB}" == "build-containers" ]; then - # containers run as root and do not have sudo - apt update - apt -y install sudo - fi - - cat <<DPKGCFG | sudo tee /etc/dpkg/dpkg.cfg.d/dpkg-speedup > /dev/null + cat <<DPKGCFG | _sudo tee /etc/dpkg/dpkg.cfg.d/dpkg-speedup > /dev/null # Skip fsync force-unsafe-io # Don't install documentation
@@ -36,7 +38,7 @@ DPKGCFG ) case "${CI_TARGET_ARCH}" in "i686") - sudo dpkg --add-architecture i386 + _sudo dpkg --add-architecture i386 pkgs=("${pkgs[@]/%/:i386}") pkgs+=( gcc-multilib
@@ -60,8 +62,10 @@ DPKGCFG librdmacm-dev pkg-config ) - echo "Removing libunwind-14-dev because of conflicts with libunwind-dev" - sudo apt remove -y libunwind-14-dev + if apt list --installed | grep -c "libunwind-14-dev"; then + echo "Removing libunwind-14-dev because of conflicts with libunwind-dev" + _sudo apt remove -y libunwind-14-dev + fi if [ "${CI_TARGET_OS}" == "linux" ] || [ "${CI_TARGET_OS}" == "ubuntu" ]; then # Only for Ubuntu pkgs+=(
@@ -77,23 +81,22 @@ DPKGCFG python3-scipy python3-sphinx python3-statsmodels + sudo + ${EXTRA_PKGS:-} ) - if [ "${GITHUB_JOB}" == "build-containers" ]; then + if [ "${GITHUB_JOB}" == "build-containers" ] || [ "${GITHUB_JOB}" == "qemu-guest" ]; then pkgs+=( bison build-essential - cmake flex - unzip - wget zlib1g-dev ) fi echo "Updating APT..." - sudo apt-get -qq update + _sudo apt-get -qq update echo "Installing packages... ${pkgs[@]}" - sudo apt-get install -o APT::Immediate-Configure=false --no-install-recommends -qq -y "${pkgs[@]}" + _sudo apt-get install -o APT::Immediate-Configure=false --no-install-recommends -qq -y "${pkgs[@]}" } # Fedora and related distributions
@@ -101,7 +104,6 @@ install_fedora() { pkgs=( bison-devel git - cmake flex-devel gperftools isa-l-devel
@@ -119,9 +121,8 @@ install_fedora() { python3-scipy python3-sphinx sudo - unzip valgrind-devel - wget + ${EXTRA_PKGS:-} ) case "${CI_TARGET_OS}" in
configure +4 -2
@@ -1005,8 +1005,9 @@ int main(int argc, char **argv) return pd != NULL; } EOF -if test "$disable_rdma" != "yes" && compile_prog "" "-libverbs" "libverbs" ; then +if test "$disable_rdma" != "yes" && compile_prog "" "-libverbs -lnl-3 -lnl-route-3" "libverbs" ; then libverbs="yes" + LIBS="-libverbs -lnl-3 -lnl-route-3 $LIBS" fi print_config "libverbs" "$libverbs"
@@ -1024,8 +1025,9 @@ int main(int argc, char **argv) return 0; } EOF -if test "$disable_rdma" != "yes" && compile_prog "" "-lrdmacm" "rdma"; then +if test "$disable_rdma" != "yes" && compile_prog "" "-lrdmacm -lnl-3 -lnl-route-3" "rdma"; then rdmacm="yes" + LIBS="-libverbs -lnl-3 -lnl-route-3 $LIBS" fi print_config "rdmacm" "$rdmacm"
engines/io_uring.c +56 -7
113 lines not on this page · raw message
engines/nvme.c +3 -2
@@ -363,7 +363,8 @@ void fio_nvme_uring_cmd_trim_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,int fio_nvme_uring_cmd_prep (struct nvme_uring_cmd *cmd,struct io_u *io_u,struct iovec *iov,struct nvme_dsm *dsm, -uint8_t write_opcode,unsigned int cdw12_flags) +uint8_t read_opcode,uint8_t write_opcode, +unsigned int cdw12_flags) {struct nvme_data *data =FILE_ENG_DATA (io_u->file);__u64 slba;
@@ -373,7 +374,7 @@ int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,switch (io_u->ddir) {case DDIR_READ : - cmd->opcode = nvme_cmd_read; + cmd->opcode = read_opcode;break ;case DDIR_WRITE : cmd->opcode = write_opcode;
engines/nvme.h +3 -1
@@ -77,6 +77,7 @@ enum nvme_io_opcode { nvme_cmd_write =0x01 , nvme_cmd_read =0x02 , nvme_cmd_write_uncor =0x04 , + nvme_cmd_compare =0x05 , nvme_cmd_write_zeroes =0x08 , nvme_cmd_dsm =0x09 , nvme_cmd_verify = 0x0c,
@@ -431,7 +432,8 @@ int fio_nvme_get_info(struct fio_file *f, __u64 *nlba, __u32 pi_act,int fio_nvme_uring_cmd_prep (struct nvme_uring_cmd *cmd,struct io_u *io_u,struct iovec *iov,struct nvme_dsm *dsm, -uint8_t write_opcode,unsigned int cdw12_flags); +uint8_t read_opcode,uint8_t write_opcode, +unsigned int cdw12_flags);void fio_nvme_pi_fill (struct nvme_uring_cmd *cmd,struct io_u *io_u,struct nvme_cmd_ext_io_opts *opts);
fio.1 +14 -0
@@ -2675,6 +2675,20 @@ Use Verify commands for write operations .RE .RE .TP +.BI (io_uring_cmd)verify_mode \fR=\fPstr +Specifies the type of command to be used in the verification phase. Defaults to 'read'. +.RS +.RS +.TP +.B read +Use Read commands for data verification +.TP +.B compare +Use Compare commands for data verification +.TP +.RE +.RE +.TP .BI (sg)sg_write_mode \fR=\fPstr Specify the type of write commands to issue. This option can take multiple values:
io_u.c +6 -3
@@ -1956,7 +1956,8 @@ static void __io_u_log_error(struct thread_data *td, struct io_u *io_u)log_err ("fio: io_u error%s%s: %s: %s offset=%llu, buflen=%llu\n" , io_u->file ?" on file " :"" , io_u->file ? io_u->file->file_name :"" , -strerror (io_u->error), + (io_u->flags &IO_U_F_DEVICE_ERROR ) ? +"Device-specific error" :strerror (io_u->error),io_ddir_name (io_u->ddir), io_u->offset, io_u->xfer_buflen);
@@ -1965,8 +1966,10 @@ static void __io_u_log_error(struct thread_data *td, struct io_u *io_u)if (td->io_ops->errdetails) {char *err = td->io_ops->errdetails (td, io_u); -log_err ("fio: %s\n" , err); -free (err); +if (err) { +log_err ("fio: %s\n" , err); +free (err); + } }if (!td->error)
io_u.h +2 -0
@@ -22,6 +22,8 @@ enum {IO_U_F_BARRIER =1 <<6 ,IO_U_F_VER_LIST =1 <<7 ,IO_U_F_PATTERN_DONE =1 <<8 , +IO_U_F_DEVICE_ERROR =1 <<9 , + IO_U_F_VER_IN_DEV = 1 << 10, /* Verify data in device */ };/*
verify.c +7 -0
@@ -901,6 +901,13 @@ int verify_io_u(struct thread_data *td, struct io_u **io_u_ptr)if (td_ioengine_flagged (td,FIO_FAKEIO ))return 0 ; +/* +* If data has already been verified from the device, we can skip +* the actual verification phase here. +*/ +if (io_u->flags &IO_U_F_VER_IN_DEV ) +return 0 ; +if (io_u->flags &IO_U_F_TRIMMED ) { ret =verify_trimmed_io_u (td, io_u); goto done;