fio: add NVMe engine
PATCH v2 1/1 fio: add NVMe engine
Add simple synchronous NVMe engine:
ioengine=nvme
It works via standard Linux NVMe ioctls.
It can be used for testing/stress testing upcoming ZNS stuff.
Currently Linux doesn't recognize NVMe ZNS devices as zoned block devices so zone ioctls (BLKRESETZONE et al) can't be used.
Passthrough ioctls allow Zone Append and whatever commands new specs bring.
Support read, write, fsync, fdatasync.
Don't support sync_file_range obviously.
Don't support trim for now, until I figure all qemu options and the story behind broken qemu trim support.
---
v2) man page, delete debugging
Makefile | 3 configure | 20 +++++ engines/nvme.c | 217 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ file.h | 1 fio.1 | 3 5 files changed, 244 insertions(+)
Makefile +3 -0
@@ -163,6 +163,9 @@ endif ifdef CONFIG_LINUX_BLKZONED SOURCE += zbd.c endif +ifdef CONFIG_NVME + SOURCE += engines/nvme.c +endif ifeq ($(CONFIG_TARGET_OS), Linux) SOURCE += diskutil.c fifo.c blktrace.c cgroup.c trim.c engines/sg.c \
configure +20 -0
@@ -2397,6 +2397,22 @@ if compile_prog "" "" "linux_blkzoned"; then fi print_config "Zoned block device support" "$linux_blkzoned" +########################################## +if test "$linux_nvme" != "yes" ; then + linux_nvme="no" +fi +cat >$TMPC <<EOF +#include <linux/nvme_ioctl.h> +int main(void) +{ + return 0; +} +EOF +if compile_prog "" "" "linux_nvme"; then + linux_nvme="yes" +fi +print_config "NVMe engine" "$linux_nvme" + ########################################## # check march=armv8-a+crc+crypto if test "$march_armv8_a_crc_crypto" != "yes" ; then
@@ -2912,6 +2928,10 @@ if test "$libnbd" = "yes" ; then echo "LIBNBD_CFLAGS=$libnbd_cflags" >> $config_host_mak echo "LIBNBD_LIBS=$libnbd_libs" >> $config_host_mak fi +if test "$linux_nvme" = "yes" ; then + output_sym "CONFIG_NVME" +fi + cat > $TMPC << EOF int main(int argc, char **argv) {
new file mode 100644
engines/nvme.c new file +217 -0
217 lines not on this page · raw message
file.h +1 -0
@@ -99,6 +99,7 @@ struct fio_file {uint64_t real_file_size;uint64_t file_offset;uint64_t io_size; +unsigned int lba_shift;/* * Zoned block device information. See also zonemode=zbd.
fio.1 +3 -0
@@ -1789,6 +1789,9 @@ Read and write iscsi lun with libiscsi. .TP .B nbd Synchronous read and write a Network Block Device (NBD). +.TP +.B nvme +Synchronous NVMe I/O via Linux NVME_IOCTL_* ioctls. .SS "I/O engine specific parameters" In addition, there are some parameters which are only valid when a specific \fBioengine\fR is in use. These are used identically to normal parameters,
Quoted 34 lines · Alexey Dobriyan
engines/nvme.c new file +34
-0+static inline uint32_t get_nsid (const struct fio_file *f) +{ +return (uintptr_t )f->engine_data; +} + +static int nvme_open_file (struct thread_data *td,struct fio_file *f) +{ +struct nvme_admin_cmd cmd; +struct nvme_id_ns id; +struct stat st; +uint32_t nsid; + +/* NVMe ioctls ignore open flags, require CAP_SYS_ADMIN only. */ + f->fd =open (f->file_name,O_RDONLY ); +if (f->fd <0 ) { +return -errno; + } +if (fstat (f->fd, &st) ==-1 ) { +return -errno; + } +if (!S_ISBLK (st.st_mode)) { +log_err ("%s: nvme engine requires NVMe block device\n" , + f->file_name); +return 1 ; + } + + nsid =ioctl (f->fd,NVME_IOCTL_ID ); +if (nsid <1 ) { +log_err ("%s: ioctl NVME_IOCTL_ID\n" , f->file_name); +return 1 ; + } + + f->engine_data = (void *)(uintptr_t )nsid; +
...
Quoted 4 lines
engines/nvme.c new file +4
-0+ + f->lba_shift = id.lbaf[id.flbas &15 ].ds; +return 0 ; +}
...
file.h struct fio_file { · line 102
Quoted 7 lines
file.h +1
-0@@ -99,6 +99,7 @@ struct fio_file {uint64_t real_file_size;uint64_t file_offset;uint64_t io_size; +unsigned int lba_shift;
We have the engine_data to stash engine specific data. We shouldn't add the 'lba_shift' to the fio_file just for the nvme engine. Rather than saving just the 'nsid' in 'engine_data', you should save a struct that has everything the engine needs.
Just FYI, this is how small the nvme engine becomes with libnvme:
#include <libnvme.h>
#include "../fio.h"
static int nvme_open_file(struct thread_data *td, struct fio_file *f) {
nvme_ns_t n = nvme_ns_open(f->file_name);
if (!n) {
log_err("%s: failed to open, %s\n", f->file_name,
strerror(errno));
return 1;
}
f->fd = nvme_ns_get_fd(n);
f->engine_data = n;
return 0;
}static enum fio_q_status nvme_queue(struct thread_data *td, struct io_u *io_u)
{
struct fio_file *f = io_u->file;
nvme_ns_t n = f->engine_data;
fio_ro_check(td, io_u);
switch (io_u->ddir) {
case DDIR_READ:
io_u->error = nvme_ns_read(n, io_u->xfer_buf, io_u->offset,
io_u->xfer_buflen);
break;
case DDIR_WRITE:
io_u->error = nvme_ns_write(n, io_u->xfer_buf, io_u->offset,
io_u->xfer_buflen);
break;
case DDIR_SYNC:
case DDIR_DATASYNC:
io_u->error = nvme_ns_flush(n);
break;
default:
io_u->error = EINVAL;
break;
}
if (io_u->error == 0) {
io_u->resid = 0;
io_u->error = 0;
} else {
io_u_log_error(td, io_u);
td_verror(td, io_u->error, "xfer");
}
return FIO_Q_COMPLETED;
}static struct ioengine_ops ioengine = {
.name = "nvme",
.version = FIO_IOOPS_VERSION,
.flags = FIO_SYNCIO|FIO_RAWIO|FIO_NOEXTEND,
.queue = nvme_queue,
.open_file = nvme_open_file,
.close_file = generic_close_file,
.get_file_size = generic_get_file_size,};
fio_init
static void register_nvme_ioengine(void)
{
register_ioengine(&ioengine);
}fio_exit
static void unregister_nvme_ioengine(void)
{
unregister_ioengine(&ioengine);
}