Recent changes (master)
The following changes since commit ebc403fe282864eddfd68ab1793f149a1b0eb1cd:
zbd: fixup ->zone_size_log2 if zone size is not power of 2 (2020-04-06 19:41:45 -0600)
are available in the Git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 9d87c646c45227c86c5a15faee2a6717a4bf1b46:
zbd: Fix build errors on Windows and MacOS (2020-04-07 20:20:36 -0600)
----------------------------------------------------------------
Damien Le Moal (3):
fio: Generalize zonemode=zbd
ioengines: Add zoned block device operations
zbd: Fix build errors on Windows and MacOS
Dmitry Fomichev (2):
fio: Introduce libzbc IO engine t/zbd: Add support for libzbc IO engine tests
Dmitry Monakhov (2):
engines: check options before dereference
engine/rdmaio: fix io_u initialization
Jens Axboe (1):
Merge branch 'rdma-fixes' of https://github.com/dmonakhov/fio
Makefile | 9 +- configure | 36 +++- engines/e4defrag.c | 2 +- engines/libzbc.c | 422 ++++++++++++++++++++++++++++++++++++++++++++ engines/rbd.c | 8 + engines/rdma.c | 17 +- engines/skeleton_external.c | 43 +++++ fio.1 | 6 + fio.h | 2 - io_u.h | 2 - ioengines.h | 9 +- options.c | 3 +- oslib/blkzoned.h | 49 +++++ oslib/linux-blkzoned.c | 219 +++++++++++++++++++++++ t/run-fio-tests.py | 2 +- t/zbd/functions | 38 +++- t/zbd/test-zbd-support | 221 +++++++++++++++-------- zbd.c | 404 ++++++++++++++++++++---------------------- zbd.h | 70 +------- zbd_types.h | 57 ++++++ 20 files changed, 1247 insertions(+), 372 deletions(-) create mode 100644 engines/libzbc.c create mode 100644 oslib/blkzoned.h create mode 100644 oslib/linux-blkzoned.c create mode 100644 zbd_types.h
---
Diff of recent changes:
Makefile +6 -3
@@ -50,7 +50,7 @@ SOURCE := $(sort $(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/crc/*.c)) \ gettime-thread.c helpers.c json.c idletime.c td_error.c \ profiles/tiobench.c profiles/act.c io_u_queue.c filelock.c \ workqueue.c rate-submit.c optgroup.c helper_thread.c \ - steadystate.c zone-dist.c + steadystate.c zone-dist.c zbd.c ifdef CONFIG_LIBHDFS HDFSFLAGS= -I $(JAVA_HOME)/include -I $(JAVA_HOME)/include/linux -I $(FIO_LIBHDFS_INCLUDE)
@@ -160,13 +160,16 @@ endif ifdef CONFIG_IME SOURCE += engines/ime.c endif -ifdef CONFIG_LINUX_BLKZONED - SOURCE += zbd.c +ifdef CONFIG_LIBZBC + SOURCE += engines/libzbc.c endif ifeq ($(CONFIG_TARGET_OS), Linux) SOURCE += diskutil.c fifo.c blktrace.c cgroup.c trim.c engines/sg.c \ oslib/linux-dev-lookup.c engines/io_uring.c +ifdef CONFIG_HAS_BLKZONED + SOURCE += oslib/linux-blkzoned.c +endif LIBS += -lpthread -ldl LDFLAGS += -rdynamic endif
configure +35 -1
@@ -2397,6 +2397,37 @@ if compile_prog "" "" "linux_blkzoned"; then fi print_config "Zoned block device support" "$linux_blkzoned" +########################################## +# libzbc probe +if test "$libzbc" != "yes" ; then + libzbc="no" +fi +cat > $TMPC << EOF +#include <libzbc/zbc.h> +int main(int argc, char **argv) +{ + struct zbc_device *dev = NULL; + + return zbc_open("foo=bar", O_RDONLY, &dev); +} +EOF +if compile_prog "" "-lzbc" "libzbc"; then + libzbcvermaj=$(pkg-config --modversion libzbc | sed 's/\.[0-9]*\.[0-9]*//') + if test "$libzbcvermaj" -ge "5" ; then + libzbc="yes" + LIBS="-lzbc $LIBS" + else + print_config "libzbc engine" "Unsupported libzbc version (version 5 or above required)" + libzbc="no" + fi +else + if test "$libzbc" = "yes" ; then + feature_not_found "libzbc" "libzbc or libzbc/zbc.h" + fi + libzbc="no" +fi +print_config "libzbc engine" "$libzbc" + ########################################## # check march=armv8-a+crc+crypto if test "$march_armv8_a_crc_crypto" != "yes" ; then
@@ -2862,7 +2893,10 @@ if test "$valgrind_dev" = "yes"; then output_sym "CONFIG_VALGRIND_DEV" fi if test "$linux_blkzoned" = "yes" ; then - output_sym "CONFIG_LINUX_BLKZONED" + output_sym "CONFIG_HAS_BLKZONED" +fi +if test "$libzbc" = "yes" ; then + output_sym "CONFIG_LIBZBC" fi if test "$zlib" = "no" ; then echo "Consider installing zlib-dev (zlib-devel, some fio features depend on it."
engines/e4defrag.c +1 -1
@@ -72,7 +72,7 @@ static int fio_e4defrag_init(struct thread_data *td)struct stat stub;char donor_name[PATH_MAX ]; -if (!strlen (o->donor_name)) { +if (!o->donor_name || !strlen (o->donor_name)) {log_err ("'donorname' options required\n" );return 1 ; }
engines/libzbc.c new file +422 -0
422 lines not on this page · raw message
engines/rbd.c +8 -0
@@ -200,6 +200,14 @@ static int _fio_rbd_connect(struct thread_data *td)log_err ("rados_create failed.\n" ); goto failed_early; } +if (o->pool_name ==NULL ) { +log_err ("rbd pool name must be provided.\n" ); + goto failed_early; + } +if (!o->rbd_name) { +log_err ("rbdname must be provided.\n" ); + goto failed_early; + } r =rados_conf_read_file (rbd->cluster,NULL );if (r <0 ) {
engines/rdma.c +12 -5
@@ -1050,7 +1050,7 @@ static int fio_rdmaio_setup_connect(struct thread_data *td, const char *host,return err;/* resolve route */ -if (strcmp (o->bindname,"" ) !=0 ) { +if (o->bindname &&strlen (o->bindname)) { addrb.ss_family =AF_INET ; err =aton (td, o->bindname, (struct sockaddr_in *)&addrb);if (err)
@@ -1116,7 +1116,7 @@ static int fio_rdmaio_setup_listen(struct thread_data *td, short port) rd->addr.sin_family =AF_INET ; rd->addr.sin_port =htons (port); -if (strcmp (o->bindname,"" ) ==0 ) +if (!o->bindname || !strlen (o->bindname)) rd->addr.sin_addr.s_addr =htonl (INADDR_ANY );else rd->addr.sin_addr.s_addr =htonl (*o->bindname);
@@ -1249,8 +1249,7 @@ static int fio_rdmaio_init(struct thread_data *td) {struct rdmaio_data *rd = td->io_ops_data;struct rdmaio_options *o = td->eo; -unsigned int max_bs; -int ret, i; +int ret;if (td_rw (td)) {log_err ("fio: rdma connections must be read OR write\n" );
@@ -1318,6 +1317,13 @@ static int fio_rdmaio_init(struct thread_data *td) rd->is_client =1 ; ret =fio_rdmaio_setup_connect (td, td->o.filename, o->port); } +return ret; +} +static int fio_rdmaio_post_init (struct thread_data *td) +{ +unsigned int max_bs; +int i; +struct rdmaio_data *rd = td->io_ops_data; max_bs =max (td->o.max_bs[DDIR_READ ], td->o.max_bs[DDIR_WRITE ]); rd->send_buf.max_bs =htonl (max_bs);
@@ -1351,7 +1357,7 @@ static int fio_rdmaio_init(struct thread_data *td) rd->send_buf.nr =htonl (i); -return ret; +return 0 ; }static void fio_rdmaio_cleanup (struct thread_data *td)
@@ -1388,6 +1394,7 @@ static struct ioengine_ops ioengine_rw = { .version = FIO_IOOPS_VERSION, .setup = fio_rdmaio_setup, .init = fio_rdmaio_init, + .post_init = fio_rdmaio_post_init, .prep = fio_rdmaio_prep, .queue = fio_rdmaio_queue, .commit = fio_rdmaio_commit,
engines/skeleton_external.c +43 -0
55 lines not on this page · raw message
fio.1 +6 -0
@@ -1629,6 +1629,12 @@ I/O. Requires \fBfilename\fR option to specify either block or character devices. This engine supports trim operations. The sg engine includes engine specific options. .TP +.B libzbc +Synchronous I/O engine for SMR hard-disks using the \fBlibzbc\fR +library. The target can be either an sg character device or +a block device file. This engine supports the zonemode=zbd zone +operations. +.TP .B null Doesn't transfer any data, just pretends to. This is mainly used to exercise fio itself and for debugging/testing purposes.
fio.h +0 -2
@@ -172,8 +172,6 @@ struct zone_split_index {uint64_t size_prev; }; -#define FIO_MAX_OPEN_ZBD_ZONES 128 -/* * This describes a single thread/process executing a fio job. */
io_u.h +0 -2
@@ -93,7 +93,6 @@ struct io_u {struct workqueue_work work; }; -#ifdef CONFIG_LINUX_BLKZONED /* * ZBD mode zbd_queue_io callback: called after engine->queue operation * to advance a zone write pointer and eventually unlock the I/O zone.
@@ -108,7 +107,6 @@ struct io_u {* or commit of an async I/O to unlock the I/O target zone. */ void (*zbd_put_io)(const struct io_u *); -#endif /* * Callback for io completion
ioengines.h +8 -1
@@ -6,8 +6,9 @@#include "compiler/compiler.h" #include "flist.h" #include "io_u.h" +#include "zbd_types.h" -#define FIO_IOOPS_VERSION 25 +#define FIO_IOOPS_VERSION 26/* * io_ops->queue() return values
@@ -44,6 +45,12 @@ struct ioengine_ops {void (*iomem_free)(struct thread_data *);int (*io_u_init)(struct thread_data *,struct io_u *);void (*io_u_free)(struct thread_data *,struct io_u *); +int (*get_zoned_model)(struct thread_data *td, +struct fio_file *f,enum zbd_zoned_model *); +int (*report_zones)(struct thread_data *,struct fio_file *, +uint64_t ,struct zbd_zone *,unsigned int ); +int (*reset_wp)(struct thread_data *,struct fio_file *, +uint64_t ,uint64_t );int option_struct_size;struct fio_option *options; };
options.c +2 -1
@@ -13,6 +13,7 @@#include "lib/pattern.h" #include "options.h" #include "optgroup.h" +#include "zbd.h" char client_sockaddr_str[INET6_ADDRSTRLEN ] = {0 };
@@ -3362,7 +3363,7 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .lname = "Maximum number of open zones", .type = FIO_OPT_INT, .off1 = offsetof(struct thread_options, max_open_zones), - .maxval = FIO_MAX_OPEN_ZBD_ZONES, + .maxval = ZBD_MAX_OPEN_ZONES, .help ="Limit random writes to SMR drives to the specified" " number of sequential zones", .def = "0",
oslib/blkzoned.h new file +49 -0
49 lines not on this page · raw message
oslib/linux-blkzoned.c new file +219 -0
219 lines not on this page · raw message
t/run-fio-tests.py +1 -1
@@ -465,7 +465,7 @@ class Requirements(object): print("Unable to open {0} to check requirements".format(config_file)) Requirements._zbd = True else: - Requirements._zbd = "CONFIG_LINUX_BLKZONED" in contents + Requirements._zbd = "CONFIG_HAS_BLKZONED" in contents Requirements._libaio = "CONFIG_LIBAIO" in contents Requirements._root = (os.geteuid() == 0)
t/zbd/functions +35 -3
@@ -4,18 +4,27 @@ blkzone=$(type -p blkzone 2>/dev/null) sg_inq=$(type -p sg_inq 2>/dev/null) zbc_report_zones=$(type -p zbc_report_zones 2>/dev/null) zbc_reset_zone=$(type -p zbc_reset_zone 2>/dev/null) +zbc_info=$(type -p zbc_info 2>/dev/null) if [ -z "${blkzone}" ] && { [ -z "${zbc_report_zones}" ] || [ -z "${zbc_reset_zone}" ]; }; then echo "Error: neither blkzone nor zbc_report_zones is available" exit 1 fi +if [ -n "${use_libzbc}" ] && + { [ -z "${zbc_report_zones}" ] || [ -z "${zbc_reset_zone}" ] || + [ -z "${zbc_info}" ]; }; then + echo "Error: zbc_report_zones, or zbc_reset_zone or zbc_info is not available" + echo "Error: reinstall libzbc tools" + exit 1 +fi + # Reports the starting sector and length of the first sequential zone of device # $1. first_sequential_zone() { local dev=$1 - if [ -n "${blkzone}" ]; then + if [ -n "${blkzone}" ] && [ ! -n "${use_libzbc}" ]; then ${blkzone} report "$dev" | sed -n 's/^[[:blank:]]*start:[[:blank:]]\([0-9a-zA-Z]*\),[[:blank:]]len[[:blank:]]\([0-9a-zA-Z]*\),.*type:[[:blank:]]2(.*/\1 \2/p' | {
@@ -33,7 +42,7 @@ first_sequential_zone() { max_open_zones() { local dev=$1 - if [ -n "${sg_inq}" ]; then + if [ -n "${sg_inq}" ] && [ ! -n "${use_libzbc}" ]; then if ! ${sg_inq} -e --page=0xB6 --len=20 --hex "$dev" 2> /dev/null; then # Non scsi device such as null_blk can not return max open zones. # Use default value.
@@ -56,13 +65,36 @@ max_open_zones() { fi } +is_zbc() { + local dev=$1 + + [[ -z "$(${zbc_info} "$dev" | grep "is not a zoned block device")" ]] +} + +zbc_logical_block_size() { + local dev=$1 + + ${zbc_info} "$dev" | + grep "logical blocks" | + sed -n 's/^[[:blank:]]*[0-9]* logical blocks of[[:blank:]]*//p' | + sed 's/ B//' +} + +zbc_disk_sectors() { + local dev=$1 + + zbc_info "$dev" | + grep "512-bytes sectors" | + sed -e 's/[[:blank:]]*\([0-9]*\)512-bytes sectors.*/\1/' +} + # Reset the write pointer of one zone on device $1 at offset $2. The offset # must be specified in units of 512 byte sectors. Offset -1 means reset all # zones. reset_zone() { local dev=$1 offset=$2 sectors - if [ -n "${blkzone}" ]; then + if [ -n "${blkzone}" ] && [ ! -n "${use_libzbc}" ]; then if [ "$offset" -lt 0 ]; then sectors=$(<"/sys/class/block/${dev#/dev/}/size") ${blkzone} reset -o "${offset}" -l "$sectors" "$dev"
t/zbd/test-zbd-support +143 -78
513 lines not on this page · raw message
zbd.c +192 -212
640 lines not on this page · raw message
zbd.h +8 -62
110 lines not on this page · raw message
zbd_types.h new file +57 -0
57 lines not on this page · raw message