Historical Service Time Path Selector
Subsystems (2)
v3 cover Historical Service Time Path Selector
Hi,
This is the third version of HST considering the review by Mike Snitzer. This version drops the encapsulation patches and passes the parameter directly to end_io. It also uses jiffies_to_nsecs directly in dm_start_time_ns_from_clone, instead of converting to msecs in an intermediary step.
This didn't show any regressions compared to the benchmark I presented on previous version:
v2: https://www.redhat.com/archives/dm-devel/2020-April/msg00270.html
v1: https://www.redhat.com/archives/dm-devel/2020-April/msg00176.html
This was tested primarily on a Google cloud SAN with real data and usage patterns and with artificial benchmarks using fio.
Gabriel Krisman Bertazi (1):
md: mpath: Pass IO start time to path selector
Khazhismel Kumykov (1):
md: mpath: Add Historical Service Time Path Selector
drivers/md/Kconfig | 11 + drivers/md/Makefile | 1 + drivers/md/dm-historical-service-time.c | 561 ++++++++++++++++++++++++ drivers/md/dm-mpath.c | 9 +- drivers/md/dm-path-selector.h | 2 +- drivers/md/dm-queue-length.c | 2 +- drivers/md/dm-service-time.c | 2 +- drivers/md/dm.c | 10 + include/linux/device-mapper.h | 2 + 9 files changed, 594 insertions(+), 6 deletions(-) create mode 100644 drivers/md/dm-historical-service-time.c
Signature · 1 line
2.26.2
PATCH v3 1/2 md: mpath: Pass IO start time to path selector
HST is going to need this information to perform path prediction. For request-based mpath, we use the struct request io_start_time, while for bio based, use the DM layer start_time.
--- drivers/md/dm-mpath.c | 9 ++++++--- drivers/md/dm-path-selector.h | 2 +- drivers/md/dm-queue-length.c | 2 +- drivers/md/dm-service-time.c | 2 +- drivers/md/dm.c | 10 ++++++++++ include/linux/device-mapper.h | 2 ++ 6 files changed, 21 insertions(+), 6 deletions(-)
drivers/md/dm-mpath.c +6 -3
@@ -567,7 +567,8 @@ static void multipath_release_clone(struct request *clone,if (pgpath && pgpath->pg->ps.type->end_io) pgpath->pg->ps.type->end_io (&pgpath->pg->ps, &pgpath->path, - mpio->nr_bytes); + mpio->nr_bytes, + clone->io_start_time_ns); }blk_put_request (clone);
@@ -1620,7 +1621,8 @@ static int multipath_end_io(struct dm_target *ti, struct request *clone,struct path_selector *ps = &pgpath->pg->ps;if (ps->type->end_io) - ps->type->end_io (ps, &pgpath->path, mpio->nr_bytes); + ps->type->end_io (ps, &pgpath->path, mpio->nr_bytes, + clone->io_start_time_ns); }return r;
@@ -1664,7 +1666,8 @@ static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone,struct path_selector *ps = &pgpath->pg->ps;if (ps->type->end_io) - ps->type->end_io (ps, &pgpath->path, mpio->nr_bytes); + ps->type->end_io (ps, &pgpath->path, mpio->nr_bytes, +dm_start_time_ns_from_clone (clone)); }return r;
drivers/md/dm-path-selector.h +1 -1
@@ -74,7 +74,7 @@ struct path_selector_type {int (*start_io) (struct path_selector *ps,struct dm_path *path,size_t nr_bytes);int (*end_io) (struct path_selector *ps,struct dm_path *path, -size_t nr_bytes); +size_t nr_bytes,u64 start_time); };/* Register a path selector */
drivers/md/dm-queue-length.c +1 -1
@@ -227,7 +227,7 @@ static int ql_start_io(struct path_selector *ps, struct dm_path *path, }static int ql_end_io (struct path_selector *ps,struct dm_path *path, -size_t nr_bytes) +size_t nr_bytes,u64 start_time) {struct path_info *pi = path->pscontext;
drivers/md/dm-service-time.c +1 -1
@@ -309,7 +309,7 @@ static int st_start_io(struct path_selector *ps, struct dm_path *path, }static int st_end_io (struct path_selector *ps,struct dm_path *path, -size_t nr_bytes) +size_t nr_bytes,u64 start_time) {struct path_info *pi = path->pscontext;
drivers/md/dm.c +10 -0
@@ -674,6 +674,16 @@ static bool md_in_flight(struct mapped_device *md)return md_in_flight_bios (md); } +u64 dm_start_time_ns_from_clone (struct bio *bio) +{ + struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); +struct dm_io *io = tio->io; + +/* FIXME: convert io->start_time from jiffies to nanoseconds */ +return jiffies_to_nsecs (io->start_time); +} +EXPORT_SYMBOL_GPL (dm_start_time_ns_from_clone); +static void start_io_acct (struct dm_io *io) {struct mapped_device *md = io->md;
include/linux/device-mapper.h +2 -0
@@ -329,6 +329,8 @@ void *dm_per_bio_data(struct bio *bio, size_t data_size);struct bio *dm_bio_from_per_bio_data (void *data,size_t data_size);unsigned dm_bio_get_target_bio_nr (const struct bio *bio); +u64 dm_start_time_ns_from_clone (struct bio *bio); +int dm_register_target (struct target_type *t);void dm_unregister_target (struct target_type *t);
Signature · 1 line
2.26.2
PATCH v3 2/2 md: mpath: Add Historical Service Time Path Selector
In reply to Gabriel Krisman BertaziFrom: Khazhismel Kumykov <[email protected]>
This new selector keeps an exponential moving average of the service time for each path (losely defined as delta between start_io and end_io), and uses this along with the number of inflight requests to estimate future service time for a path. Since we don't have a prober to account for temporally slow paths, re-try "slow" paths every once in a while (num_paths * historical_service_time). To account for fast paths transitioning to slow, if a path has not completed any request within (num_paths * historical_service_time), limit the number of outstanding requests. To account for low volume situations where number of inflights would be zero, the last finish time of each path is factored in.
--- drivers/md/Kconfig | 11 + drivers/md/Makefile | 1 + drivers/md/dm-historical-service-time.c | 561 ++++++++++++++++++++++++ 3 files changed, 573 insertions(+) create mode 100644 drivers/md/dm-historical-service-time.c
drivers/md/Kconfig +11 -0
@@ -452,6 +452,17 @@ config DM_MULTIPATH_ST If unsure, say N. +config DM_MULTIPATH_HST + tristate "I/O Path Selector based on historical service time" + depends on DM_MULTIPATH + help + This path selector is a dynamic load balancer which selects + the path expected to complete the incoming I/O in the shortest + time by comparing estimated service time (based on historical + service time). + + If unsure, say N. + config DM_DELAY tristate "I/O delaying target" depends on BLK_DEV_DM
drivers/md/Makefile +1 -0
@@ -55,6 +55,7 @@ obj-$(CONFIG_DM_FLAKEY) += dm-flakey.o obj-$(CONFIG_DM_MULTIPATH) += dm-multipath.o dm-round-robin.o obj-$(CONFIG_DM_MULTIPATH_QL) += dm-queue-length.o obj-$(CONFIG_DM_MULTIPATH_ST) += dm-service-time.o +obj-$(CONFIG_DM_MULTIPATH_HST) += dm-historical-service-time.o obj-$(CONFIG_DM_SWITCH) += dm-switch.o obj-$(CONFIG_DM_SNAPSHOT) += dm-snapshot.o obj-$(CONFIG_DM_PERSISTENT_DATA) += persistent-data/
drivers/md/dm-historical-service-time.c new file +561 -0
561 lines not on this page · raw message
Signature · 1 line
2.26.2