Skip to contentSkip to conversation

nvme multipath eBPF path selector

RFC PATCH In review [email protected] 5 participants 13 messages Jul 29, 2025 – Jul 30, 2025

Subsystems (4)
lore
Share & tools
lore
#2 lore raw

PATCH 2/6 nvme: export nvme_find_get_subsystem()/nvme_put_subsystem()

In reply to [email protected]

From: Hannes Reinecke <[email protected]>

Export functions to find and release a subsystem based on the subsystem NQN.

---
 drivers/nvme/host/core.c | 14 ++++++++++++--
 drivers/nvme/host/nvme.h |  2 ++
 2 files changed, 14 insertions(+), 2 deletions(-)
drivers/nvme/host/core.c +12 -2
index 895fb163d48e..a2f3da453af4 100644
@@ -147,7 +147,6 @@ static const struct class nvme_ns_chr_class = {
 	.name = "nvme-generic",
 };
 
-static void nvme_put_subsystem(struct nvme_subsystem *subsys);
 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
 					   unsigned nsid);
 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
@@ -3115,7 +3114,7 @@ static void nvme_destroy_subsystem(struct kref *ref)
 	put_device(&subsys->dev);
 }
 
-static void nvme_put_subsystem(struct nvme_subsystem *subsys)
+void nvme_put_subsystem(struct nvme_subsystem *subsys)
 {
 	kref_put(&subsys->ref, nvme_destroy_subsystem);
 }
@@ -3148,6 +3147,17 @@ static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
 	return NULL;
 }
 
+struct nvme_subsystem *nvme_find_get_subsystem(const char *subsysnqn)
+{
+	struct nvme_subsystem *subsys;
+
+	mutex_lock(&nvme_subsystems_lock);
+	subsys = __nvme_find_get_subsystem(subsysnqn);
+	mutex_unlock(&nvme_subsystems_lock);
+	return subsys;
+}
+EXPORT_SYMBOL_GPL(nvme_find_get_subsystem);
+
 static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
 {
 	return ctrl->opts && ctrl->opts->discovery_nqn;
drivers/nvme/host/nvme.h +2 -0
index 7df2ea21851f..f1eb8ae57c84 100644
@@ -948,6 +948,8 @@ extern const struct attribute_group *nvme_subsys_attrs_groups[];
 extern const struct attribute_group *nvme_dev_attr_groups[];
 extern const struct block_device_operations nvme_bdev_ops;
 
+struct nvme_subsystem *nvme_find_get_subsystem(const char *subsysnqn);
+void nvme_put_subsystem(struct nvme_subsystem *subsys);
 void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl);
 struct nvme_ns *nvme_find_path(struct nvme_ns_head *head);
 #ifdef CONFIG_NVME_MULTIPATH
Signature · 1 line
2.43.0
#4 lore raw

PATCH 4/6 nvme: add 'sector' parameter to nvme_find_path()

In reply to [email protected]

From: Hannes Reinecke <[email protected]>

An nvme multipath iopolicy might need to make scheduling decisions based on the starting sector of the I/O, so add an argument 'sector' to nvme_find_path().

---
 drivers/nvme/host/ioctl.c     | 7 ++++---
 drivers/nvme/host/multipath.c | 8 ++++----
 drivers/nvme/host/nvme.h      | 2 +-
 drivers/nvme/host/pr.c        | 2 +-
 drivers/nvme/host/sysfs.c     | 2 +-
 5 files changed, 11 insertions(+), 10 deletions(-)
drivers/nvme/host/ioctl.c +4 -3
index 6b3ac8ae3f34..e210908ad78b 100644
@@ -716,7 +716,8 @@ int nvme_ns_head_ioctl(struct block_device *bdev, blk_mode_t mode,
 		flags |= NVME_IOCTL_PARTITION;
 
 	srcu_idx = srcu_read_lock(&head->srcu);
-	ns = nvme_find_path(head);
+	/* TBD: extract LBA and size to get the routing right */
+	ns = nvme_find_path(head, 0);
 	if (!ns)
 		goto out_unlock;
 
@@ -747,7 +748,7 @@ long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd,
 	int srcu_idx, ret = -EWOULDBLOCK;
 
 	srcu_idx = srcu_read_lock(&head->srcu);
-	ns = nvme_find_path(head);
+	ns = nvme_find_path(head, 0);
 	if (!ns)
 		goto out_unlock;
 
@@ -767,7 +768,7 @@ int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,
 	struct cdev *cdev = file_inode(ioucmd->file)->i_cdev;
 	struct nvme_ns_head *head = container_of(cdev, struct nvme_ns_head, cdev);
 	int srcu_idx = srcu_read_lock(&head->srcu);
-	struct nvme_ns *ns = nvme_find_path(head);
+	struct nvme_ns *ns = nvme_find_path(head, 0);
 	int ret = -EINVAL;
 
 	if (ns)
drivers/nvme/host/multipath.c +4 -4
index 696c2f817bed..dee40bd73449 100644
@@ -466,7 +466,7 @@ static struct nvme_ns *nvme_numa_path(struct nvme_ns_head *head)
 	return ns;
 }
 
-inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head)
+inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head, sector_t sector)
 {
 	switch (READ_ONCE(head->subsys->iopolicy)) {
 	case NVME_IOPOLICY_QD:
@@ -528,7 +528,7 @@ static void nvme_ns_head_submit_bio(struct bio *bio)
 		return;
 
 	srcu_idx = srcu_read_lock(&head->srcu);
-	ns = nvme_find_path(head);
+	ns = nvme_find_path(head, bio->bi_iter.bi_sector);
 	if (likely(ns)) {
 		bio_set_dev(bio, ns->disk->part0);
 		bio->bi_opf |= REQ_NVME_MPATH;
@@ -570,7 +570,7 @@ static int nvme_ns_head_get_unique_id(struct gendisk *disk, u8 id[16],
 	int srcu_idx, ret = -EWOULDBLOCK;
 
 	srcu_idx = srcu_read_lock(&head->srcu);
-	ns = nvme_find_path(head);
+	ns = nvme_find_path(head, 0);
 	if (ns)
 		ret = nvme_ns_get_unique_id(ns, id, type);
 	srcu_read_unlock(&head->srcu, srcu_idx);
@@ -586,7 +586,7 @@ static int nvme_ns_head_report_zones(struct gendisk *disk, sector_t sector,
 	int srcu_idx, ret = -EWOULDBLOCK;
 
 	srcu_idx = srcu_read_lock(&head->srcu);
-	ns = nvme_find_path(head);
+	ns = nvme_find_path(head, sector);
 	if (ns)
 		ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data);
 	srcu_read_unlock(&head->srcu, srcu_idx);
drivers/nvme/host/nvme.h +1 -1
index 2aff8df55d1c..73b966a6653a 100644
@@ -951,7 +951,7 @@ extern const struct block_device_operations nvme_bdev_ops;
 struct nvme_subsystem *nvme_find_get_subsystem(const char *subsysnqn);
 void nvme_put_subsystem(struct nvme_subsystem *subsys);
 void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl);
-struct nvme_ns *nvme_find_path(struct nvme_ns_head *head);
+struct nvme_ns *nvme_find_path(struct nvme_ns_head *head, sector_t sector);
 #ifdef CONFIG_NVME_MULTIPATH
 static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)
 {
drivers/nvme/host/pr.c +1 -1
index ca6a74607b13..ba8cee2d6001 100644
@@ -54,7 +54,7 @@ static int nvme_send_ns_head_pr_command(struct block_device *bdev,
 {
 	struct nvme_ns_head *head = bdev->bd_disk->private_data;
 	int srcu_idx = srcu_read_lock(&head->srcu);
-	struct nvme_ns *ns = nvme_find_path(head);
+	struct nvme_ns *ns = nvme_find_path(head, 0);
 	int ret = -EWOULDBLOCK;
 
 	if (ns) {
drivers/nvme/host/sysfs.c +1 -1
index 378107cf7a21..ebbdf512a31d 100644
@@ -194,7 +194,7 @@ static int ns_head_update_nuse(struct nvme_ns_head *head)
 		return 0;
 
 	srcu_idx = srcu_read_lock(&head->srcu);
-	ns = nvme_find_path(head);
+	ns = nvme_find_path(head, 0);
 	if (!ns)
 		goto out_unlock;
 
Signature · 1 line
2.43.0
#5 lore raw

PATCH 5/6 nvme-bpf: eBPF struct_ops path selectors

In reply to [email protected]

From: Hannes Reinecke <[email protected]>

Add support for eBPF struct_ops based path selectors. Due to eBPF limitations we cannot pass in a 'struct nvme_ns' as argument nor return a 'struct nvme_ns' from a function, so the eBPF path selectors use a 'struct nvme_bpf_iter' to iterate over all paths in a struct nvme_ns_head.
That satifies the constrains of the eBPF verifier and allows us to provide two helper functions 'nvme_bpf_first_path()' and 'nvme_bpf_next_path()' to iterate over all paths in a namespace.

---
 drivers/nvme/host/Kconfig     |   9 +
 drivers/nvme/host/Makefile    |   1 +
 drivers/nvme/host/bpf.h       |  33 ++++
 drivers/nvme/host/bpf_ops.c   | 347 ++++++++++++++++++++++++++++++++++
 drivers/nvme/host/core.c      |   3 +
 drivers/nvme/host/multipath.c |   9 +-
 drivers/nvme/host/nvme.h      |   6 +-
 include/linux/nvme-bpf.h      |  54 ++++++
 8 files changed, 460 insertions(+), 2 deletions(-)
 create mode 100644 drivers/nvme/host/bpf.h
 create mode 100644 drivers/nvme/host/bpf_ops.c
 create mode 100644 include/linux/nvme-bpf.h
drivers/nvme/host/Kconfig +9 -0
index 31974c7dd20c..7cc1f3898712 100644
@@ -122,6 +122,15 @@ config NVME_HOST_AUTH
 
 	  If unsure, say N.
 
+config NVME_BPF
+	bool "NVMe multipath BPF path selector"
+	depends on NVME_MULTIPATH
+	depends on BPF_SYSCALL
+	help
+	  Provide support for eBPF multipath path selectors
+
+	  if unsure, say N.
+
 config NVME_APPLE
 	tristate "Apple ANS2 NVM Express host driver"
 	depends on OF && BLOCK
drivers/nvme/host/Makefile +1 -0
index 6414ec968f99..f81d6349faf6 100644
@@ -18,6 +18,7 @@ nvme-core-$(CONFIG_BLK_DEV_ZONED)	+= zns.o
 nvme-core-$(CONFIG_FAULT_INJECTION_DEBUG_FS)	+= fault_inject.o
 nvme-core-$(CONFIG_NVME_HWMON)		+= hwmon.o
 nvme-core-$(CONFIG_NVME_HOST_AUTH)	+= auth.o
+nvme-core-$(CONFIG_NVME_BPF)		+= bpf_ops.o
 
 nvme-y					+= pci.o
 
drivers/nvme/host/bpf.h new file +33 -0
new file mode 100644
index 000000000000..f819332d5293
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#ifndef NVME_INT_BPF_HEADER
+#define NVME_INT_BPF_HEADER
+
+#ifdef CONFIG_NVME_BPF
+#include <linux/filter.h>
+#include <linux/nvme-bpf.h>
+
+static inline bool nvme_bpf_enabled(struct nvme_ns_head *head)
+{
+	return !!(srcu_dereference(head->bpf_ops, &head->srcu));
+}
+
+void nvme_bpf_detach(struct nvme_ns_head *head);
+struct nvme_ns *nvme_bpf_select_path(struct nvme_ns_head *head, sector_t sector);
+
+int __init nvme_bpf_struct_ops_init(void);
+
+#else
+
+static inline bool nvme_bpf_enabled(struct nvme_ns_head *head)
+{
+	return false;
+}
+
+static inline void nvme_bpf_detach(struct nvme_ns_head *head) {}
+static inline struct nvme_ns *nvme_bpf_select_path(struct nvme_ns_head *head, sector_t sector)
+{
+	return NULL;
+}
+
+#endif
+#endif
drivers/nvme/host/bpf_ops.c new file +347 -0

347 lines not on this page · raw message

drivers/nvme/host/core.c +3 -0
index a2f3da453af4..e4f69b2f946b 100644
@@ -27,6 +27,7 @@
 #include "nvme.h"
 #include "fabrics.h"
 #include <linux/nvme-auth.h>
+#include "bpf.h"
 
 #define CREATE_TRACE_POINTS
 #include "trace.h"
@@ -5381,6 +5382,8 @@ static int __init nvme_core_init(void)
 	result = nvme_init_auth();
 	if (result)
 		goto destroy_ns_chr;
+	if (IS_ENABLED(CONFIG_NVME_BPF))
+		nvme_bpf_struct_ops_init();
 	return 0;
 
 destroy_ns_chr:
drivers/nvme/host/multipath.c +8 -1
index dee40bd73449..e2c6b13591c4 100644
@@ -8,6 +8,7 @@
 #include <linux/vmalloc.h>
 #include <trace/events/block.h>
 #include "nvme.h"
+#include "bpf.h"
 
 bool multipath = true;
 static bool multipath_always_on;
@@ -462,12 +463,15 @@ static struct nvme_ns *nvme_numa_path(struct nvme_ns_head *head)
 	else if (unlikely(!nvme_path_is_optimized(ns)))
 		ns = __nvme_find_path(head, node);
 	if (ns)
-		rcu_assign_pointer(head->current_path[node], found);
+		rcu_assign_pointer(head->current_path[node], ns);
 	return ns;
 }
 
 inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head, sector_t sector)
 {
+	if (nvme_bpf_enabled(head))
+		return nvme_bpf_select_path(head, sector);
+
 	switch (READ_ONCE(head->subsys->iopolicy)) {
 	case NVME_IOPOLICY_QD:
 		return nvme_queue_depth_path(head);
@@ -693,6 +697,7 @@ static void nvme_remove_head(struct nvme_ns_head *head)
 		kblockd_schedule_work(&head->requeue_work);
 
 		nvme_cdev_del(&head->cdev, &head->cdev_device);
+		nvme_bpf_detach(head);
 		synchronize_srcu(&head->srcu);
 		del_gendisk(head->disk);
 	}
@@ -727,6 +732,8 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
 	INIT_WORK(&head->requeue_work, nvme_requeue_work);
 	INIT_WORK(&head->partition_scan_work, nvme_partition_scan_work);
 	INIT_DELAYED_WORK(&head->remove_work, nvme_remove_head_work);
+	if (IS_ENABLED(CONFIG_NVME_BPF))
+		INIT_LIST_HEAD(&head->bpf_list);
 	head->delayed_removal_secs = 0;
 
 	/*
drivers/nvme/host/nvme.h +5 -1
index 73b966a6653a..3498620d650b 100644
@@ -17,7 +17,7 @@
 #include <linux/wait.h>
 #include <linux/t10-pi.h>
 #include <linux/ratelimit_types.h>
-
+#include <linux/bpf.h>
 #include <trace/events/block.h>
 
 extern const struct pr_ops nvme_pr_ops;
@@ -499,6 +499,10 @@ struct nvme_ns_head {
 
 	u16			nr_plids;
 	u16			*plids;
+#ifdef CONFIG_NVME_BPF
+	struct list_head	bpf_list;
+	struct nvme_bpf_ops __rcu *bpf_ops;
+#endif
 #ifdef CONFIG_NVME_MULTIPATH
 	struct bio_list		requeue_list;
 	spinlock_t		requeue_lock;
include/linux/nvme-bpf.h new file +54 -0

54 lines not on this page · raw message

Signature · 1 line
2.43.0
#6 lore raw

PATCH 6/6 tools/testing/selftests: add sample nvme bpf path selector

In reply to [email protected]

From: Hannes Reinecke <[email protected]>

As a simple nvme bpf path selector to demonstrate the namespace path iteration.

---
 .../selftests/bpf/progs/bpf_nvme_simple.c     | 52 +++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 tools/testing/selftests/bpf/progs/bpf_nvme_simple.c
tools/testing/selftests/bpf/progs/bpf_nvme_simple.c new file +52 -0
new file mode 100644
index 000000000000..c9cafb6bd253
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * simple nvme ebpf path selector
+ *
+ * Simulates a RAID layout with chunk size 2M
+ */
+
+#include <vmlinux.h>
+#include <errno.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+char _license[] SEC("license") = "GPL";
+
+static sector_t simple_offset = 0;
+static sector_t simple_blocksize = 1048576;
+
+SEC("struct_ops")
+int BPF_PROG(simple_select, struct nvme_bpf_iter *iter, sector_t sector)
+{
+	sector_t offset = simple_offset;
+	sector_t block_size = simple_blocksize;
+	u32 num_blks, num_paths, num_iter, i;
+	int cntlid;
+
+	if (sector > offset)
+		sector -= offset;
+	cntlid = nvme_bpf_first_path(iter);
+	if (cntlid < 0)
+		return cntlid;
+	if (!block_size || sector < block_size)
+		return cntlid;
+
+	num_blks = (sector / block_size);
+	num_paths = nvme_bpf_count_paths(iter);
+	num_iter = num_blks % num_paths;
+	bpf_for (i, 1, num_iter) {
+		cntlid = nvme_bpf_next_path(iter);
+		if (cntlid < 0)
+			break;
+	}
+	return cntlid;
+}
+
+SEC(".struct_ops")
+struct nvme_bpf_ops bpf_nvme_simple = {
+	.uuid = { 0x86, 0xee, 0x41, 0xd5, 0x25, 0x6b, 0x45, 0xd0, 0xa4, 0x81, 0x5e, 0x35, 0xf6, 0x02, 0xf5, 0x11 },
+	.subsysnqn = "blktests-subsystem-1",
+	.nsid = 1,
+	.select_path = (void *)simple_select,
+};
Signature · 1 line
2.43.0
#3 lore raw

PATCH 3/6 nvme: add per-namespace iopolicy sysfs attribute

In reply to [email protected]

From: Hannes Reinecke <[email protected]>

To display iopolicies which are different from the subsystem.

---
 drivers/nvme/host/multipath.c | 14 ++++++++++++++
 drivers/nvme/host/nvme.h      |  1 +
 drivers/nvme/host/sysfs.c     |  7 +++++++
 3 files changed, 22 insertions(+)
drivers/nvme/host/multipath.c +14 -0
index 116b2e71d339..696c2f817bed 100644
@@ -1137,6 +1137,20 @@ static ssize_t numa_nodes_show(struct device *dev, struct device_attribute *attr
 }
 DEVICE_ATTR_RO(numa_nodes);
 
+static ssize_t iopolicy_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct gendisk *disk = dev_to_disk(dev);
+	struct nvme_ns_head *head = disk->private_data;
+	int iopolicy;
+
+	if (nvme_bpf_enabled(head))
+		return sysfs_emit(buf, "bpf(%pUb)\n", &head->bpf_ops->uuid);
+	iopolicy = READ_ONCE(head->subsys->iopolicy);
+	return sysfs_emit(buf, "%s\n", nvme_iopolicy_names[iopolicy]);
+}
+DEVICE_ATTR_RO(iopolicy);
+
 static ssize_t delayed_removal_secs_show(struct device *dev,
 		struct device_attribute *attr, char *buf)
 {
drivers/nvme/host/nvme.h +1 -0
index f1eb8ae57c84..2aff8df55d1c 100644
@@ -994,6 +994,7 @@ extern struct device_attribute dev_attr_ana_grpid;
 extern struct device_attribute dev_attr_ana_state;
 extern struct device_attribute dev_attr_queue_depth;
 extern struct device_attribute dev_attr_numa_nodes;
+extern struct device_attribute dev_attr_iopolicy;
 extern struct device_attribute dev_attr_delayed_removal_secs;
 extern struct device_attribute subsys_attr_iopolicy;
 
drivers/nvme/host/sysfs.c +7 -0
index 29430949ce2f..378107cf7a21 100644
@@ -260,6 +260,7 @@ static struct attribute *nvme_ns_attrs[] = {
 	&dev_attr_ana_state.attr,
 	&dev_attr_queue_depth.attr,
 	&dev_attr_numa_nodes.attr,
+	&dev_attr_iopolicy.attr,
 	&dev_attr_delayed_removal_secs.attr,
 #endif
 	&dev_attr_io_passthru_err_log_enabled.attr,
@@ -297,6 +298,12 @@ static umode_t nvme_ns_attrs_are_visible(struct kobject *kobj,
 		if (nvme_disk_is_ns_head(dev_to_disk(dev)))
 			return 0;
 	}
+	if (a == &dev_attr_iopolicy.attr) {
+		struct gendisk *disk = dev_to_disk(dev);
+
+		if (!nvme_disk_is_ns_head(disk))
+			return 0;
+	}
 	if (a == &dev_attr_delayed_removal_secs.attr) {
 		struct gendisk *disk = dev_to_disk(dev);
 
Signature · 1 line
2.43.0
#1 lore raw

PATCH 1/6 nvme-multipath: do not assign ->current_path in __nvme_find_path()

In reply to [email protected]

From: Hannes Reinecke <[email protected]>

__nvme_find_path() is the fallback if no paths are selected, but setting ->current_path[] is only relevant for 'numa' and 'round-robin' iopolicies. So move it out of __nvme_find_path() to avoid a pointless assignment for other path checkers.

---
 drivers/nvme/host/multipath.c | 40 +++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 13 deletions(-)
drivers/nvme/host/multipath.c +27 -13
index 3da980dc60d9..116b2e71d339 100644
@@ -342,8 +342,6 @@ static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node)
 
 	if (!found)
 		found = fallback;
-	if (found)
-		rcu_assign_pointer(head->current_path[node], found);
 	return found;
 }
 
@@ -364,8 +362,13 @@ static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head)
 	struct nvme_ns *old = srcu_dereference(head->current_path[node],
 					       &head->srcu);
 
-	if (unlikely(!old))
-		return __nvme_find_path(head, node);
+	if (unlikely(!old)) {
+		ns = __nvme_find_path(head, node);
+		if (ns)
+			rcu_assign_pointer(head->current_path[node], found);
+		return ns;
+	}
+
 
 	if (list_is_singular(&head->list)) {
 		if (nvme_path_is_disabled(old))
@@ -455,9 +458,11 @@ static struct nvme_ns *nvme_numa_path(struct nvme_ns_head *head)
 
 	ns = srcu_dereference(head->current_path[node], &head->srcu);
 	if (unlikely(!ns))
-		return __nvme_find_path(head, node);
-	if (unlikely(!nvme_path_is_optimized(ns)))
-		return __nvme_find_path(head, node);
+		ns = __nvme_find_path(head, node);
+	else if (unlikely(!nvme_path_is_optimized(ns)))
+		ns = __nvme_find_path(head, node);
+	if (ns)
+		rcu_assign_pointer(head->current_path[node], found);
 	return ns;
 }
 
@@ -798,12 +803,21 @@ static void nvme_mpath_set_live(struct nvme_ns *ns)
 
 	mutex_lock(&head->lock);
 	if (nvme_path_is_optimized(ns)) {
-		int node, srcu_idx;
-
-		srcu_idx = srcu_read_lock(&head->srcu);
-		for_each_online_node(node)
-			__nvme_find_path(head, node);
-		srcu_read_unlock(&head->srcu, srcu_idx);
+		int srcu_idx, iopolicy;
+
+		iopolicy = READ_ONCE(head->subsys->iopolicy);
+		if (iopolicy == NVME_IOPOLICY_NUMA ||
+		    iopolicy == NVME_IOPOLICY_RR) {
+			int node;
+
+			srcu_idx = srcu_read_lock(&head->srcu);
+			for_each_online_node(node) {
+				struct nvme_ns *ns = __nvme_find_path(head, node);
+				if (ns)
+					rcu_assign_pointer(head->current_path[node], ns);
+			}
+			srcu_read_unlock(&head->srcu, srcu_idx);
+		}
 	}
 	mutex_unlock(&head->lock);
 
Signature · 1 line
2.43.0
#0 lore raw

cover nvme multipath eBPF path selector

From: Hannes Reinecke <[email protected]>

Hi all,

there are discussion on having to deploy more complex I/O scheduling algorithms for NVMe, but then there's the question whether we really want to carry these in the kernel.
Which sounded like an ideal testbed for eBPF struct_ops to me. Taking a cue from Ming Lei's patchset for eBPF on ublk (thanks, Ming!) I've started messing around with eBPF.

So here's a patchset to implement nvme multipath eBPF path selectors. Idea's quite simple: the eBPF 'struct_ops' program is providing a 'select_path' function, which selects a nvme_ns struct to use for the I/O starting at a given sector.
Unfortunately ePBF doesn't allow to pass pointers, _and_ the definitions for 'struct nvme_ns_head' and 'struct nvme_ns' are internal to the nvme subsystem. So I kept those structures as opaque pointers for ePBF, and introduced a 'nvme_bpf_iter' structure as a path iterator. There are two functions 'nvme_bpf_first_path' and 'nvme_bpf_next_path' which can be used for an open-coded loop over all paths. I've also added sample code as an example how the loop can be coded.

It's all pretty rudimentary (as I'm sure people will need accessors to get to any namespace or controller details), but that's why I sent it out as an RFC. And I am by no means an eBPF expert, so I'd be glad for any corrections or suggestions for a better eBPF integration.

The entire patchset can be found at:
git.kernel.org:/pub/scm/linux/kernel/git/hare/scsi-devel.git branch nvme-bpf

As usual, reviews and comments are welcome.

Hannes Reinecke (6):
nvme-multipath: do not assign ->current_path in __nvme_find_path()
nvme: export nvme_find_get_subsystem()/nvme_put_subsystem()
nvme: add per-namespace iopolicy sysfs attribute
nvme: add 'sector' parameter to nvme_find_path()
nvme-bpf: eBPF struct_ops path selectors tools/testing/selftests: add sample nvme bpf path selector

 drivers/nvme/host/Kconfig                     |   9 +
 drivers/nvme/host/Makefile                    |   1 +
 drivers/nvme/host/bpf.h                       |  33 ++
 drivers/nvme/host/bpf_ops.c                   | 347 ++++++++++++++++++
 drivers/nvme/host/core.c                      |  17 +-
 drivers/nvme/host/ioctl.c                     |   7 +-
 drivers/nvme/host/multipath.c                 |  69 +++-
 drivers/nvme/host/nvme.h                      |  11 +-
 drivers/nvme/host/pr.c                        |   2 +-
 drivers/nvme/host/sysfs.c                     |   9 +-
 include/linux/nvme-bpf.h                      |  54 +++
 .../selftests/bpf/progs/bpf_nvme_simple.c     |  52 +++
 12 files changed, 585 insertions(+), 26 deletions(-)
 create mode 100644 drivers/nvme/host/bpf.h
 create mode 100644 drivers/nvme/host/bpf_ops.c
 create mode 100644 include/linux/nvme-bpf.h
 create mode 100644 tools/testing/selftests/bpf/progs/bpf_nvme_simple.c
Signature · 1 line
2.43.0
#9 lore raw
Quoted 6 lines · On Tue, Jul 29, 2025 at 09:06:47AM +0200, [email protected] wrote:

From: Hannes Reinecke <[email protected]>

Hi all,

there are discussion on having to deploy more complex I/O scheduling algorithms for NVMe,

Are "we"? Where?

Quoted 2 lines

but then there's the question whether we really
want to carry these in the kernel.

If it makes sense, "we" do, yes.

"We" don't want to export random crap for sure.

#10 lore raw
Quoted 9 lines · Christoph Hellwig
Quoted 6 lines · On Tue, Jul 29, 2025 at 09:06:47AM +0200, [email protected] wrote:

From: Hannes Reinecke <[email protected]>

Hi all,

there are discussion on having to deploy more complex I/O scheduling algorithms for NVMe,

Are "we"? Where?

I think that's me. I was originally looking into eBPF path selectors for ADNN:

https://www.snia.org/educational-library/adaptive-distributed-nvme-namespaces-2020

I was bugging Hannes about what happened to the spec. The idea was just that if the spec is dead, then something like eBPF path selectors could be an option.

I'm not currently looking at it, because I got busy with other work. But, maybe the ceph people would want it for their nvme frontend like in the video. For the ceph iscsi project, I had looked at SCSI referrals, and it didn't fit (I can't remember the details), and ADNN looked promising.

Quoted 9 lines
Quoted 3 lines

but then there's the question whether we really
want to carry these in the kernel.

If it makes sense, "we" do, yes.

"We" don't want to export random crap for sure.

#12 lore raw
In reply to [email protected]

Hi Hannes,

Quoted 11 lines · On Tue, 2025-07-29 at 09:06 +0200, [email protected] wrote:

From: Hannes Reinecke <[email protected]>

Hi all,

there are discussion on having to deploy more complex I/O scheduling algorithms for NVMe, but then there's the question whether we really want to carry these in the kernel.
Which sounded like an ideal testbed for eBPF struct_ops to me. Taking a cue from Ming Lei's patchset for eBPF on ublk (thanks, Ming!)
I've started messing around with eBPF.

I happen to have experience in this area and would like to participate in the development of nvme-bpf. I have previously developed the MPTCP BPF packet scheduler [1], which is already in the export branch of the MPTCP repository [2].

[1]
https://github.com/multipath-tcp/mptcp_net-next/issues/75

[2]
https://github.com/multipath-tcp/mptcp_net-next/commit/e83320eb669f48effae8a2d203d834ca2454308a
https://github.com/multipath-tcp/mptcp_net-next/commit/397b7213a2e45bc0c188d5fefa0889899657716f
https://github.com/multipath-tcp/mptcp_net-next/commit/0c59c5d43f6babd016bdbbf00365257ea57796e9

Quoted 54 lines

So here's a patchset to implement nvme multipath eBPF path selectors. Idea's quite simple: the eBPF 'struct_ops' program is providing a 'select_path' function, which selects a nvme_ns struct to use for the I/O starting at a given sector.
Unfortunately ePBF doesn't allow to pass pointers, _and_ the definitions
for 'struct nvme_ns_head' and 'struct nvme_ns' are internal to the nvme subsystem. So I kept those structures as opaque pointers for ePBF, and introduced a 'nvme_bpf_iter' structure as a path iterator. There are two functions 'nvme_bpf_first_path' and 'nvme_bpf_next_path'
which can be used for an open-coded loop over all paths. I've also added sample code as an example how the loop can be coded.

It's all pretty rudimentary (as I'm sure people will need accessors to get to any namespace or controller details), but that's why I sent it out as an RFC. And I am by no means an eBPF expert, so I'd be glad for any corrections or suggestions for a better eBPF integration.

The entire patchset can be found at:
git.kernel.org:/pub/scm/linux/kernel/git/hare/scsi-devel.git branch nvme-bpf

As usual, reviews and comments are welcome.

Hannes Reinecke (6):
  nvme-multipath: do not assign ->current_path in __nvme_find_path()
  nvme: export nvme_find_get_subsystem()/nvme_put_subsystem()
  nvme: add per-namespace iopolicy sysfs attribute
  nvme: add 'sector' parameter to nvme_find_path()
  nvme-bpf: eBPF struct_ops path selectors tools/testing/selftests: add sample nvme bpf path selector

 drivers/nvme/host/Kconfig                     |   9 + drivers/nvme/host/Makefile                    |   1 + drivers/nvme/host/bpf.h                       |  33 ++ drivers/nvme/host/bpf_ops.c                   | 347
++++++++++++++++++
 drivers/nvme/host/core.c                      |  17 +- drivers/nvme/host/ioctl.c                     |   7 +- drivers/nvme/host/multipath.c                 |  69 +++- drivers/nvme/host/nvme.h                      |  11 +- drivers/nvme/host/pr.c                        |   2 +- drivers/nvme/host/sysfs.c                     |   9 +- include/linux/nvme-bpf.h                      |  54 +++ .../selftests/bpf/progs/bpf_nvme_simple.c     |  52 +++

 12 files changed, 585 insertions(+), 26 deletions(-)

 create mode 100644 drivers/nvme/host/bpf.h create mode 100644 drivers/nvme/host/bpf_ops.c create mode 100644 include/linux/nvme-bpf.h create mode 100644 tools/testing/selftests/bpf/progs/bpf_nvme_simple.c

#7 lore raw

Re: [PATCH 6/6] tools/testing/selftests: add sample nvme bpf path selector

In reply to [email protected]

Hi Hannes,

I tried to compile this BPF program, but got a compilation error. To fix it, NVME_BPF and its dependencies need to be enabled in tools/testing/selftests/bpf/config:

+++ b/tools/testing/selftests/bpf/config
@@ -121,3 +121,8 @@ CONFIG_XDP_SOCKETS=y
CONFIG_XFRM_INTERFACE=y
CONFIG_TCP_CONG_DCTCP=y
CONFIG_TCP_CONG_BBR=y
+CONFIG_NVME_CORE=y
+CONFIG_NVME_FABRICS=y
+CONFIG_NVME_TCP=y
+CONFIG_NVME_MULTIPATH=y
+CONFIG_NVME_BPF=y

Quoted 9 lines · On Tue, 2025-07-29 at 09:06 +0200, [email protected] wrote:

From: Hannes Reinecke <[email protected]>

As a simple nvme bpf path selector to demonstrate the namespace path iteration.

---
 .../selftests/bpf/progs/bpf_nvme_simple.c     | 52
+++++++++++++++++++

We also need to create an nvme test program in tools/testing/selftests/bpf/prog_tests/ to load and verify this bpf_nvme_simple BPF program, just like I did in tools/testing/selftests/bpf/prog_tests/mptcp.c. I can try to implement it if you think it's necessary.

Thanks,
-Geliang

Quoted 65 lines
 1 file changed, 52 insertions(+)

 create mode 100644 tools/testing/selftests/bpf/progs/bpf_nvme_simple.c

tools/testing/selftests/bpf/progs/bpf_nvme_simple.c new file +20 -0
new file mode 100644
index 000000000000..c9cafb6bd253
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * simple nvme ebpf path selector
+ *
+ * Simulates a RAID layout with chunk size 2M
+ */
+
+#include <vmlinux.h>
+#include <errno.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+char _license[] SEC("license") = "GPL";
+
+static sector_t simple_offset = 0;
+static sector_t simple_blocksize = 1048576;
+
+SEC("struct_ops")
+int BPF_PROG(simple_select, struct nvme_bpf_iter *iter, sector_t

sector)

tools/testing/selftests/bpf/progs/bpf_nvme_simple.c new file +28 -0
+{
+	sector_t offset = simple_offset;
+	sector_t block_size = simple_blocksize;
+	u32 num_blks, num_paths, num_iter, i;
+	int cntlid;
+
+	if (sector > offset)
+		sector -= offset;
+	cntlid = nvme_bpf_first_path(iter);
+	if (cntlid < 0)
+		return cntlid;
+	if (!block_size || sector < block_size)
+		return cntlid;
+
+	num_blks = (sector / block_size);
+	num_paths = nvme_bpf_count_paths(iter);
+	num_iter = num_blks % num_paths;
+	bpf_for (i, 1, num_iter) {
+		cntlid = nvme_bpf_next_path(iter);
+		if (cntlid < 0)
+			break;
+	}
+	return cntlid;
+}
+
+SEC(".struct_ops")
+struct nvme_bpf_ops bpf_nvme_simple = {
+	.uuid = { 0x86, 0xee, 0x41, 0xd5, 0x25, 0x6b, 0x45, 0xd0,

0xa4, 0x81, 0x5e, 0x35, 0xf6, 0x02, 0xf5, 0x11 },

tools/testing/selftests/bpf/progs/bpf_nvme_simple.c new file +4 -0
+	.subsysnqn = "blktests-subsystem-1",
+	.nsid = 1,
+	.select_path = (void *)simple_select,
+};
#8 lore raw

Re: [PATCH 6/6] tools/testing/selftests: add sample nvme bpf path selector

Quoted 9 lines · Geliang Tang

Hi Hannes,

I tried to compile this BPF program, but got a compilation error. To fix it, NVME_BPF and its dependencies need to be enabled in tools/testing/selftests/bpf/config:

+++ b/tools/testing/selftests/bpf/config

Review CONFIG_XDP_SOCKETS=y · line 128

Quoted 8 lines
(quoted diff) +5 -0
@@ -121,3 +121,8 @@ CONFIG_XDP_SOCKETS=y
  CONFIG_XFRM_INTERFACE=y
  CONFIG_TCP_CONG_DCTCP=y
  CONFIG_TCP_CONG_BBR=y
+CONFIG_NVME_CORE=y
+CONFIG_NVME_FABRICS=y
+CONFIG_NVME_TCP=y
+CONFIG_NVME_MULTIPATH=y
+CONFIG_NVME_BPF=y

Yeah; that's correct.
Thanks for that.

That's one thing I forgot to mention: currently one has to compile in NVME_CORE and NVME_BPF; one will get linking errors otherwise. Will see if I can fix it up.

Cheers,

Hannes

Signature · 4 lines
Dr. Hannes Reinecke                  Kernel Storage Architect
[email protected]                                +49 911 74053 688
SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg
HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich
#11 lore raw
In reply to Mike Christie
Quoted 9 lines · Mike Christie
Quoted 8 lines · Christoph Hellwig
Quoted 6 lines · On Tue, Jul 29, 2025 at 09:06:47AM +0200, [email protected] wrote:

From: Hannes Reinecke <[email protected]>

Hi all,

there are discussion on having to deploy more complex I/O scheduling algorithms for NVMe,

Quoted 6 lines
Quoted 1 line

Are "we"? Where?

I think that's me. I was originally looking into eBPF path selectors for ADNN:

https://www.snia.org/educational-library/adaptive-distributed-nvme-namespaces-2020

ADNN needs to use eBPF, but controller provided eBPF, not from the user. This has been clearly communicated to the group in 2022, but as usual FMDS is drifting off into their own parallel universe, not only on the issue of path selection. Based on these factors ADNN in it's current trajectory is unlikely to ever be implemented in Linux.