nvmet: Reference namespaces percpu
Subsystems (2)
RFC PATCH 1/1 nvmet: Reference namespaces percpu
In order to avoid cross-cpu or cross-socket contention on our namespaces referencing in the hot path, use the much more efficient percpu_ref.
Signed-off-by Sagi Grimberg <sagig at mellanox.com>
---
FYI, with this patch applied I'm able to get 95% IOPs out of nvme_loop comparing to raw null_blk. I'm also able to survive hot namespace removal during stress IO.
drivers/nvme/target/core.c | 38 ++++++++++++++++++++++++++++++++------ drivers/nvme/target/nvmet.h | 4 +++- 2 files changed, 35 insertions(+), 7 deletions(-)
drivers/nvme/target/core.c +32 -6
@@ -37,16 +37,18 @@ struct nvmet_ns *nvmet_find_namespace(struct nvmet_ctrl *ctrl, __le32 nsid)rcu_read_lock (); ns =__nvmet_find_namespace (ctrl, nsid); -if (ns && !kref_get_unless_zero (&ns->ref)) - ns =NULL ; +if (ns) +percpu_ref_get (&ns->ref);rcu_read_unlock ();return ns; } -static void nvmet_destroy_namespace (struct kref *ref) +static void nvmet_free_ns (struct work_struct *work) { - struct nvmet_ns *ns = container_of(ref, struct nvmet_ns, ref); + struct nvmet_ns *ns = container_of(work, struct nvmet_ns, work); + +percpu_ref_exit (&ns->ref);if (ns->bdev)blkdev_put (ns->bdev,FMODE_WRITE |FMODE_READ );
@@ -54,9 +56,19 @@ static void nvmet_destroy_namespace(struct kref *ref)kfree (ns); } +static void nvmet_destroy_namespace (struct percpu_ref *ref) +{ + struct nvmet_ns *ns = container_of(ref, struct nvmet_ns, ref); + +/* Now that we're off the ns list, we can safely kill percpu_ref */ +percpu_ref_kill (&ns->ref); +INIT_WORK (&ns->work, nvmet_free_ns); +schedule_work (&ns->work); +} +void nvmet_put_namespace (struct nvmet_ns *ns) { -kref_put (&ns->ref, nvmet_destroy_namespace); +percpu_ref_put (&ns->ref); }int nvmet_ns_enable (struct nvmet_ns *ns,const char *path)
@@ -109,22 +121,36 @@ void nvmet_ns_free(struct nvmet_ns *ns)list_del_init (&ns->dev_link);mutex_unlock (&subsys->lock); +/* Should be the final ref! */ nvmet_put_namespace (ns); }struct nvmet_ns *nvmet_ns_alloc (struct nvmet_subsys *subsys,u32 nsid) {struct nvmet_ns *ns; +int ret; ns =kzalloc (sizeof (*ns),GFP_KERNEL );if (!ns)return NULL ;INIT_LIST_HEAD (&ns->dev_link); -kref_init (&ns->ref); + ret =percpu_ref_init (&ns->ref, nvmet_destroy_namespace, +0 ,GFP_KERNEL ); +if (ret) + goto free_ns; + +percpu_ref_get (&ns->ref); + ns->nsid = nsid; ns->subsys = subsys; +return ns; + +free_ns: +kfree (ns); + +return NULL ; }void nvmet_req_complete (struct nvmet_req *req,u16 status)
drivers/nvme/target/nvmet.h +3 -1
@@ -5,6 +5,7 @@#include <linux/types.h> #include <linux/device.h> #include <linux/kref.h> +#include <linux/percpu-refcount.h> #include <linux/list.h> #include <linux/mutex.h> #include <linux/nvme.h>
@@ -13,7 +14,7 @@struct nvmet_ns {struct list_head dev_link; -struct kref ref; +struct percpu_ref ref;struct block_device *bdev;u32 nsid;u32 blksize_shift;
@@ -26,6 +27,7 @@ struct nvmet_ns {struct config_group default_groups[2 ];struct config_group group;struct rcu_head rcu; +struct work_struct work; };static inline struct nvmet_ns *to_nvmet_ns (struct config_item *item)
Signature · 1 line
1.7.1
RFC PATCH 1/1 nvmet: Reference namespaces percpu
Thanks Sagi,
this looks good to me. If only the percpu_ref API would call ->release in a useful context we could get rid of all this workqueue boilerplate code in most users..