rbd: client list locking improvements

PATCH In review Alex Elder 2 participants 6 messages Feb 29, 2012 – Mar 2, 2012

Subsystems (2)

Block layer RADOS block device (RBD)

lore

Share & tools

lore

Alex Elder

author

2012-02-29 03:44 UTC #0 lore raw

cover rbd: client list locking improvements

This series reduces the window during which the client list lock is held, gives it a more meaningful name, and moves the locking calls closer to the places they're really needed.

-Alex

Alex Elder

author

2012-02-29 03:45 UTC #1 lore raw

PATCH 1/4 rbd: release client list lock sooner

In rbd_get_client(), if a client is reused, a number of things get done while still holding the list lock unnecessarily.

This just moves a few things that need no lock protection outside the lock.

Signed-off-by Alex Elder <[email protected]>

---
 drivers/block/rbd.c |   10 ++++++----
 1 files changed, 6 insertions(+), 4 deletions(-)

drivers/block/rbd.c +6 -4

index 4c5bb39..4bf8300 100644
@@ -383,13 +383,15 @@ static int rbd_get_client(struct rbd_device *rbd_dev, const char *mon_addr,
 	spin_lock(&node_lock);
 	rbdc = __rbd_client_find(opt);
 	if (rbdc) {
-		ceph_destroy_options(opt);
-		kfree(rbd_opts);
-
 		/* using an existing client */
 		kref_get(&rbdc->kref);
-		rbd_dev->rbd_client = rbdc;
 		spin_unlock(&node_lock);
+
+		rbd_dev->rbd_client = rbdc;
+
+		ceph_destroy_options(opt);
+		kfree(rbd_opts);
+
 		return 0;
 	}
 	spin_unlock(&node_lock);

Signature · 1 line

1.7.5.4

Alex Elder

author

2012-02-29 03:45 UTC #2 lore raw

PATCH 2/4 rbd: move ctl_mutex lock inside rbd_get_client()

In reply to Alex Elder

Since rbd_get_client() is only called in one place, move the acquisition of the mutex around that call inside that function.

Furthermore, within rbd_get_client(), it appears the mutex only needs to be held while calling rbd_client_create(). (Moving the lock inside that function will wait for the next patch.)

Signed-off-by Alex Elder <[email protected]>

---
 drivers/block/rbd.c |    6 +++---
 1 files changed, 3 insertions(+), 3 deletions(-)

drivers/block/rbd.c +3 -3

index 4bf8300..99d5920 100644
@@ -396,7 +396,10 @@ static int rbd_get_client(struct rbd_device *rbd_dev, const char *mon_addr,
 	}
 	spin_unlock(&node_lock);
 
+	mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
 	rbdc = rbd_client_create(opt, rbd_opts);
+	mutex_unlock(&ctl_mutex);
+
 	if (IS_ERR(rbdc)) {
 		ret = PTR_ERR(rbdc);
 		goto done_err;

@@ -2273,10 +2276,7 @@ static ssize_t rbd_add(struct bus_type *bus,
 	/* initialize rest of new object */
 	snprintf(rbd_dev->name, DEV_NAME_LEN, DRV_NAME "%d", rbd_dev->id);
 
-	mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
 	rc = rbd_get_client(rbd_dev, mon_dev_name, options);
-	mutex_unlock(&ctl_mutex);
-
 	if (rc < 0)
 		goto err_out_slot;

Signature · 1 line

1.7.5.4

Alex Elder

author

2012-02-29 03:45 UTC #3 lore raw

PATCH 3/4 rbd: move ctl_mutex lock inside rbd_client_create()

In reply to Alex Elder

Since rbd_client_create() is only called in one place, move the acquisition of the mutex around that call inside that function.

Signed-off-by Alex Elder <[email protected]>

---
 drivers/block/rbd.c |   11 +++++++----
 1 files changed, 7 insertions(+), 4 deletions(-)

drivers/block/rbd.c +7 -4

index 99d5920..b3c4140 100644
@@ -257,9 +257,11 @@ static struct rbd_client *rbd_client_create(struct ceph_options *opt,
 	kref_init(&rbdc->kref);
 	INIT_LIST_HEAD(&rbdc->node);
 
+	mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
+
 	rbdc->client = ceph_create_client(opt, rbdc, 0, 0);
 	if (IS_ERR(rbdc->client))
-		goto out_rbdc;
+		goto out_mutex;
 	opt = NULL; /* Now rbdc->client is responsible for opt */
 
 	ret = ceph_open_session(rbdc->client);

@@ -272,12 +274,15 @@ static struct rbd_client *rbd_client_create(struct ceph_options *opt,
 	list_add_tail(&rbdc->node, &rbd_client_list);
 	spin_unlock(&node_lock);
 
+	mutex_unlock(&ctl_mutex);
+
 	dout("rbd_client_create created %p\n", rbdc);
 	return rbdc;
 
 out_err:
 	ceph_destroy_client(rbdc->client);
-out_rbdc:
+out_mutex:
+	mutex_unlock(&ctl_mutex);
 	kfree(rbdc);
 out_opt:
 	if (opt)

@@ -396,9 +401,7 @@ static int rbd_get_client(struct rbd_device *rbd_dev, const char *mon_addr,
 	}
 	spin_unlock(&node_lock);
 
-	mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
 	rbdc = rbd_client_create(opt, rbd_opts);
-	mutex_unlock(&ctl_mutex);
 
 	if (IS_ERR(rbdc)) {
 		ret = PTR_ERR(rbdc);

Signature · 1 line

1.7.5.4

Alex Elder

author

2012-02-29 03:45 UTC #4 lore raw

PATCH 4/4 rbd: rename "node_lock"

In reply to Alex Elder

The spinlock used to protect rbd_client_list is named "node_lock". Rename it to "rbd_client_list_lock" to make it more obvious what it's for.

Signed-off-by Alex Elder <[email protected]>

---
 drivers/block/rbd.c |   20 ++++++++++----------
 1 files changed, 10 insertions(+), 10 deletions(-)

drivers/block/rbd.c +10 ~~-10~~

index b3c4140..f6dba36 100644
@@ -179,8 +179,8 @@ static DEFINE_MUTEX(ctl_mutex);	  /* Serialize open/close/setup/teardown */
 static LIST_HEAD(rbd_dev_list);    /* devices */
 static DEFINE_SPINLOCK(rbd_dev_list_lock);
 
-static LIST_HEAD(rbd_client_list);      /* clients */
-static DEFINE_SPINLOCK(node_lock);      /* protects client get/put */
+static LIST_HEAD(rbd_client_list);      	/* clients */
+static DEFINE_SPINLOCK(rbd_client_list_lock);
 
 static int __rbd_init_snaps_header(struct rbd_device *rbd_dev);
 static void rbd_dev_release(struct device *dev);

@@ -270,9 +270,9 @@ static struct rbd_client *rbd_client_create(struct ceph_options *opt,
 
 	rbdc->rbd_opts = rbd_opts;
 
-	spin_lock(&node_lock);
+	spin_lock(&rbd_client_list_lock);
 	list_add_tail(&rbdc->node, &rbd_client_list);
-	spin_unlock(&node_lock);
+	spin_unlock(&rbd_client_list_lock);
 
 	mutex_unlock(&ctl_mutex);

@@ -385,12 +385,12 @@ static int rbd_get_client(struct rbd_device *rbd_dev, const char *mon_addr,
 		goto done_err;
 	}
 
-	spin_lock(&node_lock);
+	spin_lock(&rbd_client_list_lock);
 	rbdc = __rbd_client_find(opt);
 	if (rbdc) {
 		/* using an existing client */
 		kref_get(&rbdc->kref);
-		spin_unlock(&node_lock);
+		spin_unlock(&rbd_client_list_lock);
 
 		rbd_dev->rbd_client = rbdc;

@@ -399,7 +399,7 @@ static int rbd_get_client(struct rbd_device *rbd_dev, const char *mon_addr,
 
 		return 0;
 	}
-	spin_unlock(&node_lock);
+	spin_unlock(&rbd_client_list_lock);
 
 	rbdc = rbd_client_create(opt, rbd_opts);

@@ -418,7 +418,7 @@ done_err:
 /*
  * Destroy ceph client
  *
- * Caller must hold node_lock.
+ * Caller must hold rbd_client_list_lock.
  */
 static void rbd_client_release(struct kref *kref)
 {

@@ -438,9 +438,9 @@ static void rbd_client_release(struct kref *kref)
  */
 static void rbd_put_client(struct rbd_device *rbd_dev)
 {
-	spin_lock(&node_lock);
+	spin_lock(&rbd_client_list_lock);
 	kref_put(&rbd_dev->rbd_client->kref, rbd_client_release);
-	spin_unlock(&node_lock);
+	spin_unlock(&rbd_client_list_lock);
 	rbd_dev->rbd_client = NULL;
 }

Signature · 1 line

1.7.5.4

Sage Weil

Reviewed-by on the v1 series

2012-03-02 19:56 UTC #5 lore raw

In reply to Alex Elder

Looks good,

Reviewed-by Sage Weil <[email protected]>

Quoted 11 lines · Alex Elder

This series reduces the window during which the client list lock is held, gives it a more meaningful name, and moves the locking calls closer to the places they're really needed.
-Alex
--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html