Skip to contentSkip to conversation

rbd: add ioctl for rbd

PATCH New Guangliang Zhao 4 participants 7 messages Sep 24, 2013 – Oct 8, 2013

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH v4 1/1 rbd: add ioctl for rbd

When running the following commands:

    [root@ceph0 mnt]# blockdev --setro /dev/rbd1
    [root@ceph0 mnt]# blockdev --getro /dev/rbd1
    0

The block setro didn't take effect, it is because the rbd doesn't support ioctl of block driver.

This resolves:

	http://tracker.ceph.com/issues/6265
---
 drivers/block/rbd.c |   62 +++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 60 insertions(+), 2 deletions(-)
drivers/block/rbd.c +60 -2
index 2f00778..34bcdb7 100644
@@ -508,10 +508,69 @@ static void rbd_release(struct gendisk *disk, fmode_t mode)
 	put_device(&rbd_dev->dev);
 }
 
+static int rbd_ioctl_set_ro(struct rbd_device *rbd_dev, unsigned long arg)
+{
+	int val;
+	bool ro;
+
+	if (get_user(val, (int __user *)(arg)))
+		return -EFAULT;
+
+	ro = val ? true : false;
+	/* Snapshot doesn't allow to write*/
+	if (rbd_dev->spec->snap_id != CEPH_NOSNAP && !ro)
+		return -EROFS;
+
+	if (rbd_dev->mapping.read_only != ro) {
+		rbd_dev->mapping.read_only = ro;
+		set_disk_ro(rbd_dev->disk, ro ? 1 : 0);
+	}
+
+	return 0;
+}
+
+static int rbd_ioctl(struct block_device *bdev, fmode_t mode,
+			unsigned int cmd, unsigned long arg)
+{
+	struct rbd_device *rbd_dev = bdev->bd_disk->private_data;
+	int ret = 0;
+
+	spin_lock_irq(&rbd_dev->lock);
+	/* prevent others open this device */
+	if (rbd_dev->open_count > 1) {
+		ret = -EBUSY;
+		goto out;
+	}
+
+	switch (cmd) {
+	case BLKROSET:
+		ret = rbd_ioctl_set_ro(rbd_dev, arg);
+		break;
+	default:
+		ret = -ENOTTY;
+	}
+
+out:
+	spin_unlock_irq(&rbd_dev->lock);
+	return ret;
+}
+
+#ifdef CONFIG_COMPAT
+static int rbd_compat_ioctl(struct block_device *bdev, fmode_t mode,
+				unsigned int cmd, unsigned long arg)
+{
+	return rbd_ioctl(bdev, mode, cmd, arg);
+}
+#endif /* CONFIG_COMPAT */
+
 static const struct block_device_operations rbd_bd_ops = {
 	.owner			= THIS_MODULE,
 	.open			= rbd_open,
 	.release		= rbd_release,
+	.ioctl			= rbd_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl		= rbd_compat_ioctl,
+#endif
 };
 
 /*
@@ -3027,7 +3086,6 @@ static void rbd_request_fn(struct request_queue *q)
 		__releases(q->queue_lock) __acquires(q->queue_lock)
 {
 	struct rbd_device *rbd_dev = q->queuedata;
-	bool read_only = rbd_dev->mapping.read_only;
 	struct request *rq;
 	int result;
 
@@ -3063,7 +3121,7 @@ static void rbd_request_fn(struct request_queue *q)
 
 		if (write_request) {
 			result = -EROFS;
-			if (read_only)
+			if (rbd_dev->mapping.read_only)
 				goto end_request;
 			rbd_assert(rbd_dev->spec->snap_id == CEPH_NOSNAP);
 		}
Signature · 1 line
1.7.9.5
#1 lore raw

cover clean up read only mode

Running with lockdep showed that there were problems holding rbd_dev->lock while doing in the BLKROSET ioctl patch. Fix this, and clean up the initial read-only setting to occur during device initialization instead of every time the device is opened.

These are also available in the wip-rbd-ro branch of ceph-client.git.

Josh Durgin (2):
rbd: move calls that may sleep out of spin lock range
rbd: only set disk to read-only once

 drivers/block/rbd.c |   38 ++++++++++++++++++++++++--------------
 1 files changed, 24 insertions(+), 14 deletions(-)
Signature · 1 line
1.7.2.5
#2 lore raw

Re: [PATCH 0/2] clean up read only mode

Quoted 10 lines · Josh Durgin

Running with lockdep showed that there were problems holding rbd_dev->lock while doing in the BLKROSET ioctl patch. Fix this, and clean up the initial read-only setting to occur during device initialization instead of every time the device is opened.

These are also available in the wip-rbd-ro branch of ceph-client.git.

Josh Durgin (2):
rbd: move calls that may sleep out of spin lock range
rbd: only set disk to read-only once

Both look good.

Quoted 11 lines
 drivers/block/rbd.c |   38 ++++++++++++++++++++++++--------------
 1 files changed, 24 insertions(+), 14 deletions(-)
Signature · 6 lines
1.7.2.5

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Signature · 2 lines
Best regards,
Guangliang
#3 lore raw

PATCH 1/2 rbd: move calls that may sleep out of spin lock range

In reply to Guangliang Zhao

get_user() and set_disk_ro() may allocate memory, leading to a potential deadlock if theye are called while a spin lock is held.

Move the acquisition and release of rbd_dev->lock from rbd_ioctl() into rbd_ioctl_set_ro(), so it can occur between get_user() and set_disk_ro().

---
 drivers/block/rbd.c |   36 +++++++++++++++++++++++-------------
 1 files changed, 23 insertions(+), 13 deletions(-)
drivers/block/rbd.c +23 -13
index 34bcdb7..b3b1b57 100644
@@ -510,23 +510,42 @@ static void rbd_release(struct gendisk *disk, fmode_t mode)
 
 static int rbd_ioctl_set_ro(struct rbd_device *rbd_dev, unsigned long arg)
 {
+	int ret = 0;
 	int val;
 	bool ro;
+	bool ro_changed = false;
 
+	/* get_user() may sleep, so call it before taking rbd_dev->lock */
 	if (get_user(val, (int __user *)(arg)))
 		return -EFAULT;
 
+	spin_lock_irq(&rbd_dev->lock);
+	/* prevent others open this device */
+	if (rbd_dev->open_count > 1) {
+		ret = -EBUSY;
+		goto out;
+	}
+
 	ro = val ? true : false;
 	/* Snapshot doesn't allow to write*/
-	if (rbd_dev->spec->snap_id != CEPH_NOSNAP && !ro)
-		return -EROFS;
+	if (rbd_dev->spec->snap_id != CEPH_NOSNAP && !ro) {
+		ret = -EROFS;
+		goto out;
+	}
 
 	if (rbd_dev->mapping.read_only != ro) {
 		rbd_dev->mapping.read_only = ro;
-		set_disk_ro(rbd_dev->disk, ro ? 1 : 0);
+		ro_changed = true;
 	}
 
-	return 0;
+out:
+	spin_unlock_irq(&rbd_dev->lock);
+	/* set_disk_ro() may sleep, so call it after releasing rbd_dev->lock */
+	if (ret == 0 && ro_changed)
+		set_disk_ro(rbd_dev->disk, ro ? 1 : 0);
+
+
+	return ret;
 }
 
 static int rbd_ioctl(struct block_device *bdev, fmode_t mode,
@@ -535,13 +554,6 @@ static int rbd_ioctl(struct block_device *bdev, fmode_t mode,
 	struct rbd_device *rbd_dev = bdev->bd_disk->private_data;
 	int ret = 0;
 
-	spin_lock_irq(&rbd_dev->lock);
-	/* prevent others open this device */
-	if (rbd_dev->open_count > 1) {
-		ret = -EBUSY;
-		goto out;
-	}
-
 	switch (cmd) {
 	case BLKROSET:
 		ret = rbd_ioctl_set_ro(rbd_dev, arg);
@@ -550,8 +562,6 @@ static int rbd_ioctl(struct block_device *bdev, fmode_t mode,
 		ret = -ENOTTY;
 	}
 
-out:
-	spin_unlock_irq(&rbd_dev->lock);
 	return ret;
 }
 
Signature · 1 line
1.7.2.5
Reviewed-by on v1 1/2
#4 lore raw

Re: [PATCH 1/2] rbd: move calls that may sleep out of spin lock range

Quoted 6 lines · Josh Durgin

get_user() and set_disk_ro() may allocate memory, leading to a potential deadlock if theye are called while a spin lock is held.

Move the acquisition and release of rbd_dev->lock from rbd_ioctl() into rbd_ioctl_set_ro(), so it can occur between get_user() and set_disk_ro().

This fix looks good. I have a couple small comments to consider.

Quoted 18 lines
---
 drivers/block/rbd.c |   36 +++++++++++++++++++++++-------------
 1 files changed, 23 insertions(+), 13 deletions(-)
drivers/block/rbd.c +2 -0
index 34bcdb7..b3b1b57 100644
@@ -510,23 +510,42 @@ static void rbd_release(struct gendisk *disk, fmode_t mode)
 
 static int rbd_ioctl_set_ro(struct rbd_device *rbd_dev, unsigned long arg)
 {
+	int ret = 0;
 	int val;
 	bool ro;
+	bool ro_changed = false;
 

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 527

Quoted 10 lines
drivers/block/rbd.c +7 -0
+	/* get_user() may sleep, so call it before taking rbd_dev->lock */
 	if (get_user(val, (int __user *)(arg)))
 		return -EFAULT;
 
+	spin_lock_irq(&rbd_dev->lock);
+	/* prevent others open this device */
+	if (rbd_dev->open_count > 1) {
+		ret = -EBUSY;
+		goto out;
+	}

I like to do as little as possible inside spinlock protection.

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 529

Quoted 2 lines
drivers/block/rbd.c +1 -0
+
 	ro = val ? true : false;

This assignment can be made outside the lock.

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 534

Quoted 7 lines
drivers/block/rbd.c +4 -2
 	/* Snapshot doesn't allow to write*/
-	if (rbd_dev->spec->snap_id != CEPH_NOSNAP && !ro)
-		return -EROFS;
+	if (rbd_dev->spec->snap_id != CEPH_NOSNAP && !ro) {
+		ret = -EROFS;
+		goto out;
+	}

This check can be too.

Quoted 4 lines
drivers/block/rbd.c +0 -1
 	if (rbd_dev->mapping.read_only != ro) {
 		rbd_dev->mapping.read_only = ro;
-		set_disk_ro(rbd_dev->disk, ro ? 1 : 0);

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 546

Quoted 10 lines
drivers/block/rbd.c +7 -1
+		ro_changed = true;
 	}
 
-	return 0;
+out:
+	spin_unlock_irq(&rbd_dev->lock);
+	/* set_disk_ro() may sleep, so call it after releasing rbd_dev->lock */
+	if (ret == 0 && ro_changed)
+		set_disk_ro(rbd_dev->disk, ro ? 1 : 0);
+

I like white space a lot, but this is one too many.

Quoted 29 lines
drivers/block/rbd.c +2 -9
+
+	return ret;
 }
 
 static int rbd_ioctl(struct block_device *bdev, fmode_t mode,
@@ -535,13 +554,6 @@ static int rbd_ioctl(struct block_device *bdev, fmode_t mode,
 	struct rbd_device *rbd_dev = bdev->bd_disk->private_data;
 	int ret = 0;
 
-	spin_lock_irq(&rbd_dev->lock);
-	/* prevent others open this device */
-	if (rbd_dev->open_count > 1) {
-		ret = -EBUSY;
-		goto out;
-	}
-
 	switch (cmd) {
 	case BLKROSET:
 		ret = rbd_ioctl_set_ro(rbd_dev, arg);
@@ -550,8 +562,6 @@ static int rbd_ioctl(struct block_device *bdev, fmode_t mode,
 		ret = -ENOTTY;
 	}
 
-out:
-	spin_unlock_irq(&rbd_dev->lock);
 	return ret;
 }
 
#5 lore raw

PATCH 2/2 rbd: only set disk to read-only once

In reply to Guangliang Zhao

rbd_open(), called every time the device is opened, calls set_device_ro(). There's no reason to set the device read-only or read-write every time it is opened. Just do this once during device setup, using set_disk_ro() instead because the struct block_device isn't available to us there.

---
 drivers/block/rbd.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
drivers/block/rbd.c +1 -1
index b3b1b57..fc3ebd9 100644
@@ -490,7 +490,6 @@ static int rbd_open(struct block_device *bdev, fmode_t mode)
 		return -ENOENT;
 
 	(void) get_device(&rbd_dev->dev);
-	set_device_ro(bdev, rbd_dev->mapping.read_only);
 
 	return 0;
 }
@@ -4949,6 +4948,7 @@ static int rbd_dev_device_setup(struct rbd_device *rbd_dev)
 	if (ret)
 		goto err_out_disk;
 	set_capacity(rbd_dev->disk, rbd_dev->mapping.size / SECTOR_SIZE);
+	set_disk_ro(rbd_dev->disk, rbd_dev->mapping.read_only);
 
 	ret = rbd_bus_add_dev(rbd_dev);
 	if (ret)
Signature · 1 line
1.7.2.5
Reviewed-by on v1 2/2
#6 lore raw

Re: [PATCH 2/2] rbd: only set disk to read-only once

Quoted 5 lines · Josh Durgin

rbd_open(), called every time the device is opened, calls set_device_ro(). There's no reason to set the device read-only or read-write every time it is opened. Just do this once during device setup, using set_disk_ro() instead because the struct block_device isn't available to us there.

Looks good; set_disk_ro() sort of makes more sense anyway without any partitions.

Quoted 26 lines
---
 drivers/block/rbd.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)
drivers/block/rbd.c +1 -1
index b3b1b57..fc3ebd9 100644
@@ -490,7 +490,6 @@ static int rbd_open(struct block_device *bdev, fmode_t mode)
 		return -ENOENT;
 
 	(void) get_device(&rbd_dev->dev);
-	set_device_ro(bdev, rbd_dev->mapping.read_only);
 
 	return 0;
 }
@@ -4949,6 +4948,7 @@ static int rbd_dev_device_setup(struct rbd_device *rbd_dev)
 	if (ret)
 		goto err_out_disk;
 	set_capacity(rbd_dev->disk, rbd_dev->mapping.size / SECTOR_SIZE);
+	set_disk_ro(rbd_dev->disk, rbd_dev->mapping.read_only);
 
 	ret = rbd_bus_add_dev(rbd_dev);
 	if (ret)