Skip to contentSkip to conversation

rbd: add ioctl for rbd

PATCH Superseded Guangliang Zhao 4 participants 7 messages Sep 17, 2013 – Sep 18, 2013

Subsystems (2)

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 rbd: add ioctl for rbd

When running the following commands:

    [root@ceph0 mnt]# blockdev --setro /dev/rbd2
    [root@ceph0 mnt]# blockdev --getro /dev/rbd2
    0

The block setro didn't take effect, it is because the rbd doesn't support ioctl of block driver.

The results with this patch:

root@ceph-client01:~# rbd map img
root@ceph-client01:~# blockdev --getro /dev/rbd1
0
root@ceph-client01:~# cat /sys/block/rbd1/ro
0
root@ceph-client01:~# blockdev --setro /dev/rbd1
root@ceph-client01:~# blockdev --getro /dev/rbd1
1
root@ceph-client01:~# cat /sys/block/rbd1/ro
1
root@ceph-client01:~# dd if=/dev/zero of=/dev/rbd1 count=1
dd: opening `/dev/rbd1': Read-only file system`
root@ceph-client01:~# blockdev --setrw /dev/rbd1
root@ceph-client01:~# blockdev --getro /dev/rbd1
0
root@ceph-client01:~# cat /sys/block/rbd1/ro
0
root@ceph-client01:~# dd if=/dev/zero of=/dev/rbd1 count=1
1+0 records in
1+0 records out
512 bytes (512 B) copied, 0.14989 s, 3.4 kB/s

This resolves:

	http://tracker.ceph.com/issues/6265
---
 drivers/block/rbd.c |   55 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)
drivers/block/rbd.c +55 -0
index 2f00778..f700f7c 100644
@@ -508,10 +508,65 @@ static void rbd_release(struct gendisk *disk, fmode_t mode)
 	put_device(&rbd_dev->dev);
 }
 
+static int rbd_ioctl(struct block_device *bdev, fmode_t mode,
+			unsigned int cmd, unsigned long arg)
+{
+	struct rbd_device *rbd_dev = bdev->bd_disk->private_data;
+	int ro, ret = 0;
+
+	BUG_ON(!rbd_dev);
+	spin_lock_irq(&rbd_dev->lock);
+	if (rbd_dev->open_count > 1) {
+		spin_unlock_irq(&rbd_dev->lock);
+		ret = -EBUSY;
+		goto out;
+	}
+	spin_unlock_irq(&rbd_dev->lock);
+
+	switch (cmd) {
+	case BLKROSET:
+		if (get_user(ro, (int __user *)(arg))) {
+			ret = -EFAULT;
+			goto out;
+		}
+
+		/* Snapshot doesn't allow to write*/
+		if (rbd_dev->spec->snap_id != CEPH_NOSNAP && ro) {
+			ret = -EROFS;
+			goto out;
+		}
+
+		if (rbd_dev->mapping.read_only != ro) {
+			rbd_dev->mapping.read_only = ro;
+			set_disk_ro(rbd_dev->disk, ro);
+			goto out;
+		}
+
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+out:
+	return ret;
+}
+
+#ifdef CONFIG_COMPAT
+static int rbd_compat_ioctl(struct block_device *bdev, fmode_t mode,
+				unsigned int cmd, unsigned long arg)
+{
+	return rbd_ioctl(bdev, mode, cmd, arg);
+}
+#endif /* CONFIG_COMPAT */
+
 static const struct block_device_operations rbd_bd_ops = {
 	.owner			= THIS_MODULE,
 	.open			= rbd_open,
 	.release		= rbd_release,
+	.ioctl			= rbd_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl		= rbd_compat_ioctl,
+#endif
 };
 
 /*
Signature · 1 line
1.7.9.5
#1 lore raw
Quoted 4 lines · Guangliang Zhao

When running the following commands:

    [root@ceph0 mnt]# blockdev --setro /dev/rbd2
    [root@ceph0 mnt]# blockdev --getro /dev/rbd2
    0

I think this is a good change to make, and I think what you've done is overall fine.

I do see one bug though.

And despite my statement that "it's fine," I have a lot of input and suggestions for you to consider, below.

Quoted 17 lines

The block setro didn't take effect, it is because the rbd doesn't support ioctl of block driver.

The results with this patch:

root@ceph-client01:~# rbd map img
root@ceph-client01:~# blockdev --getro /dev/rbd1
0
root@ceph-client01:~# cat /sys/block/rbd1/ro
0
root@ceph-client01:~# blockdev --setro /dev/rbd1
root@ceph-client01:~# blockdev --getro /dev/rbd1
1
root@ceph-client01:~# cat /sys/block/rbd1/ro
1
root@ceph-client01:~# dd if=/dev/zero of=/dev/rbd1 count=1
dd: opening `/dev/rbd1': Read-only file system`
root@ceph-client01:~# blockdev --setrw /dev/rbd1
Quoted 8 lines
root@ceph-client01:~# blockdev --getro /dev/rbd1
0
root@ceph-client01:~# cat /sys/block/rbd1/ro
0
root@ceph-client01:~# dd if=/dev/zero of=/dev/rbd1 count=1
1+0 records in
1+0 records out
512 bytes (512 B) copied, 0.14989 s, 3.4 kB/s

As long as you're testing...

It would be good to demonstrate that it's writable after a setrw call (i.e., by actually writing to it).

And it would be good to show an attempt to change to read-write a mapped "base" image as well as a mapped snapshot image, both opened (mounted) and not.

Quoted 13 lines

This resolves:

	http://tracker.ceph.com/issues/6265
---
 drivers/block/rbd.c |   55 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)
drivers/block/rbd.c +0 -0
index 2f00778..f700f7c 100644

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 517

Quoted 10 lines
drivers/block/rbd.c +7 -0
@@ -508,10 +508,65 @@ static void rbd_release(struct gendisk *disk, fmode_t mode)
 	put_device(&rbd_dev->dev);
 }
 
+static int rbd_ioctl(struct block_device *bdev, fmode_t mode,
+			unsigned int cmd, unsigned long arg)
+{
+	struct rbd_device *rbd_dev = bdev->bd_disk->private_data;
+	int ro, ret = 0;
+
+	BUG_ON(!rbd_dev);

Use:

	rbd_assert(rbd_dev);

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 519

Quoted 2 lines
drivers/block/rbd.c +2 -0
+	spin_lock_irq(&rbd_dev->lock);
+	if (rbd_dev->open_count > 1) {

This comment is not *extremely* important, but...

I think it should be harmless (and successful) if the request to change the read-only status ends up being a no-op. That is, if it's already a read-only mapping, and this is a request to make the device read-only, it should succeed.

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 524

Quoted 5 lines
drivers/block/rbd.c +5 -0
+		spin_unlock_irq(&rbd_dev->lock);
+		ret = -EBUSY;
+		goto out;
+	}
+	spin_unlock_irq(&rbd_dev->lock);

Now that you've released the lock I think the device state could change in undesirable ways.

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 534

Quoted 9 lines
drivers/block/rbd.c +9 -0
+	switch (cmd) {
+	case BLKROSET:
+		if (get_user(ro, (int __user *)(arg))) {
+			ret = -EFAULT;
+			goto out;
+		}
+
+		/* Snapshot doesn't allow to write*/
+		if (rbd_dev->spec->snap_id != CEPH_NOSNAP && ro) {

I think the way you are interpreting the value of "ro" here is backward. If "ro" is non-zero, it's a request to *make* the device be read-only. And for a snapshot it already is. (This is the bug I referred to above.)

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 540

Quoted 6 lines
drivers/block/rbd.c +6 -0
+			ret = -EROFS;
+			goto out;
+		}
+
+		if (rbd_dev->mapping.read_only != ro) {
+			rbd_dev->mapping.read_only = ro;

I know the C standard says this int value will automatically get converted to either a Boolean true (1) or false (0), but I personally would prefer making that more explicit by applying a double unary NOT operation, i.e.:

			rbd_dev->mapping.read_only = !!ro;

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 541

Quoted 1 line
drivers/block/rbd.c +1 -0
+			set_disk_ro(rbd_dev->disk, ro);

Similarly, the int value recorded by the genhd code simply saves whatever you give it. I'd personally rather see that get passed an explicit 1 or 0 rather than whatever int value the user space caller provided. (So maybe setting the value of "ro" to 0 or 1 initially would address both of my concerns. I.e.: get_user(val, ...); ro = val ? 1 : 0;)

This is really more of an issue I have with the genhd code than what you've done.

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 545

Quoted 4 lines
drivers/block/rbd.c +4 -0
+			goto out;
+		}
+
+		break;

OK, now I have a very broad (and too detailed) suggestion. (I got a little carried away, sorry about that.)

At this point there is only one IOCTL request that is handled by this function. I know it doesn't match the general-purpose structure of your typical ioctl routine, but unless you think there are more ioctls that should be added, I think this could be made simpler by structuring the function something like:

rbd_ioctl(...)

{
	ret = 0;
	int argval;
	bool want_ro;

	if (cmd != BLKROSET)
		return -EINVAL;

	if (get_user(argval, ...))
		return -EFAULT;
	want_ro = !!argval;

	spin_lock();
	if (rbd_dev->mapping.read_only == want_ro)
		goto err_unlock;	/* No change, OK */

	if (rbd_dev->spec->snap_id != CEPH_NOSNAP && !want_ro) {
		ret = -EROFS;
		goto err_unlock;
	}

	/* Change requested; don't allow if already open */
	if (rbd_dev->open_count) {
		ret = -EBUSY;
		goto err_unlock;
	}

	rbd_dev->mapping.read_only = want_ro;
	spin_unlock();
	set_device_ro(bdev, want_ro ? 1 : 0);

	return 0;
err_unlock:
	spin_unlock();
	return ret;
}

Even if you kept the generic switch() ... structure, you could put the logic for the read-only change into a separate function.

-Alex

Quoted 28 lines
drivers/block/rbd.c +20 -0
+	default:
+		ret = -EINVAL;
+	}
+
+out:
+	return ret;
+}
+
+#ifdef CONFIG_COMPAT
+static int rbd_compat_ioctl(struct block_device *bdev, fmode_t mode,
+				unsigned int cmd, unsigned long arg)
+{
+	return rbd_ioctl(bdev, mode, cmd, arg);
+}
+#endif /* CONFIG_COMPAT */
+
 static const struct block_device_operations rbd_bd_ops = {
 	.owner			= THIS_MODULE,
 	.open			= rbd_open,
 	.release		= rbd_release,
+	.ioctl			= rbd_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl		= rbd_compat_ioctl,
+#endif
 };
 
 /*
#2 lore raw
Quoted 8 lines · Alex Elder
Quoted 4 lines · Guangliang Zhao

When running the following commands:

     [root@ceph0 mnt]# blockdev --setro /dev/rbd2
     [root@ceph0 mnt]# blockdev --getro /dev/rbd2
     0

I think this is a good change to make, and I think what you've done is overall fine.

I agree, and have a couple more comments.

Quoted 31 lines

I do see one bug though.

And despite my statement that "it's fine," I have a lot of input and suggestions for you to consider, below.

Quoted 17 lines

The block setro didn't take effect, it is because the rbd doesn't support ioctl of block driver.

The results with this patch:

root@ceph-client01:~# rbd map img
root@ceph-client01:~# blockdev --getro /dev/rbd1
0
root@ceph-client01:~# cat /sys/block/rbd1/ro
0
root@ceph-client01:~# blockdev --setro /dev/rbd1
root@ceph-client01:~# blockdev --getro /dev/rbd1
1
root@ceph-client01:~# cat /sys/block/rbd1/ro
1
root@ceph-client01:~# dd if=/dev/zero of=/dev/rbd1 count=1
dd: opening `/dev/rbd1': Read-only file system`
root@ceph-client01:~# blockdev --setrw /dev/rbd1
Quoted 8 lines
root@ceph-client01:~# blockdev --getro /dev/rbd1
0
root@ceph-client01:~# cat /sys/block/rbd1/ro
0
root@ceph-client01:~# dd if=/dev/zero of=/dev/rbd1 count=1
1+0 records in
1+0 records out
512 bytes (512 B) copied, 0.14989 s, 3.4 kB/s
Quoted 8 lines

As long as you're testing...

It would be good to demonstrate that it's writable after a setrw call (i.e., by actually writing to it).

And it would be good to show an attempt to change to read-write a mapped "base" image as well as a mapped snapshot image, both opened (mounted) and not.

Also a read-only mapped non-snapshot, which will catch a bug with this patch:

rbd_request_fn reads rbd_dev->mapping.read_only when it is first called, but once it's in the processing loop it never checks it again. This will prevent an initially read-only mapping from ever becoming read-write. The request loop needs to check for an updated rbd_dev->mapping.read_only value.

This makes me notice another cleanup, though it doesn't affect the functionality in this patch:

The place where the rbd driver calls set_device_ro() should be changed as well - it only needs to be done once, when the device is being added, not during each open() call.

Quoted 96 lines
Quoted 13 lines

This resolves:

	http://tracker.ceph.com/issues/6265
---
  drivers/block/rbd.c |   55 +++++++++++++++++++++++++++++++++++++++++++++++++++
  1 file changed, 55 insertions(+)
drivers/block/rbd.c +0 -0
index 2f00778..f700f7c 100644

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 517

Quoted 10 lines
drivers/block/rbd.c +7 -0
@@ -508,10 +508,65 @@ static void rbd_release(struct gendisk *disk, fmode_t mode)
  	put_device(&rbd_dev->dev);
  }
 
+static int rbd_ioctl(struct block_device *bdev, fmode_t mode,
+			unsigned int cmd, unsigned long arg)
+{
+	struct rbd_device *rbd_dev = bdev->bd_disk->private_data;
+	int ro, ret = 0;
+
+	BUG_ON(!rbd_dev);

Use:

	rbd_assert(rbd_dev);

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 519

Quoted 2 lines
drivers/block/rbd.c +2 -0
+	spin_lock_irq(&rbd_dev->lock);
+	if (rbd_dev->open_count > 1) {

This comment is not *extremely* important, but...

I think it should be harmless (and successful) if the request to change the read-only status ends up being a no-op. That is, if it's already a read-only mapping, and this is a request to make the device read-only, it should succeed.

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 524

Quoted 5 lines
drivers/block/rbd.c +5 -0
+		spin_unlock_irq(&rbd_dev->lock);
+		ret = -EBUSY;
+		goto out;
+	}
+	spin_unlock_irq(&rbd_dev->lock);

Now that you've released the lock I think the device state could change in undesirable ways.

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 534

Quoted 9 lines
drivers/block/rbd.c +9 -0
+	switch (cmd) {
+	case BLKROSET:
+		if (get_user(ro, (int __user *)(arg))) {
+			ret = -EFAULT;
+			goto out;
+		}
+
+		/* Snapshot doesn't allow to write*/
+		if (rbd_dev->spec->snap_id != CEPH_NOSNAP && ro) {

I think the way you are interpreting the value of "ro" here is backward. If "ro" is non-zero, it's a request to *make* the device be read-only. And for a snapshot it already is. (This is the bug I referred to above.)

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 540

Quoted 6 lines
drivers/block/rbd.c +6 -0
+			ret = -EROFS;
+			goto out;
+		}
+
+		if (rbd_dev->mapping.read_only != ro) {
+			rbd_dev->mapping.read_only = ro;

I know the C standard says this int value will automatically get converted to either a Boolean true (1) or false (0), but I personally would prefer making that more explicit by applying a double unary NOT operation, i.e.:

			rbd_dev->mapping.read_only = !!ro;

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 541

Quoted 1 line
drivers/block/rbd.c +1 -0
+			set_disk_ro(rbd_dev->disk, ro);

Similarly, the int value recorded by the genhd code simply saves whatever you give it. I'd personally rather see that get passed an explicit 1 or 0 rather than whatever int value the user space caller provided. (So maybe setting the value of "ro" to 0 or 1 initially would address both of my concerns. I.e.: get_user(val, ...); ro = val ? 1 : 0;)

This is really more of an issue I have with the genhd code than what you've done.

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 545

Quoted 4 lines
drivers/block/rbd.c +4 -0
+			goto out;
+		}
+
+		break;

OK, now I have a very broad (and too detailed) suggestion. (I got a little carried away, sorry about that.)

Quoted 5 lines

At this point there is only one IOCTL request that is handled by this function. I know it doesn't match the general-purpose structure of your typical ioctl routine, but unless you think there are more ioctls that should be added, I think this could be made simpler by structuring the function something like:

I like this refactoring of it, but there are two minor issues present in the original patch too:

Quoted 8 lines

rbd_ioctl(...)

{
	ret = 0;
	int argval;
	bool want_ro;

	if (cmd != BLKROSET)
		return -EINVAL;

According to block/ioctl.c, this should be -ENOTTY or -ENOIOCTLCOMMAND, not -EINVAL.

Quoted 15 lines
	if (get_user(argval, ...))
		return -EFAULT;
	want_ro = !!argval;

	spin_lock();
	if (rbd_dev->mapping.read_only == want_ro)
		goto err_unlock;	/* No change, OK */

	if (rbd_dev->spec->snap_id != CEPH_NOSNAP && !want_ro) {
		ret = -EROFS;
		goto err_unlock;
	}

	/* Change requested; don't allow if already open */
Quoted 8 lines
	if (rbd_dev->open_count) {
		ret = -EBUSY;
		goto err_unlock;
	}

	rbd_dev->mapping.read_only = want_ro;
	spin_unlock();
	set_device_ro(bdev, want_ro ? 1 : 0);

block/ioctl.c will already call set_device_ro() for us after this driver-specific handling completes successfully, so we don't need to call it here. Also, it appears the block layer has a bug in that it does the check for CAP_SYS_ADMIN *after* calling the driver-specific code for BLKROSET. So the driver-specific part could succeed, but the generic code could fail due to this later check without the driver knowing, possibly leaving the driver inconsistent with the block layer.

Josh

Quoted 42 lines
	return 0;
err_unlock:
	spin_unlock();
	return ret;
}

Even if you kept the generic switch() ... structure, you could put the logic for the read-only change into a separate function.

-Alex

Quoted 28 lines
drivers/block/rbd.c +20 -0
+	default:
+		ret = -EINVAL;
+	}
+
+out:
+	return ret;
+}
+
+#ifdef CONFIG_COMPAT
+static int rbd_compat_ioctl(struct block_device *bdev, fmode_t mode,
+				unsigned int cmd, unsigned long arg)
+{
+	return rbd_ioctl(bdev, mode, cmd, arg);
+}
+#endif /* CONFIG_COMPAT */
+
  static const struct block_device_operations rbd_bd_ops = {
  	.owner			= THIS_MODULE,
  	.open			= rbd_open,
  	.release		= rbd_release,
+	.ioctl			= rbd_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl		= rbd_compat_ioctl,
+#endif
  };
 
  /*
#3 lore raw

On 09/17/2013 10:24 AM, Josh Durgin wrote:

. . .

Quoted 19 lines
Quoted 9 lines

OK, now I have a very broad (and too detailed) suggestion. (I got a little carried away, sorry about that.)

At this point there is only one IOCTL request that is handled by this function. I know it doesn't match the general-purpose structure of your typical ioctl routine, but unless you think there are more ioctls that should be added, I think this could be made simpler by structuring the function something like:

I like this refactoring of it, but there are two minor issues present in the original patch too:

Quoted 6 lines

rbd_ioctl(...)

{
    ret = 0;
    int argval;
    bool want_ro;
Quoted 5 lines
Quoted 2 lines
    if (cmd != BLKROSET)
        return -EINVAL;

According to block/ioctl.c, this should be -ENOTTY or -ENOIOCTLCOMMAND, not -EINVAL.

I wondered about that, and actually looked at it but I thought I saw other ioctl functions returning -EINVAL so I stopped looking and didn't mention anything.

Quoted 23 lines
Quoted 14 lines
    if (get_user(argval, ...))
        return -EFAULT;
    want_ro = !!argval;

    spin_lock();
    if (rbd_dev->mapping.read_only == want_ro)
        goto err_unlock;    /* No change, OK */

    if (rbd_dev->spec->snap_id != CEPH_NOSNAP && !want_ro) {
        ret = -EROFS;
        goto err_unlock;
    }

    /* Change requested; don't allow if already open */
Quoted 8 lines
    if (rbd_dev->open_count) {
        ret = -EBUSY;
        goto err_unlock;
    }

    rbd_dev->mapping.read_only = want_ro;
    spin_unlock();
    set_device_ro(bdev, want_ro ? 1 : 0);
Quoted 7 lines

block/ioctl.c will already call set_device_ro() for us after this driver-specific handling completes successfully, so we don't need to call it here. Also, it appears the block layer has a bug in that it does the check for CAP_SYS_ADMIN *after* calling the driver-specific code for BLKROSET. So the driver-specific part could succeed, but the generic code could fail due to this later check without the driver knowing, possibly leaving the driver inconsistent with the block layer.

I wonder if that's intentional, but I doubt it. It predates the original Linux-2.6.12-rc2 git commit.

But I agree with you, I think it's a bug. Do you plan to submit a patch upstream to propose a fix?

-Alex

#4 lore raw
Quoted 9 lines · Alex Elder
Quoted 7 lines · Josh Durgin

block/ioctl.c will already call set_device_ro() for us after this driver-specific handling completes successfully, so we don't need to call it here. Also, it appears the block layer has a bug in that it does the check for CAP_SYS_ADMIN *after* calling the driver-specific code for BLKROSET. So the driver-specific part could succeed, but the generic code could fail due to this later check without the driver knowing, possibly leaving the driver inconsistent with the block layer.

Quoted 5 lines

I wonder if that's intentional, but I doubt it. It predates the original Linux-2.6.12-rc2 git commit.

But I agree with you, I think it's a bug. Do you plan to submit a patch upstream to propose a fix?

Sure, I'll submit a patch for it.

#5 lore raw
In reply to Josh Durgin
Quoted 10 lines · Josh Durgin
Quoted 9 lines

As long as you're testing...

It would be good to demonstrate that it's writable after a setrw call (i.e., by actually writing to it).

And it would be good to show an attempt to change to read-write a mapped "base" image as well as a mapped snapshot image, both opened (mounted) and not.

Quoted 8 lines

Also a read-only mapped non-snapshot, which will catch a bug with this patch:

rbd_request_fn reads rbd_dev->mapping.read_only when it is first called, but once it's in the processing loop it never checks it again. This will prevent an initially read-only mapping from ever becoming read-write. The request loop needs to check for an updated rbd_dev->mapping.read_only value.

You are absolutely right. The existing code assumes it won't change, basically.

Quoted 6 lines

This makes me notice another cleanup, though it doesn't affect the functionality in this patch:

The place where the rbd driver calls set_device_ro() should be changed as well - it only needs to be done once, when the device is being added, not during each open() call.

Yes, probably in rbd_dev_device_setup(), near the set_capacity() call.

Very good finds Josh.

-Alex

#6 lore raw
In reply to Josh Durgin

On Tue, Sep 17, 2013 at 08:24:17AM -0700, Josh Durgin wrote:

Hi Josh and Alex,

Thanks you very much for reviewing the patch :-)

Quoted 7 lines · Alex Elder
Quoted 6 lines · Guangliang Zhao
Quoted 4 lines

When running the following commands:

    [root@ceph0 mnt]# blockdev --setro /dev/rbd2
    [root@ceph0 mnt]# blockdev --getro /dev/rbd2
    0
Quoted 44 lines
Quoted 2 lines

I think this is a good change to make, and I think what you've done is overall fine.

I agree, and have a couple more comments.

Quoted 39 lines

I do see one bug though.

And despite my statement that "it's fine," I have a lot of input and suggestions for you to consider, below.

Quoted 17 lines

The block setro didn't take effect, it is because the rbd doesn't support ioctl of block driver.

The results with this patch:

root@ceph-client01:~# rbd map img
root@ceph-client01:~# blockdev --getro /dev/rbd1
0
root@ceph-client01:~# cat /sys/block/rbd1/ro
0
root@ceph-client01:~# blockdev --setro /dev/rbd1
root@ceph-client01:~# blockdev --getro /dev/rbd1
1
root@ceph-client01:~# cat /sys/block/rbd1/ro
1
root@ceph-client01:~# dd if=/dev/zero of=/dev/rbd1 count=1
dd: opening `/dev/rbd1': Read-only file system`
root@ceph-client01:~# blockdev --setrw /dev/rbd1
Quoted 8 lines
root@ceph-client01:~# blockdev --getro /dev/rbd1
0
root@ceph-client01:~# cat /sys/block/rbd1/ro
0
root@ceph-client01:~# dd if=/dev/zero of=/dev/rbd1 count=1
1+0 records in
1+0 records out
512 bytes (512 B) copied, 0.14989 s, 3.4 kB/s

As long as you're testing...

It would be good to demonstrate that it's writable after a setrw call (i.e., by actually writing to it).

And it would be good to show an attempt to change to read-write a mapped "base" image as well as a mapped snapshot image, both opened (mounted) and not.

If we allow user set rw when mounting(or whatever), it would bring in more complexity, see below.

Quoted 3 lines

Also a read-only mapped non-snapshot, which will catch a bug with this patch:

Yes, but I think that's would be OK if I release the lock before return, even rbd_request_fn read the rbd_dev->mapping.read_only only the first time.

Others couldn't open the device when I calling ioctl, also see below.

Quoted 7 lines

rbd_request_fn reads rbd_dev->mapping.read_only when it is first called, but once it's in the processing loop it never checks it again. This will prevent an initially read-only mapping from ever becoming read-write. The request loop needs to check for an updated rbd_dev->mapping.read_only value.

Quoted 45 lines

This makes me notice another cleanup, though it doesn't affect the functionality in this patch:

The place where the rbd driver calls set_device_ro() should be changed as well - it only needs to be done once, when the device is being added,
not during each open() call.

Quoted 37 lines
Quoted 13 lines

This resolves:

	http://tracker.ceph.com/issues/6265
---
 drivers/block/rbd.c |   55 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)
drivers/block/rbd.c +0 -0
index 2f00778..f700f7c 100644

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 517

Quoted 10 lines
drivers/block/rbd.c +7 -0
@@ -508,10 +508,65 @@ static void rbd_release(struct gendisk *disk, fmode_t mode)
 	put_device(&rbd_dev->dev);
 }
 
+static int rbd_ioctl(struct block_device *bdev, fmode_t mode,
+			unsigned int cmd, unsigned long arg)
+{
+	struct rbd_device *rbd_dev = bdev->bd_disk->private_data;
+	int ro, ret = 0;
+
+	BUG_ON(!rbd_dev);

Use:

	rbd_assert(rbd_dev);

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 519

Quoted 2 lines
drivers/block/rbd.c +2 -0
+	spin_lock_irq(&rbd_dev->lock);
+	if (rbd_dev->open_count > 1) {

This comment is not *extremely* important, but...

I think it should be harmless (and successful) if the request to change the read-only status ends up being a no-op. That is, if it's already a read-only mapping, and this is a request to make the device read-only, it should succeed.

Sorry for didn't comment here. I found that many comments relate this line.

If we allow user set rw when mounting, or even others also calling ioctl, there may be unknown and obscure concurrent issues, such as the bug what Josh mentioned above.

I need release the lock before return, but anyway, keeping only one open the device when setting rw would be more simple and safe.

Quoted 9 lines
Quoted 9 lines

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 524

Quoted 6 lines
drivers/block/rbd.c +5 -0
+		spin_unlock_irq(&rbd_dev->lock);
+		ret = -EBUSY;
+		goto out;
+	}
+	spin_unlock_irq(&rbd_dev->lock);

Now that you've released the lock I think the device state could change in undesirable ways.

Fully accept.

Quoted 15 lines
Quoted 15 lines

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 534

Quoted 10 lines
drivers/block/rbd.c +9 -0
+	switch (cmd) {
+	case BLKROSET:
+		if (get_user(ro, (int __user *)(arg))) {
+			ret = -EFAULT;
+			goto out;
+		}
+
+		/* Snapshot doesn't allow to write*/
+		if (rbd_dev->spec->snap_id != CEPH_NOSNAP && ro) {

I think the way you are interpreting the value of "ro" here is backward. If "ro" is non-zero, it's a request to *make* the device be read-only. And for a snapshot it already is. (This is the bug I referred to above.)

Yes, it's a bug.

Quoted 13 lines
Quoted 13 lines

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 540

Quoted 7 lines
drivers/block/rbd.c +6 -0
+			ret = -EROFS;
+			goto out;
+		}
+
+		if (rbd_dev->mapping.read_only != ro) {
+			rbd_dev->mapping.read_only = ro;

I know the C standard says this int value will automatically get converted to either a Boolean true (1) or false (0), but I personally would prefer making that more explicit by applying a double unary NOT operation, i.e.:

Quoted 1 line
Quoted 1 line
			rbd_dev->mapping.read_only = !!ro;

Fully accept.

Quoted 36 lines
Quoted 21 lines

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 541

Quoted 2 lines
drivers/block/rbd.c +1 -0
+			set_disk_ro(rbd_dev->disk, ro);

Similarly, the int value recorded by the genhd code simply saves whatever you give it. I'd personally rather see that get passed an explicit 1 or 0 rather than whatever int value the user space caller provided. (So maybe setting the value of "ro" to 0 or 1 initially would address both of my concerns. I.e.: get_user(val, ...); ro = val ? 1 : 0;)

This is really more of an issue I have with the genhd code than what you've done.

drivers/block/rbd.c static void rbd_release(struct gendisk *disk, fmode_t mode) · line 545

Quoted 4 lines
drivers/block/rbd.c +4 -0
+			goto out;
+		}
+
+		break;

OK, now I have a very broad (and too detailed) suggestion. (I got a little carried away, sorry about that.)

Quoted 5 lines

At this point there is only one IOCTL request that is handled by this function. I know it doesn't match the general-purpose structure of your typical ioctl routine, but unless you think there are more ioctls that should be added, I think this could be made simpler by structuring the function something like:

I like this refactoring of it, but there are two minor issues present in the original patch too:

Quoted 6 lines

rbd_ioctl(...)

{
	ret = 0;
	int argval;
	bool want_ro;
Quoted 22 lines
Quoted 2 lines
	if (cmd != BLKROSET)
		return -EINVAL;

According to block/ioctl.c, this should be -ENOTTY or -ENOIOCTLCOMMAND, not -EINVAL.

Quoted 16 lines
	if (get_user(argval, ...))
		return -EFAULT;
	want_ro = !!argval;

	spin_lock();
	if (rbd_dev->mapping.read_only == want_ro)
		goto err_unlock;	/* No change, OK */

	if (rbd_dev->spec->snap_id != CEPH_NOSNAP && !want_ro) {
		ret = -EROFS;
		goto err_unlock;
	}

	/* Change requested; don't allow if already open */
	if (rbd_dev->open_count) {

The rbd_dev->open_count would be 1 at least "here".

Quoted 8 lines
Quoted 7 lines
		ret = -EBUSY;
		goto err_unlock;
	}

	rbd_dev->mapping.read_only = want_ro;
	spin_unlock();
	set_device_ro(bdev, want_ro ? 1 : 0);
Quoted 7 lines

block/ioctl.c will already call set_device_ro() for us after this driver-specific handling completes successfully, so we don't need to call it here. Also, it appears the block layer has a bug in that it does the check for CAP_SYS_ADMIN *after* calling the driver-specific code for BLKROSET. So the driver-specific part could succeed, but the generic code could fail due to this later check without the driver knowing, possibly leaving the driver inconsistent with the block layer.

Good catch!

Quoted 13 lines

Josh

Quoted 10 lines
	return 0;
err_unlock:
	spin_unlock();
	return ret;
}

Even if you kept the generic switch() ... structure, you could put the logic for the read-only change into a separate function.

OK.

Quoted 33 lines
Quoted 32 lines

-Alex

Quoted 28 lines
drivers/block/rbd.c +20 -0
+	default:
+		ret = -EINVAL;
+	}
+
+out:
+	return ret;
+}
+
+#ifdef CONFIG_COMPAT
+static int rbd_compat_ioctl(struct block_device *bdev, fmode_t mode,
+				unsigned int cmd, unsigned long arg)
+{
+	return rbd_ioctl(bdev, mode, cmd, arg);
+}
+#endif /* CONFIG_COMPAT */
+
 static const struct block_device_operations rbd_bd_ops = {
 	.owner			= THIS_MODULE,
 	.open			= rbd_open,
 	.release		= rbd_release,
+	.ioctl			= rbd_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl		= rbd_compat_ioctl,
+#endif
 };
 
 /*
Signature · 2 lines
Best regards,
Guangliang