rbd: version 2 parent probing

We will know the image id for format 2 parent images, but won't initially know its image name. Avoid making the query for an image id in rbd_dev_image_id() if it's already known.

Signed-off-by Alex Elder <[email protected]>

---
  drivers/block/rbd.c |    8 ++++++++
  1 file changed, 8 insertions(+)

drivers/block/rbd.c +8 -0

index 8d26c0f..a852133 100644
@@ -3068,6 +3068,14 @@ static int rbd_dev_image_id(struct rbd_device
 *rbd_dev)
  	void *p;
 
  	/*
+	 * When probing a parent image, the image id is already
+	 * known (and the image name likely is not).  There's no
+	 * need to fetch the image id again in this case.
+	 */
+	if (rbd_dev->spec->image_id)
+		return 0;
+
+	/*
  	 * First, see if the format 2 image id file exists, and if
  	 * so, get the image's persistent id from it.

*/

Format 2 parent images are partially identified by their image id, but it may not be possible to determine their image name. The name is not strictly needed for correct operation, so we won't be treating it as an error if we don't know it. Handle this case gracefully in rbd_name_show().

Signed-off-by Alex Elder <[email protected]>

---
  drivers/block/rbd.c |    5 ++++-
  1 file changed, 4 insertions(+), 1 deletion(-)

drivers/block/rbd.c +4 -1

index a852133..28052ff 100644
@@ -1982,7 +1982,10 @@ static ssize_t rbd_name_show(struct device *dev,
  {
  	struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
 
-	return sprintf(buf, "%s\n", rbd_dev->spec->image_name);
+	if (rbd_dev->spec->image_name)
+		return sprintf(buf, "%s\n", rbd_dev->spec->image_name);
+
+	return sprintf(buf, "(unknown)\n");
  }
 
  static ssize_t rbd_image_id_show(struct device *dev,

Add support for getting the the information identifying the parent image for rbd images that have them. The child image holds a reference to its parent image specification structure. Create a new entry "parent" in /sys/bus/rbd/image/N/ to report the identifying information for the parent image, if any.

Signed-off-by Alex Elder <[email protected]>

---
  Documentation/ABI/testing/sysfs-bus-rbd |    4 +
  drivers/block/rbd.c                     |  131
+++++++++++++++++++++++++++++++
  include/linux/ceph/rados.h              |    2 +
  3 files changed, 137 insertions(+)

Documentation/ABI/testing/sysfs-bus-rbd +4 -0

index 1cf2adf..cd9213c 100644
@@ -70,6 +70,10 @@ snap_*
 
  	A directory per each snapshot
 
+parent
+
+	Information identifying the pool, image, and snapshot id for
+	the parent image in a layered rbd image (format 2 only).
 
  Entries under /sys/bus/rbd/devices/<dev-id>/snap_<snap-name>
  -------------------------------------------------------------

drivers/block/rbd.c +47 -0

index 28052ff..bce1fcf 100644
@@ -217,6 +217,9 @@ struct rbd_device {
  	struct ceph_osd_event   *watch_event;
  	struct ceph_osd_request *watch_request;
 
+	struct rbd_spec		*parent_spec;
+	u64			parent_overlap;
+
  	/* protects updating the header */
  	struct rw_semaphore     header_rwsem;
 

@@ -2009,6 +2012,49 @@ static ssize_t rbd_snap_show(struct device *dev,
  	return sprintf(buf, "%s\n", rbd_dev->spec->snap_name);
  }
 
+/*
+ * For an rbd v2 image, shows the pool id, image id, and snapshot id
+ * for the parent image.  If there is no parent, simply shows
+ * "(no parent image)".
+ */
+static ssize_t rbd_parent_show(struct device *dev,
+			     struct device_attribute *attr,
+			     char *buf)
+{
+	struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
+	struct rbd_spec *spec = rbd_dev->parent_spec;
+	int count;
+	char *bufp = buf;
+
+	if (!spec)
+		return sprintf(buf, "(no parent image)\n");
+
+	count = sprintf(bufp, "pool_id %llu\npool_name %s\n",
+			(unsigned long long) spec->pool_id, spec->pool_name);
+	if (count < 0)
+		return count;
+	bufp += count;
+
+	count = sprintf(bufp, "image_id %s\nimage_name %s\n", spec->image_id,
+			spec->image_name ? spec->image_name : "(unknown)");
+	if (count < 0)
+		return count;
+	bufp += count;
+
+	count = sprintf(bufp, "snap_id %llu\nsnap_name %s\n",
+			(unsigned long long) spec->snap_id, spec->snap_name);
+	if (count < 0)
+		return count;
+	bufp += count;
+
+	count = sprintf(bufp, "overlap %llu\n", rbd_dev->parent_overlap);
+	if (count < 0)
+		return count;
+	bufp += count;
+
+	return (ssize_t) (bufp - buf);
+}
+
  static ssize_t rbd_image_refresh(struct device *dev,
  				 struct device_attribute *attr,
  				 const char *buf,

@@ -2032,6 +2078,7 @@ static DEVICE_ATTR(name, S_IRUGO, rbd_name_show,
 NULL);
  static DEVICE_ATTR(image_id, S_IRUGO, rbd_image_id_show, NULL);
  static DEVICE_ATTR(refresh, S_IWUSR, NULL, rbd_image_refresh);
  static DEVICE_ATTR(current_snap, S_IRUGO, rbd_snap_show, NULL);
+static DEVICE_ATTR(parent, S_IRUGO, rbd_parent_show, NULL);
 
  static struct attribute *rbd_attrs[] = {

&dev_attr_size.attr,

drivers/block/rbd.c +2 -0

@@ -2043,6 +2090,7 @@ static struct attribute *rbd_attrs[] = {
  	&dev_attr_name.attr,
  	&dev_attr_image_id.attr,
  	&dev_attr_current_snap.attr,
+	&dev_attr_parent.attr,
  	&dev_attr_refresh.attr,
  	NULL
  };

@@ -2192,6 +2240,7 @@ struct rbd_device *rbd_dev_create(struct
 rbd_client *rbdc,
 
  static void rbd_dev_destroy(struct rbd_device *rbd_dev)
  {
+	rbd_spec_put(rbd_dev->parent_spec);
  	kfree(rbd_dev->header_name);
  	rbd_put_client(rbd_dev->rbd_client);

  	rbd_spec_put(rbd_dev->spec);

drivers/block/rbd.c +65 -0

@@ -2400,6 +2449,71 @@ static int rbd_dev_v2_features(struct rbd_device
 *rbd_dev)
  						&rbd_dev->header.features);
  }
 
+static int rbd_dev_v2_parent_info(struct rbd_device *rbd_dev)
+{
+	struct rbd_spec *parent_spec;
+	size_t size;
+	void *reply_buf = NULL;
+	__le64 snapid;
+	void *p;
+	void *end;
+	char *image_id;
+	u64 overlap;
+	size_t len = 0;
+	int ret;
+
+	parent_spec = rbd_spec_alloc();
+	if (!parent_spec)
+		return -ENOMEM;
+
+	size = sizeof (__le64) +				/* pool_id */
+		sizeof (__le32) + RBD_IMAGE_ID_LEN_MAX +	/* image_id */
+		sizeof (__le64) +				/* snap_id */
+		sizeof (__le64);				/* overlap */
+	reply_buf = kmalloc(size, GFP_KERNEL);
+	if (!reply_buf) {
+		ret = -ENOMEM;
+		goto out_err;
+	}
+
+	snapid = cpu_to_le64(CEPH_NOSNAP);
+	ret = rbd_req_sync_exec(rbd_dev, rbd_dev->header_name,
+				"rbd", "get_parent",
+				(char *) &snapid, sizeof (snapid),
+				(char *) reply_buf, size,
+				CEPH_OSD_FLAG_READ, NULL);
+	dout("%s: rbd_req_sync_exec returned %d\n", __func__, ret);
+	if (ret < 0)
+		goto out_err;
+
+	ret = -ERANGE;
+	p = reply_buf;
+	end = (char *) reply_buf + size;
+	ceph_decode_64_safe(&p, end, parent_spec->pool_id, out_err);
+	if (parent_spec->pool_id == CEPH_NOPOOL)
+		goto out;	/* No parent?  No problem. */
+
+	image_id = ceph_extract_encoded_string(&p, end, &len, GFP_KERNEL);
+	if (IS_ERR(image_id)) {
+		ret = PTR_ERR(image_id);
+		goto out_err;
+	}
+	parent_spec->image_id = image_id;
+	ceph_decode_64_safe(&p, end, parent_spec->snap_id, out_err);
+	ceph_decode_64_safe(&p, end, overlap, out_err);
+
+	rbd_dev->parent_overlap = overlap;
+	rbd_dev->parent_spec = parent_spec;
+	parent_spec = NULL;	/* rbd_dev now owns this */
+out:
+	ret = 0;
+out_err:
+	kfree(reply_buf);
+	rbd_spec_put(parent_spec);
+
+	return ret;
+}
+
  static int rbd_dev_v2_snap_context(struct rbd_device *rbd_dev, u64 *ver)
  {

size_t size;

drivers/block/rbd.c +6 -0

@@ -3154,6 +3268,12 @@ static int rbd_dev_v1_probe(struct rbd_device
 *rbd_dev)
  	ret = rbd_read_header(rbd_dev, &rbd_dev->header);
  	if (ret < 0)
  		goto out_err;
+
+	/* Version 1 images have no parent (no layering) */
+
+	rbd_dev->parent_spec = NULL;
+	rbd_dev->parent_overlap = 0;
+
  	rbd_dev->image_format = 1;
 

dout("discovered version 1 image, header name is %s\n",

drivers/block/rbd.c +8 -0

@@ -3205,6 +3325,14 @@ static int rbd_dev_v2_probe(struct rbd_device
 *rbd_dev)
  	if (ret < 0)
  		goto out_err;
 
+	/* If the image supports layering, get the parent info */
+
+	if (rbd_dev->header.features & RBD_FEATURE_LAYERING) {
+		ret = rbd_dev_v2_parent_info(rbd_dev);
+		if (ret < 0)
+			goto out_err;
+	}
+
  	/* crypto and compression type aren't (yet) supported for v2 images */
 

  	rbd_dev->header.crypt_type = 0;

drivers/block/rbd.c +3 -0

@@ -3224,6 +3352,9 @@ static int rbd_dev_v2_probe(struct rbd_device
 *rbd_dev)
 
  	return 0;
  out_err:
+	rbd_dev->parent_overlap = 0;
+	rbd_spec_put(rbd_dev->parent_spec);
+	rbd_dev->parent_spec = NULL;
  	kfree(rbd_dev->header_name);
  	rbd_dev->header_name = NULL;

  	kfree(rbd_dev->header.object_prefix);

include/linux/ceph/rados.h +2 -0

index 0a99099..15077db 100644
@@ -87,6 +87,8 @@ struct ceph_pg {
   *
   *  lpgp_num -- as above.
   */
+#define CEPH_NOPOOL  ((__u64) (-1))  /* pool id not defined */
+
  #define CEPH_PG_TYPE_REP     1
  #define CEPH_PG_TYPE_RAID4   2
  #define CEPH_PG_POOL_VERSION 2

Define and export function ceph_pg_pool_name_by_id() to supply the name of a pg pool whose id is given. This will be used by the next patch.

Signed-off-by Alex Elder <[email protected]>

---
  include/linux/ceph/osdmap.h |    1 +
  net/ceph/osdmap.c           |   16 ++++++++++++++++
  2 files changed, 17 insertions(+)

include/linux/ceph/osdmap.h +1 -0

index e88a620..5ea57ba 100644
@@ -123,6 +123,7 @@ extern int ceph_calc_pg_acting(struct ceph_osdmap
 *osdmap, struct ceph_pg pgid,
  extern int ceph_calc_pg_primary(struct ceph_osdmap *osdmap,
  				struct ceph_pg pgid);
 
+extern const char *ceph_pg_pool_name_by_id(struct ceph_osdmap *map, u64
 id);
  extern int ceph_pg_poolid_by_name(struct ceph_osdmap *map, const char

*name);

  #endif

net/ceph/osdmap.c +16 -0

index f552aa4..de73214 100644
@@ -469,6 +469,22 @@ static struct ceph_pg_pool_info
 *__lookup_pg_pool(struct rb_root *root, int id)
  	return NULL;
  }
 
+const char *ceph_pg_pool_name_by_id(struct ceph_osdmap *map, u64 id)
+{
+	struct ceph_pg_pool_info *pi;
+
+	if (id == CEPH_NOPOOL)
+		return NULL;
+
+	if (WARN_ON_ONCE(id > (u64) INT_MAX))
+		return NULL;
+
+	pi = __lookup_pg_pool(&map->pg_pools, (int) id);
+
+	return pi ? pi->name : NULL;
+}
+EXPORT_SYMBOL(ceph_pg_pool_name_by_id);
+
  int ceph_pg_poolid_by_name(struct ceph_osdmap *map, const char *name)
  {

struct rb_node *rbp;

drivers/block/rbd.c +29 -0

@@ -2514,6 +2532,115 @@ out_err:
 	return ret;
 }
 
+static char *rbd_dev_image_name(struct rbd_device *rbd_dev)
+{
+	size_t image_id_size;
+	char *image_id;
+	void *p;
+	void *end;
+	size_t size;
+	void *reply_buf = NULL;
+	size_t len = 0;
+	char *image_name = NULL;
+	int ret;
+
+	rbd_assert(!rbd_dev->spec->image_name);
+
+	image_id_size = sizeof (__le32) + rbd_dev->spec->image_id_len;
+	image_id = kmalloc(image_id_size, GFP_KERNEL);
+	if (!image_id)
+		return NULL;
+
+	p = image_id;
+	end = (char *) image_id + image_id_size;
+	ceph_encode_string(&p, end, rbd_dev->spec->image_id,
+				(u32) rbd_dev->spec->image_id_len);
+
+	size = sizeof (__le32) + RBD_IMAGE_NAME_LEN_MAX;
+	reply_buf = kmalloc(size, GFP_KERNEL);
+	if (!reply_buf)
+		goto out;
+

When a layered rbd image has a parent, that parent is identified only by its pool id, image id, and snapshot id. Images that have been mapped also record *names* for those three id's.

Add code to look up these names for parent images so they match mapped images more closely. Skip doing this for an image if it already has its pool name defined (this will be the case for images mapped by the user).

It is possible that an the name of a parent image can't be determined, even if the image id is valid. If this occurs it does not preclude correct operation, so don't treat this as an error.

On the other hand, defined pools will always have both an id and a
name. And any snapshot of an image identified as a parent for a
clone image will exist, and will have a name (if not it indicates some other internal error). So treat failure to get these bits of information as errors.

Signed-off-by Alex Elder <[email protected]>

---
  drivers/block/rbd.c |  131
+++++++++++++++++++++++++++++++++++++++++++++++++++
  1 file changed, 131 insertions(+)

drivers/block/rbd.c +131 -0

index bce1fcf..04062c1 100644
@@ -70,7 +70,10 @@
 
  #define RBD_SNAP_HEAD_NAME	"-"
 
+/* This allows a single page to hold an image name sent by OSD */
+#define RBD_IMAGE_NAME_LEN_MAX	(PAGE_SIZE - sizeof (__le32) - 1)
  #define RBD_IMAGE_ID_LEN_MAX	64
+
  #define RBD_OBJ_PREFIX_LEN_MAX	64
 
  /* Feature bits */

@@ -658,6 +661,20 @@ out_err:
  	return -ENOMEM;
  }
 
+static const char *rbd_snap_name(struct rbd_device *rbd_dev, u64 snap_id)
+{
+	struct rbd_snap *snap;
+
+	if (snap_id == CEPH_NOSNAP)
+		return RBD_SNAP_HEAD_NAME;
+
+	list_for_each_entry(snap, &rbd_dev->snaps, node)
+		if (snap_id == snap->id)
+			return snap->name;
+
+	return NULL;
+}
+
  static int snap_by_name(struct rbd_device *rbd_dev, const char *snap_name)
  {
 

@@ -2499,6 +2516,7 @@ static int rbd_dev_v2_parent_info(struct
 rbd_device *rbd_dev)
  		goto out_err;
  	}
  	parent_spec->image_id = image_id;
+	parent_spec->image_id_len = len;
  	ceph_decode_64_safe(&p, end, parent_spec->snap_id, out_err);
  	ceph_decode_64_safe(&p, end, overlap, out_err);

@@ -2514,6 +2532,115 @@ out_err:
  	return ret;
  }
 
+static char *rbd_dev_image_name(struct rbd_device *rbd_dev)
+{
+	size_t image_id_size;
+	char *image_id;
+	void *p;
+	void *end;
+	size_t size;
+	void *reply_buf = NULL;
+	size_t len = 0;
+	char *image_name = NULL;
+	int ret;
+
+	rbd_assert(!rbd_dev->spec->image_name);
+
+	image_id_size = sizeof (__le32) + rbd_dev->spec->image_id_len;
+	image_id = kmalloc(image_id_size, GFP_KERNEL);
+	if (!image_id)
+		return NULL;
+
+	p = image_id;
+	end = (char *) image_id + image_id_size;
+	ceph_encode_string(&p, end, rbd_dev->spec->image_id,
+				(u32) rbd_dev->spec->image_id_len);
+
+	size = sizeof (__le32) + RBD_IMAGE_NAME_LEN_MAX;
+	reply_buf = kmalloc(size, GFP_KERNEL);
+	if (!reply_buf)
+		goto out;
+
+	ret = rbd_req_sync_exec(rbd_dev, RBD_DIRECTORY,
+				"rbd", "dir_get_name",
+				image_id, image_id_size,
+				(char *) reply_buf, size,
+				CEPH_OSD_FLAG_READ, NULL);
+	if (ret < 0)
+		goto out;
+	p = reply_buf;
+	end = (char *) reply_buf + size;
+	image_name = ceph_extract_encoded_string(&p, end, &len, GFP_KERNEL);
+	if (image_name)
+		dout("%s: name is %s len is %zd\n", __func__, image_name, len);
+out:
+	kfree(reply_buf);
+	kfree(image_id);
+
+	return image_name;
+}
+
+/*
+ * When a parent image gets probed, we only have the pool, image,
+ * and snapshot ids but not the names of any of them.  This call
+ * is made later to fill in those names.  It has to be done after
+ * rbd_dev_snaps_update() has completed because some of the
+ * information (in particular, snapshot name) is not available
+ * until then.
+ */
+static int rbd_dev_probe_update_spec(struct rbd_device *rbd_dev)
+{
+	struct ceph_osd_client *osdc;
+	const char *name;
+	void *reply_buf = NULL;
+	int ret;
+
+	if (rbd_dev->spec->pool_name)
+		return 0;	/* Already have the names */
+
+	/* Look up the pool name */
+
+	osdc = &rbd_dev->rbd_client->client->osdc;
+	name = ceph_pg_pool_name_by_id(osdc->osdmap, rbd_dev->spec->pool_id);
+	if (!name)
+		return -EIO;	/* pool id too large (>= 2^31) */
+
+	rbd_dev->spec->pool_name = kstrdup(name, GFP_KERNEL);
+	if (!rbd_dev->spec->pool_name)
+		return -ENOMEM;
+
+	/* Fetch the image name; tolerate failure here */
+
+	name = rbd_dev_image_name(rbd_dev);
+	if (name) {
+		rbd_dev->spec->image_name_len = strlen(name);
+		rbd_dev->spec->image_name = (char *) name;
+	} else {
+		pr_warning(RBD_DRV_NAME "%d "
+			"unable to get image name for image id %s\n",
+			rbd_dev->major, rbd_dev->spec->image_id);
+	}
+
+	/* Look up the snapshot name. */
+
+	name = rbd_snap_name(rbd_dev, rbd_dev->spec->snap_id);
+	if (!name) {
+		ret = -EIO;
+		goto out_err;
+	}
+	rbd_dev->spec->snap_name = kstrdup(name, GFP_KERNEL);
+	if(!rbd_dev->spec->snap_name)
+		goto out_err;
+
+	return 0;
+out_err:
+	kfree(reply_buf);
+	kfree(rbd_dev->spec->pool_name);
+	rbd_dev->spec->pool_name = NULL;
+
+	return ret;
+}
+
  static int rbd_dev_v2_snap_context(struct rbd_device *rbd_dev, u64 *ver)
  {
  	size_t size;

@@ -3372,6 +3499,10 @@ static int rbd_dev_probe_finish(struct rbd_device
 *rbd_dev)
  	if (ret)
  		return ret;
 
+	ret = rbd_dev_probe_update_spec(rbd_dev);
+	if (ret)
+		goto err_out_snaps;
+
  	ret = rbd_dev_set_mapping(rbd_dev);
  	if (ret)

goto err_out_snaps;

I know you've got a queue of these already, but here's another:
rbd_dev_probe_update_spec() could definitely use some warnings to distinguish its error cases.

Reviewed-by Josh Durgin <[email protected]>

Quoted 204 lines

When a layered rbd image has a parent, that parent is identified only by its pool id, image id, and snapshot id. Images that have been mapped also record *names* for those three id's.

Add code to look up these names for parent images so they match mapped images more closely. Skip doing this for an image if it already has its pool name defined (this will be the case for images mapped by the user).

It is possible that an the name of a parent image can't be determined, even if the image id is valid. If this occurs it does not preclude correct operation, so don't treat this as an error.

On the other hand, defined pools will always have both an id and a
name. And any snapshot of an image identified as a parent for a
clone image will exist, and will have a name (if not it indicates some other internal error). So treat failure to get these bits of information as errors.

Signed-off-by Alex Elder <[email protected]>

---
  drivers/block/rbd.c |  131
+++++++++++++++++++++++++++++++++++++++++++++++++++
  1 file changed, 131 insertions(+)

drivers/block/rbd.c +17 -0

index bce1fcf..04062c1 100644
@@ -70,7 +70,10 @@
 
  #define RBD_SNAP_HEAD_NAME    "-"
 
+/* This allows a single page to hold an image name sent by OSD */
+#define RBD_IMAGE_NAME_LEN_MAX    (PAGE_SIZE - sizeof (__le32) - 1)
  #define RBD_IMAGE_ID_LEN_MAX    64
+
  #define RBD_OBJ_PREFIX_LEN_MAX    64
 
  /* Feature bits */

@@ -658,6 +661,20 @@ out_err:
      return -ENOMEM;
  }
 
+static const char *rbd_snap_name(struct rbd_device *rbd_dev, u64
 snap_id)
+{
+    struct rbd_snap *snap;
+
+    if (snap_id == CEPH_NOSNAP)
+        return RBD_SNAP_HEAD_NAME;
+
+    list_for_each_entry(snap, &rbd_dev->snaps, node)
+        if (snap_id == snap->id)
+            return snap->name;
+
+    return NULL;
+}
+
  static int snap_by_name(struct rbd_device *rbd_dev, const char
 *snap_name)

  {

drivers/block/rbd.c +110 -0

@@ -2499,6 +2516,7 @@ static int rbd_dev_v2_parent_info(struct
 rbd_device *rbd_dev)
          goto out_err;
      }
      parent_spec->image_id = image_id;
+    parent_spec->image_id_len = len;
      ceph_decode_64_safe(&p, end, parent_spec->snap_id, out_err);
      ceph_decode_64_safe(&p, end, overlap, out_err);

@@ -2514,6 +2532,115 @@ out_err:
      return ret;
  }
 
+static char *rbd_dev_image_name(struct rbd_device *rbd_dev)
+{
+    size_t image_id_size;
+    char *image_id;
+    void *p;
+    void *end;
+    size_t size;
+    void *reply_buf = NULL;
+    size_t len = 0;
+    char *image_name = NULL;
+    int ret;
+
+    rbd_assert(!rbd_dev->spec->image_name);
+
+    image_id_size = sizeof (__le32) + rbd_dev->spec->image_id_len;
+    image_id = kmalloc(image_id_size, GFP_KERNEL);
+    if (!image_id)
+        return NULL;
+
+    p = image_id;
+    end = (char *) image_id + image_id_size;
+    ceph_encode_string(&p, end, rbd_dev->spec->image_id,
+                (u32) rbd_dev->spec->image_id_len);
+
+    size = sizeof (__le32) + RBD_IMAGE_NAME_LEN_MAX;
+    reply_buf = kmalloc(size, GFP_KERNEL);
+    if (!reply_buf)
+        goto out;
+
+    ret = rbd_req_sync_exec(rbd_dev, RBD_DIRECTORY,
+                "rbd", "dir_get_name",
+                image_id, image_id_size,
+                (char *) reply_buf, size,
+                CEPH_OSD_FLAG_READ, NULL);
+    if (ret < 0)
+        goto out;
+    p = reply_buf;
+    end = (char *) reply_buf + size;
+    image_name = ceph_extract_encoded_string(&p, end, &len, GFP_KERNEL);
+    if (image_name)
+        dout("%s: name is %s len is %zd\n", __func__, image_name, len);
+out:
+    kfree(reply_buf);
+    kfree(image_id);
+
+    return image_name;
+}
+
+/*
+ * When a parent image gets probed, we only have the pool, image,
+ * and snapshot ids but not the names of any of them.  This call
+ * is made later to fill in those names.  It has to be done after
+ * rbd_dev_snaps_update() has completed because some of the
+ * information (in particular, snapshot name) is not available
+ * until then.
+ */
+static int rbd_dev_probe_update_spec(struct rbd_device *rbd_dev)
+{
+    struct ceph_osd_client *osdc;
+    const char *name;
+    void *reply_buf = NULL;
+    int ret;
+
+    if (rbd_dev->spec->pool_name)
+        return 0;    /* Already have the names */
+
+    /* Look up the pool name */
+
+    osdc = &rbd_dev->rbd_client->client->osdc;
+    name = ceph_pg_pool_name_by_id(osdc->osdmap,
 rbd_dev->spec->pool_id);
+    if (!name)
+        return -EIO;    /* pool id too large (>= 2^31) */
+
+    rbd_dev->spec->pool_name = kstrdup(name, GFP_KERNEL);
+    if (!rbd_dev->spec->pool_name)
+        return -ENOMEM;
+
+    /* Fetch the image name; tolerate failure here */
+
+    name = rbd_dev_image_name(rbd_dev);
+    if (name) {
+        rbd_dev->spec->image_name_len = strlen(name);
+        rbd_dev->spec->image_name = (char *) name;
+    } else {
+        pr_warning(RBD_DRV_NAME "%d "
+            "unable to get image name for image id %s\n",
+            rbd_dev->major, rbd_dev->spec->image_id);
+    }
+
+    /* Look up the snapshot name. */
+
+    name = rbd_snap_name(rbd_dev, rbd_dev->spec->snap_id);
+    if (!name) {
+        ret = -EIO;
+        goto out_err;
+    }
+    rbd_dev->spec->snap_name = kstrdup(name, GFP_KERNEL);
+    if(!rbd_dev->spec->snap_name)
+        goto out_err;
+
+    return 0;
+out_err:
+    kfree(reply_buf);
+    kfree(rbd_dev->spec->pool_name);
+    rbd_dev->spec->pool_name = NULL;
+
+    return ret;
+}
+
  static int rbd_dev_v2_snap_context(struct rbd_device *rbd_dev, u64
 *ver)

  {
      size_t size;

drivers/block/rbd.c +4 -0

@@ -3372,6 +3499,10 @@ static int rbd_dev_probe_finish(struct rbd_device
 *rbd_dev)
      if (ret)
          return ret;
 
+    ret = rbd_dev_probe_update_spec(rbd_dev);
+    if (ret)
+        goto err_out_snaps;
+
      ret = rbd_dev_set_mapping(rbd_dev);
      if (ret)

          goto err_out_snaps;

Hello,

I'm doing some tests with fio from a qemu 1.2 guest (virtio disk,cache=none), randread, with 4K block size on a small size of 1G (so it can be handle by the buffer cache on ceph cluster)

fio --filename=/dev/vdb -rw=randread --bs=4K --size=1000M --iodepth=40 --group_reporting --name=file1 --ioengine=libaio --direct=1

I can't get more than 5000 iops.

RBD cluster is :
---------------
3 nodes,with each node :
-6 x osd 15k drives (xfs), journal on tmpfs, 1 mon -cpu: 2x 4 cores intel xeon [email protected] rbd 0.53

ceph.conf

journal dio = false
filestore fiemap = false
filestore flusher = false
osd op threads = 24
osd disk threads = 24
filestore op threads = 6

kvm host is : 4 x 12 cores opteron
------------

During the bench:

on ceph nodes:
- cpu is around 10% used
- iostat show no disks activity on osds. (so I think that the 1G file is handle in the linux buffer)

on kvm host:

-cpu is around 20% used

I really don't see where is the bottleneck....

Any Ideas, hints ?

Regards,

Alexandre
--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 1 line

Have you tried increasing the iodepth?

Hello,

I'm doing some tests with fio from a qemu 1.2 guest (virtio disk,cache=none), randread, with 4K block size on a small size of 1G (so it can be handle by the buffer cache on ceph cluster)

fio --filename=/dev/vdb -rw=randread --bs=4K --size=1000M --iodepth=40 --group_reporting --name=file1 --ioengine=libaio --direct=1

I can't get more than 5000 iops.

RBD cluster is :
---------------
3 nodes,with each node :
-6 x osd 15k drives (xfs), journal on tmpfs, 1 mon -cpu: 2x 4 cores intel xeon [email protected] rbd 0.53

ceph.conf

journal dio = false
filestore fiemap = false
filestore flusher = false
osd op threads = 24
osd disk threads = 24
filestore op threads = 6

kvm host is : 4 x 12 cores opteron
------------

During the bench:

on ceph nodes:
- cpu is around 10% used
- iostat show no disks activity on osds. (so I think that the 1G file is handle in the linux buffer)

on kvm host:

-cpu is around 20% used

I really don't see where is the bottleneck....

Any Ideas, hints ?

Regards,

Alexandre

Signature · 3 lines

To unsubscribe from this list: send the line "unsubscribe ceph-devel" in 
the body of a message to [email protected] 
More majordomo info at http://vger.kernel.org/majordomo-info.html 

Quoted 1 line

Have you tried increasing the iodepth?

Hello,

I'm doing some tests with fio from a qemu 1.2 guest (virtio disk,cache=none), randread, with 4K block size on a small size of 1G (so it can be handle by the buffer cache on ceph cluster)

fio --filename=/dev/vdb -rw=randread --bs=4K --size=1000M --iodepth=40 --group_reporting --name=file1 --ioengine=libaio --direct=1

I can't get more than 5000 iops.

RBD cluster is :
---------------
3 nodes,with each node :
-6 x osd 15k drives (xfs), journal on tmpfs, 1 mon -cpu: 2x 4 cores intel xeon [email protected] rbd 0.53

ceph.conf

journal dio = false
filestore fiemap = false
filestore flusher = false
osd op threads = 24
osd disk threads = 24
filestore op threads = 6

kvm host is : 4 x 12 cores opteron
------------

During the bench:

on ceph nodes:
- cpu is around 10% used
- iostat show no disks activity on osds. (so I think that the 1G file is handle in the linux buffer)

on kvm host:

-cpu is around 20% used

I really don't see where is the bottleneck....

Any Ideas, hints ?

Regards,

Alexandre

Signature · 3 lines

To unsubscribe from this list: send the line "unsubscribe ceph-devel" in 
the body of a message to [email protected] 
More majordomo info at http://vger.kernel.org/majordomo-info.html 

Quoted 1 line

Quoted 1 line

Have you tried increasing the iodepth?

Yes, I have try with 100 and 200, same results.

I have also try directly from the host, with /dev/rbd1, and I have same result. I have also try with 3 differents hosts, with differents cpus models.

(note: I can reach around 40.000 iops with same fio config on a zfs iscsi array)

My test ceph cluster nodes cpus are old (xeon E5420), but they are around 10% usage, so I think it's ok.

Do you have an idea if I can trace something ?

Thanks,

Alexandre

----- Mail original -----

De: "Sage Weil" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 16:57:05
Objet: Re: slow fio random read benchmark, need help

Quoted 9 lines · Alexandre DERUMIER

Hello,

I'm doing some tests with fio from a qemu 1.2 guest (virtio disk,cache=none), randread, with 4K block size on a small size of 1G (so it can be handle by the buffer cache on ceph cluster)

fio --filename=/dev/vdb -rw=randread --bs=4K --size=1000M --iodepth=40 --group_reporting --name=file1 --ioengine=libaio --direct=1

I can't get more than 5000 iops.

Have you tried increasing the iodepth?

sage

Quoted 41 lines

RBD cluster is :
---------------
3 nodes,with each node :
-6 x osd 15k drives (xfs), journal on tmpfs, 1 mon -cpu: 2x 4 cores intel xeon [email protected] rbd 0.53

ceph.conf

journal dio = false
filestore fiemap = false
filestore flusher = false
osd op threads = 24
osd disk threads = 24
filestore op threads = 6

kvm host is : 4 x 12 cores opteron
------------

During the bench:

on ceph nodes:
- cpu is around 10% used
- iostat show no disks activity on osds. (so I think that the 1G file is handle in the linux buffer)

on kvm host:

-cpu is around 20% used

I really don't see where is the bottleneck....

Any Ideas, hints ?

Regards,

Quoted 7 lines

Alexandre
--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 1 line

Quoted 1 line

Have you tried increasing the iodepth?

Yes, I have try with 100 and 200, same results.

I have also try directly from the host, with /dev/rbd1, and I have same result. I have also try with 3 differents hosts, with differents cpus models.

(note: I can reach around 40.000 iops with same fio config on a zfs iscsi array)

My test ceph cluster nodes cpus are old (xeon E5420), but they are around 10% usage, so I think it's ok.

Do you have an idea if I can trace something ?

Thanks,

Alexandre

----- Mail original -----

De: "Sage Weil" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 16:57:05
Objet: Re: slow fio random read benchmark, need help

Quoted 9 lines · Alexandre DERUMIER

Hello,

I'm doing some tests with fio from a qemu 1.2 guest (virtio disk,cache=none), randread, with 4K block size on a small size of 1G (so it can be handle by the buffer cache on ceph cluster)

fio --filename=/dev/vdb -rw=randread --bs=4K --size=1000M --iodepth=40 --group_reporting --name=file1 --ioengine=libaio --direct=1

I can't get more than 5000 iops.

Have you tried increasing the iodepth?

sage

Quoted 41 lines

RBD cluster is :
---------------
3 nodes,with each node :
-6 x osd 15k drives (xfs), journal on tmpfs, 1 mon -cpu: 2x 4 cores intel xeon [email protected] rbd 0.53

ceph.conf

journal dio = false
filestore fiemap = false
filestore flusher = false
osd op threads = 24
osd disk threads = 24
filestore op threads = 6

kvm host is : 4 x 12 cores opteron
------------

During the bench:

on ceph nodes:
- cpu is around 10% used
- iostat show no disks activity on osds. (so I think that the 1G file is handle in the linux buffer)

on kvm host:

-cpu is around 20% used

I really don't see where is the bottleneck....

Any Ideas, hints ?

Regards,

Quoted 7 lines

Alexandre

Signature · 3 lines

To unsubscribe from this list: send the line "unsubscribe ceph-devel" in 
the body of a message to [email protected] 
More majordomo info at http://vger.kernel.org/majordomo-info.html 

Signature · 3 lines

To unsubscribe from this list: send the line "unsubscribe ceph-devel" in 
the body of a message to [email protected] 
More majordomo info at http://vger.kernel.org/majordomo-info.html 

Thanks Marcus,

indeed gigabit ethernet.

note that my iscsi results (40k)was with multipath, so multiple gigabit links.

I have also done tests with a netapp array, with nfs, single link, I'm around 13000 iops

I will do more tests with multiples vms, from differents hosts, and with --numjobs.

I'll keep you in touch,

Thanks for help,

Regards,

Alexandre

----- Mail original -----

De: "Marcus Sorensen" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "Sage Weil" <[email protected]>, "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 18:08:11
Objet: Re: slow fio random read benchmark, need help

5000 is actually really good, if you ask me. Assuming everything is connected via gigabit. If you get 40k iops locally, you add the latency of tcp, as well as that of the ceph services and VM layer, and that's what you get. On my network I get about a .1ms round trip on gigabit over the same switch, which by definition can only do 10,000 iops. Then if you have storage on the other end capable of 40k iops, you add the latencies together (.1ms + .025ms) and you're at 8k iops. Then add the small latency of the application servicing the io (NFS, Ceph, etc), and the latency introduced by your VM layer, and 5k sounds about right.

The good news is that you probably aren't taxing the storage, you can likely do many simultaneous tests from several VMs and get the same results.

You can try adding --numjobs to your fio to parallelize the specific test you're doing, or launching a second VM and doing the same test at the same time. This would be a good indicator if it's latency.

Quoted 92 lines · Alexandre DERUMIER

Quoted 1 line

Quoted 1 line

Have you tried increasing the iodepth?

Yes, I have try with 100 and 200, same results.

I have also try directly from the host, with /dev/rbd1, and I have same result. I have also try with 3 differents hosts, with differents cpus models.

(note: I can reach around 40.000 iops with same fio config on a zfs iscsi array)

My test ceph cluster nodes cpus are old (xeon E5420), but they are around 10% usage, so I think it's ok.

Do you have an idea if I can trace something ?

Thanks,

Alexandre

----- Mail original -----

De: "Sage Weil" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 16:57:05
Objet: Re: slow fio random read benchmark, need help

Quoted 9 lines · Alexandre DERUMIER

Hello,

I'm doing some tests with fio from a qemu 1.2 guest (virtio disk,cache=none), randread, with 4K block size on a small size of 1G (so it can be handle by the buffer cache on ceph cluster)

fio --filename=/dev/vdb -rw=randread --bs=4K --size=1000M --iodepth=40 --group_reporting --name=file1 --ioengine=libaio --direct=1

I can't get more than 5000 iops.

Have you tried increasing the iodepth?

sage

Quoted 39 lines

RBD cluster is :
---------------
3 nodes,with each node :
-6 x osd 15k drives (xfs), journal on tmpfs, 1 mon -cpu: 2x 4 cores intel xeon [email protected] rbd 0.53

ceph.conf

journal dio = false
filestore fiemap = false
filestore flusher = false
osd op threads = 24
osd disk threads = 24
filestore op threads = 6

kvm host is : 4 x 12 cores opteron
------------

During the bench:

on ceph nodes:
- cpu is around 10% used
- iostat show no disks activity on osds. (so I think that the 1G file is handle in the linux buffer)

on kvm host:

-cpu is around 20% used

I really don't see where is the bottleneck....

Any Ideas, hints ?

Quoted 9 lines

Regards,

Alexandre
--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Thanks Marcus,

indeed gigabit ethernet.

note that my iscsi results (40k)was with multipath, so multiple gigabit links.

I have also done tests with a netapp array, with nfs, single link, I'm around 13000 iops

I will do more tests with multiples vms, from differents hosts, and with --numjobs.

I'll keep you in touch,

Thanks for help,

Regards,

Alexandre

----- Mail original -----

De: "Marcus Sorensen" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "Sage Weil" <[email protected]>, "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 18:08:11
Objet: Re: slow fio random read benchmark, need help

5000 is actually really good, if you ask me. Assuming everything is connected via gigabit. If you get 40k iops locally, you add the latency of tcp, as well as that of the ceph services and VM layer, and that's what you get. On my network I get about a .1ms round trip on gigabit over the same switch, which by definition can only do 10,000 iops. Then if you have storage on the other end capable of 40k iops, you add the latencies together (.1ms + .025ms) and you're at 8k iops. Then add the small latency of the application servicing the io (NFS, Ceph, etc), and the latency introduced by your VM layer, and 5k sounds about right.

The good news is that you probably aren't taxing the storage, you can likely do many simultaneous tests from several VMs and get the same results.

You can try adding --numjobs to your fio to parallelize the specific test you're doing, or launching a second VM and doing the same test at the same time. This would be a good indicator if it's latency.

Quoted 92 lines · Alexandre DERUMIER

Quoted 1 line

Quoted 1 line

Have you tried increasing the iodepth?

Yes, I have try with 100 and 200, same results.

I have also try directly from the host, with /dev/rbd1, and I have same result. I have also try with 3 differents hosts, with differents cpus models.

(note: I can reach around 40.000 iops with same fio config on a zfs iscsi array)

My test ceph cluster nodes cpus are old (xeon E5420), but they are around 10% usage, so I think it's ok.

Do you have an idea if I can trace something ?

Thanks,

Alexandre

----- Mail original -----

De: "Sage Weil" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 16:57:05
Objet: Re: slow fio random read benchmark, need help

Quoted 9 lines · Alexandre DERUMIER

Hello,

I'm doing some tests with fio from a qemu 1.2 guest (virtio disk,cache=none), randread, with 4K block size on a small size of 1G (so it can be handle by the buffer cache on ceph cluster)

fio --filename=/dev/vdb -rw=randread --bs=4K --size=1000M --iodepth=40 --group_reporting --name=file1 --ioengine=libaio --direct=1

I can't get more than 5000 iops.

Have you tried increasing the iodepth?

sage

Quoted 48 lines

RBD cluster is :
---------------
3 nodes,with each node :
-6 x osd 15k drives (xfs), journal on tmpfs, 1 mon -cpu: 2x 4 cores intel xeon [email protected] rbd 0.53

ceph.conf

journal dio = false
filestore fiemap = false
filestore flusher = false
osd op threads = 24
osd disk threads = 24
filestore op threads = 6

kvm host is : 4 x 12 cores opteron
------------

During the bench:

on ceph nodes:
- cpu is around 10% used
- iostat show no disks activity on osds. (so I think that the 1G file is handle in the linux buffer)

on kvm host:

-cpu is around 20% used

I really don't see where is the bottleneck....

Any Ideas, hints ?

Regards,

Alexandre

Signature · 3 lines

To unsubscribe from this list: send the line "unsubscribe ceph-devel" in 
the body of a message to [email protected] 
More majordomo info at http://vger.kernel.org/majordomo-info.html 

Signature · 3 lines

To unsubscribe from this list: send the line "unsubscribe ceph-devel" in 
the body of a message to [email protected] 
More majordomo info at http://vger.kernel.org/majordomo-info.html 

Yes, I think you are right, round trip with mon must cut by half the performance.

I have just done test with 2 parallel fio bench, from 2 differents host, I get 2 x 5000 iops

so it must be related to network latency.

I have also done tests with --numjob 1000, it doesn't help, same results.

Do you have an idea how I can have more io from 1 host ? Doing lacp with multiple links ?

I think that 10gigabit latency is almost same, i'm not sure it will improve iops too much Maybe InfiniBand can help?

----- Mail original -----

De: "Marcus Sorensen" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "Sage Weil" <[email protected]>, "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 18:38:46
Objet: Re: slow fio random read benchmark, need help

Yes, I was going to say that the most I've ever seen out of gigabit is about 15k iops, with parallel tests and NFS (or iSCSI). Multipathing may not really parallelize the io for you. It can send an io down one path, then move to the next path and send the next io without necessarily waiting for the previous one to respond, but it only shaves a slight amount from your latency under some scenarios as opposed to sending down all paths simultaneously. I have seen it help with high latency links.

I don't remember the Ceph design that well, but with distributed storage systems you're going to pay a penalty. If you can do 10-15k with one TCP round trip, you'll get half that with the round trip to talk to the metadata server to find your blocks and then to fetch them. Like I said, that might not be exactly what Ceph does, but you're going to have more traffic than just a straight single attached NFS or iscsi server.

Quoted 140 lines · Alexandre DERUMIER

Thanks Marcus,

indeed gigabit ethernet.

note that my iscsi results (40k)was with multipath, so multiple gigabit links.

I have also done tests with a netapp array, with nfs, single link, I'm around 13000 iops

I will do more tests with multiples vms, from differents hosts, and with --numjobs.

I'll keep you in touch,

Thanks for help,

Regards,

Alexandre

----- Mail original -----

De: "Marcus Sorensen" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "Sage Weil" <[email protected]>, "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 18:08:11
Objet: Re: slow fio random read benchmark, need help

5000 is actually really good, if you ask me. Assuming everything is connected via gigabit. If you get 40k iops locally, you add the latency of tcp, as well as that of the ceph services and VM layer, and that's what you get. On my network I get about a .1ms round trip on gigabit over the same switch, which by definition can only do 10,000 iops. Then if you have storage on the other end capable of 40k iops, you add the latencies together (.1ms + .025ms) and you're at 8k iops. Then add the small latency of the application servicing the io (NFS, Ceph, etc), and the latency introduced by your VM layer, and 5k sounds about right.

The good news is that you probably aren't taxing the storage, you can likely do many simultaneous tests from several VMs and get the same results.

You can try adding --numjobs to your fio to parallelize the specific test you're doing, or launching a second VM and doing the same test at the same time. This would be a good indicator if it's latency.

Quoted 92 lines · Alexandre DERUMIER

Quoted 1 line

Quoted 1 line

Have you tried increasing the iodepth?

Yes, I have try with 100 and 200, same results.

I have also try directly from the host, with /dev/rbd1, and I have same result. I have also try with 3 differents hosts, with differents cpus models.

(note: I can reach around 40.000 iops with same fio config on a zfs iscsi array)

My test ceph cluster nodes cpus are old (xeon E5420), but they are around 10% usage, so I think it's ok.

Do you have an idea if I can trace something ?

Thanks,

Alexandre

----- Mail original -----

De: "Sage Weil" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 16:57:05
Objet: Re: slow fio random read benchmark, need help

Quoted 9 lines · Alexandre DERUMIER

Hello,

I'm doing some tests with fio from a qemu 1.2 guest (virtio disk,cache=none), randread, with 4K block size on a small size of 1G (so it can be handle by the buffer cache on ceph cluster)

fio --filename=/dev/vdb -rw=randread --bs=4K --size=1000M --iodepth=40 --group_reporting --name=file1 --ioengine=libaio --direct=1

I can't get more than 5000 iops.

Have you tried increasing the iodepth?

sage

Quoted 39 lines

RBD cluster is :
---------------
3 nodes,with each node :
-6 x osd 15k drives (xfs), journal on tmpfs, 1 mon -cpu: 2x 4 cores intel xeon [email protected] rbd 0.53

ceph.conf

journal dio = false
filestore fiemap = false
filestore flusher = false
osd op threads = 24
osd disk threads = 24
filestore op threads = 6

kvm host is : 4 x 12 cores opteron
------------

During the bench:

on ceph nodes:
- cpu is around 10% used
- iostat show no disks activity on osds. (so I think that the 1G file is handle in the linux buffer)

on kvm host:

-cpu is around 20% used

I really don't see where is the bottleneck....

Any Ideas, hints ?

Quoted 9 lines

Regards,

Alexandre
--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 3 lines

Come to think of it that 15k iops I mentioned was on 10G ethernet with NFS. I have tried infiniband with ipoib and tcp, it's similar to 10G ethernet.

Quoted 4 lines

You will need to get creative. What you're asking for really is to have local latencies with remote storage. Just off of the top of my head you may look into some way to do local caching on SSD for your RBD volume, like bcache or flashcache.

Quoted 4 lines

At any rate, 5000 iops is not as good as a new SSD, but far better than a normal disk. Is there some specific application requirement, or is it just that you are feeling like you want the full performance from the VM?

Yes, I think you are right, round trip with mon must cut by half the performance.

I have just done test with 2 parallel fio bench, from 2 differents host, I get 2 x 5000 iops

so it must be related to network latency.

I have also done tests with --numjob 1000, it doesn't help, same results.

Do you have an idea how I can have more io from 1 host ? Doing lacp with multiple links ?

I think that 10gigabit latency is almost same, i'm not sure it will improve iops too much Maybe InfiniBand can help?

----- Mail original -----

De: "Marcus Sorensen" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "Sage Weil" <[email protected]>, "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 18:38:46
Objet: Re: slow fio random read benchmark, need help

Yes, I was going to say that the most I've ever seen out of gigabit is about 15k iops, with parallel tests and NFS (or iSCSI). Multipathing may not really parallelize the io for you. It can send an io down one path, then move to the next path and send the next io without necessarily waiting for the previous one to respond, but it only shaves a slight amount from your latency under some scenarios as opposed to sending down all paths simultaneously. I have seen it help with high latency links.

I don't remember the Ceph design that well, but with distributed storage systems you're going to pay a penalty. If you can do 10-15k with one TCP round trip, you'll get half that with the round trip to talk to the metadata server to find your blocks and then to fetch them. Like I said, that might not be exactly what Ceph does, but you're going to have more traffic than just a straight single attached NFS or iscsi server.

Quoted 140 lines · Alexandre DERUMIER

Thanks Marcus,

indeed gigabit ethernet.

note that my iscsi results (40k)was with multipath, so multiple gigabit links.

I have also done tests with a netapp array, with nfs, single link, I'm around 13000 iops

I will do more tests with multiples vms, from differents hosts, and with --numjobs.

I'll keep you in touch,

Thanks for help,

Regards,

Alexandre

----- Mail original -----

De: "Marcus Sorensen" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "Sage Weil" <[email protected]>, "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 18:08:11
Objet: Re: slow fio random read benchmark, need help

5000 is actually really good, if you ask me. Assuming everything is connected via gigabit. If you get 40k iops locally, you add the latency of tcp, as well as that of the ceph services and VM layer, and that's what you get. On my network I get about a .1ms round trip on gigabit over the same switch, which by definition can only do 10,000 iops. Then if you have storage on the other end capable of 40k iops, you add the latencies together (.1ms + .025ms) and you're at 8k iops. Then add the small latency of the application servicing the io (NFS, Ceph, etc), and the latency introduced by your VM layer, and 5k sounds about right.

The good news is that you probably aren't taxing the storage, you can likely do many simultaneous tests from several VMs and get the same results.

You can try adding --numjobs to your fio to parallelize the specific test you're doing, or launching a second VM and doing the same test at the same time. This would be a good indicator if it's latency.

Quoted 92 lines · Alexandre DERUMIER

Quoted 1 line

Quoted 1 line

Have you tried increasing the iodepth?

Yes, I have try with 100 and 200, same results.

I have also try directly from the host, with /dev/rbd1, and I have same result. I have also try with 3 differents hosts, with differents cpus models.

(note: I can reach around 40.000 iops with same fio config on a zfs iscsi array)

My test ceph cluster nodes cpus are old (xeon E5420), but they are around 10% usage, so I think it's ok.

Do you have an idea if I can trace something ?

Thanks,

Alexandre

----- Mail original -----

De: "Sage Weil" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 16:57:05
Objet: Re: slow fio random read benchmark, need help

Quoted 9 lines · Alexandre DERUMIER

Hello,

I'm doing some tests with fio from a qemu 1.2 guest (virtio disk,cache=none), randread, with 4K block size on a small size of 1G (so it can be handle by the buffer cache on ceph cluster)

fio --filename=/dev/vdb -rw=randread --bs=4K --size=1000M --iodepth=40 --group_reporting --name=file1 --ioengine=libaio --direct=1

I can't get more than 5000 iops.

Have you tried increasing the iodepth?

sage

Quoted 39 lines

RBD cluster is :
---------------
3 nodes,with each node :
-6 x osd 15k drives (xfs), journal on tmpfs, 1 mon -cpu: 2x 4 cores intel xeon [email protected] rbd 0.53

ceph.conf

journal dio = false
filestore fiemap = false
filestore flusher = false
osd op threads = 24
osd disk threads = 24
filestore op threads = 6

kvm host is : 4 x 12 cores opteron
------------

During the bench:

on ceph nodes:
- cpu is around 10% used
- iostat show no disks activity on osds. (so I think that the 1G file is handle in the linux buffer)

on kvm host:

-cpu is around 20% used

I really don't see where is the bottleneck....

Any Ideas, hints ?

Quoted 9 lines

Regards,

Alexandre
--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 3 lines

Quoted 3 lines

Come to think of it that 15k iops I mentioned was on 10G ethernet with NFS. I have tried infiniband with ipoib and tcp, it's similar to 10G ethernet.

I have see new arista 10GBe switch with latency around 1microsecond, that seem pretty good to do the job.

Quoted 7 lines

Quoted 7 lines

You will need to get creative. What you're asking for really is to have local latencies with remote storage. Just off of the top of my head you may look into some way to do local caching on SSD for your RBD volume, like bcache or flashcache.

I have already thinked about it. (But I would like to use qemu-rbd if possible)

Quoted 4 lines

Quoted 4 lines

At any rate, 5000 iops is not as good as a new SSD, but far better than a normal disk. Is there some specific application requirement, or is it just that you are feeling like you want the full performance from the VM?

I have some customers with some huge databases (too big to be handle in the bufer), require a lot of ios. (around 10K).

I have redone tests with 4 guest in parallel, I get 4 x 5000iops, so it seem to scale ! (and cpu is very low on the ceph cluster).

So I'll try some tricks, like raid over multiple rbd devices, maybe it'll help.

Thanks again for the help Marcus, I was not aware of these latencies problems.

Regards,

Alexandre

----- Mail original -----

De: "Marcus Sorensen" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "Sage Weil" <[email protected]>, "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 20:50:36
Objet: Re: slow fio random read benchmark, need help

Come to think of it that 15k iops I mentioned was on 10G ethernet with NFS. I have tried infiniband with ipoib and tcp, it's similar to 10G ethernet.

You will need to get creative. What you're asking for really is to have local latencies with remote storage. Just off of the top of my head you may look into some way to do local caching on SSD for your RBD volume, like bcache or flashcache.

Depending on your application, it may actually be a bonus that no single server (or handful of servers) can crush your storage's performance. if you only have one or two clients anyway then that may not be much consolation, but if you're going to have dozens or more then there's not much benefit to having one take all performance at the expense of everyone, except for perhaps in bursts.

At any rate, 5000 iops is not as good as a new SSD, but far better than a normal disk. Is there some specific application requirement, or is it just that you are feeling like you want the full performance from the VM?

Quoted 41 lines · Alexandre DERUMIER

Yes, I think you are right, round trip with mon must cut by half the performance.

I have just done test with 2 parallel fio bench, from 2 differents host, I get 2 x 5000 iops

so it must be related to network latency.

I have also done tests with --numjob 1000, it doesn't help, same results.

Do you have an idea how I can have more io from 1 host ? Doing lacp with multiple links ?

I think that 10gigabit latency is almost same, i'm not sure it will improve iops too much Maybe InfiniBand can help?

----- Mail original -----

De: "Marcus Sorensen" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "Sage Weil" <[email protected]>, "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 18:38:46
Objet: Re: slow fio random read benchmark, need help

Yes, I was going to say that the most I've ever seen out of gigabit is about 15k iops, with parallel tests and NFS (or iSCSI). Multipathing may not really parallelize the io for you. It can send an io down one path, then move to the next path and send the next io without necessarily waiting for the previous one to respond, but it only shaves a slight amount from your latency under some scenarios as opposed to sending down all paths simultaneously. I have seen it help with high latency links.

I don't remember the Ceph design that well, but with distributed storage systems you're going to pay a penalty. If you can do 10-15k with one TCP round trip, you'll get half that with the round trip to talk to the metadata server to find your blocks and then to fetch them. Like I said, that might not be exactly what Ceph does, but you're going to have more traffic than just a straight single attached NFS or iscsi server.

Quoted 142 lines

Quoted 140 lines · Alexandre DERUMIER

Thanks Marcus,

indeed gigabit ethernet.

note that my iscsi results (40k)was with multipath, so multiple gigabit links.

I have also done tests with a netapp array, with nfs, single link, I'm around 13000 iops

I will do more tests with multiples vms, from differents hosts, and with --numjobs.

I'll keep you in touch,

Thanks for help,

Regards,

Alexandre

----- Mail original -----

De: "Marcus Sorensen" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "Sage Weil" <[email protected]>, "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 18:08:11
Objet: Re: slow fio random read benchmark, need help

5000 is actually really good, if you ask me. Assuming everything is connected via gigabit. If you get 40k iops locally, you add the latency of tcp, as well as that of the ceph services and VM layer, and that's what you get. On my network I get about a .1ms round trip on gigabit over the same switch, which by definition can only do 10,000 iops. Then if you have storage on the other end capable of 40k iops, you add the latencies together (.1ms + .025ms) and you're at 8k iops. Then add the small latency of the application servicing the io (NFS, Ceph, etc), and the latency introduced by your VM layer, and 5k sounds about right.

The good news is that you probably aren't taxing the storage, you can likely do many simultaneous tests from several VMs and get the same results.

You can try adding --numjobs to your fio to parallelize the specific test you're doing, or launching a second VM and doing the same test at the same time. This would be a good indicator if it's latency.

Quoted 92 lines · Alexandre DERUMIER

Quoted 1 line

Quoted 1 line

Have you tried increasing the iodepth?

Yes, I have try with 100 and 200, same results.

I have also try directly from the host, with /dev/rbd1, and I have same result. I have also try with 3 differents hosts, with differents cpus models.

(note: I can reach around 40.000 iops with same fio config on a zfs iscsi array)

My test ceph cluster nodes cpus are old (xeon E5420), but they are around 10% usage, so I think it's ok.

Do you have an idea if I can trace something ?

Thanks,

Alexandre

----- Mail original -----

De: "Sage Weil" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 16:57:05
Objet: Re: slow fio random read benchmark, need help

Quoted 9 lines · Alexandre DERUMIER

Hello,

I'm doing some tests with fio from a qemu 1.2 guest (virtio disk,cache=none), randread, with 4K block size on a small size of 1G (so it can be handle by the buffer cache on ceph cluster)

fio --filename=/dev/vdb -rw=randread --bs=4K --size=1000M --iodepth=40 --group_reporting --name=file1 --ioengine=libaio --direct=1

I can't get more than 5000 iops.

Have you tried increasing the iodepth?

sage

Quoted 39 lines

RBD cluster is :
---------------
3 nodes,with each node :
-6 x osd 15k drives (xfs), journal on tmpfs, 1 mon -cpu: 2x 4 cores intel xeon [email protected] rbd 0.53

ceph.conf

journal dio = false
filestore fiemap = false
filestore flusher = false
osd op threads = 24
osd disk threads = 24
filestore op threads = 6

kvm host is : 4 x 12 cores opteron
------------

During the bench:

on ceph nodes:
- cpu is around 10% used
- iostat show no disks activity on osds. (so I think that the 1G file is handle in the linux buffer)

on kvm host:

-cpu is around 20% used

I really don't see where is the bottleneck....

Any Ideas, hints ?

Quoted 9 lines

Regards,

Alexandre
--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Yes, I think you are right, round trip with mon must cut by half the performance.

I have just done test with 2 parallel fio bench, from 2 differents host, I get 2 x 5000 iops

so it must be related to network latency.

I have also done tests with --numjob 1000, it doesn't help, same results.

Do you have an idea how I can have more io from 1 host ? Doing lacp with multiple links ?

I think that 10gigabit latency is almost same, i'm not sure it will improve iops too much Maybe InfiniBand can help?

----- Mail original -----

De: "Marcus Sorensen" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "Sage Weil" <[email protected]>, "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 18:38:46
Objet: Re: slow fio random read benchmark, need help

Yes, I was going to say that the most I've ever seen out of gigabit is about 15k iops, with parallel tests and NFS (or iSCSI). Multipathing may not really parallelize the io for you. It can send an io down one path, then move to the next path and send the next io without necessarily waiting for the previous one to respond, but it only shaves a slight amount from your latency under some scenarios as opposed to sending down all paths simultaneously. I have seen it help with high latency links.

I don't remember the Ceph design that well, but with distributed storage systems you're going to pay a penalty. If you can do 10-15k with one TCP round trip, you'll get half that with the round trip to talk to the metadata server to find your blocks and then to fetch them. Like I said, that might not be exactly what Ceph does, but you're going to have more traffic than just a straight single attached NFS or iscsi server.

Quoted 122 lines · Alexandre DERUMIER

Thanks Marcus,

indeed gigabit ethernet.

note that my iscsi results (40k)was with multipath, so multiple gigabit links.

I have also done tests with a netapp array, with nfs, single link, I'm around 13000 iops

I will do more tests with multiples vms, from differents hosts, and with --numjobs.

I'll keep you in touch,

Thanks for help,

Regards,

Alexandre

----- Mail original -----

De: "Marcus Sorensen" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "Sage Weil" <[email protected]>, "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 18:08:11
Objet: Re: slow fio random read benchmark, need help

5000 is actually really good, if you ask me. Assuming everything is connected via gigabit. If you get 40k iops locally, you add the latency of tcp, as well as that of the ceph services and VM layer, and that's what you get. On my network I get about a .1ms round trip on gigabit over the same switch, which by definition can only do 10,000 iops. Then if you have storage on the other end capable of 40k iops, you add the latencies together (.1ms + .025ms) and you're at 8k iops. Then add the small latency of the application servicing the io (NFS, Ceph, etc), and the latency introduced by your VM layer, and 5k sounds about right.

The good news is that you probably aren't taxing the storage, you can likely do many simultaneous tests from several VMs and get the same results.

You can try adding --numjobs to your fio to parallelize the specific test you're doing, or launching a second VM and doing the same test at the same time. This would be a good indicator if it's latency.

Quoted 74 lines · Alexandre DERUMIER

Quoted 1 line

Quoted 1 line

Have you tried increasing the iodepth?

Yes, I have try with 100 and 200, same results.

I have also try directly from the host, with /dev/rbd1, and I have same result. I have also try with 3 differents hosts, with differents cpus models.

(note: I can reach around 40.000 iops with same fio config on a zfs iscsi array)

My test ceph cluster nodes cpus are old (xeon E5420), but they are around 10% usage, so I think it's ok.

Do you have an idea if I can trace something ?

Thanks,

Alexandre

----- Mail original -----

De: "Sage Weil" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 16:57:05
Objet: Re: slow fio random read benchmark, need help

Quoted 9 lines · Alexandre DERUMIER

Hello,

I'm doing some tests with fio from a qemu 1.2 guest (virtio disk,cache=none), randread, with 4K block size on a small size of 1G (so it can be handle by the buffer cache on ceph cluster)

fio --filename=/dev/vdb -rw=randread --bs=4K --size=1000M --iodepth=40 --group_reporting --name=file1 --ioengine=libaio --direct=1

I can't get more than 5000 iops.

Have you tried increasing the iodepth?

sage

Quoted 34 lines

RBD cluster is :
---------------
3 nodes,with each node :
-6 x osd 15k drives (xfs), journal on tmpfs, 1 mon -cpu: 2x 4 cores intel xeon [email protected] rbd 0.53

ceph.conf

journal dio = false
filestore fiemap = false
filestore flusher = false
osd op threads = 24
osd disk threads = 24
filestore op threads = 6

kvm host is : 4 x 12 cores opteron
------------

During the bench:

on ceph nodes:
- cpu is around 10% used
- iostat show no disks activity on osds. (so I think that the 1G file is handle in the linux buffer)

on kvm host:

-cpu is around 20% used

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

I really don't see where is the bottleneck....

Any Ideas, hints ?

Regards,

Alexandre

-----Original Message-----
From: [email protected] [mailto:ceph-devel- [email protected]] On Behalf Of Alexandre DERUMIER
Sent: Mittwoch, 31. Oktober 2012 18:27
To: Marcus Sorensen
Cc: Sage Weil; ceph-devel
Subject: Re: slow fio random read benchmark, need help

Thanks Marcus,

indeed gigabit ethernet.

note that my iscsi results (40k)was with multipath, so multiple gigabit links.

I have also done tests with a netapp array, with nfs, single link, I'm around 13000 iops

I will do more tests with multiples vms, from differents hosts, and with -- numjobs.

I'll keep you in touch,

Thanks for help,

Regards,

Alexandre

----- Mail original -----

De: "Marcus Sorensen" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "Sage Weil" <[email protected]>, "ceph-devel" <ceph- [email protected]>
Envoyé: Mercredi 31 Octobre 2012 18:08:11
Objet: Re: slow fio random read benchmark, need help

5000 is actually really good, if you ask me. Assuming everything is connected via gigabit. If you get 40k iops locally, you add the latency of tcp, as well as that of the ceph services and VM layer, and that's what you get. On my network I get about a .1ms round trip on gigabit over the same switch, which by definition can only do 10,000 iops. Then if you have storage on the other end capable of 40k iops, you add the latencies together (.1ms + .025ms) and you're at 8k iops.
Then add the small latency of the application servicing the io (NFS, Ceph, etc), and the latency introduced by your VM layer, and 5k sounds about right.

The good news is that you probably aren't taxing the storage, you can likely do many simultaneous tests from several VMs and get the same results.

You can try adding --numjobs to your fio to parallelize the specific test you're doing, or launching a second VM and doing the same test at the same time. This would be a good indicator if it's latency.

Quoted 4 lines · Alexandre DERUMIER

Quoted 1 line

Quoted 1 line

Have you tried increasing the iodepth?

Yes, I have try with 100 and 200, same results.

I have also try directly from the host, with /dev/rbd1, and I have same

result.

Quoted 6 lines

I have also try with 3 differents hosts, with differents cpus models.

(note: I can reach around 40.000 iops with same fio config on a zfs iscsi array)

My test ceph cluster nodes cpus are old (xeon E5420), but they are around

10% usage, so I think it's ok.

Quoted 74 lines

Do you have an idea if I can trace something ?

Thanks,

Alexandre

----- Mail original -----

De: "Sage Weil" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 16:57:05
Objet: Re: slow fio random read benchmark, need help

Quoted 7 lines · Alexandre DERUMIER

Hello,

I'm doing some tests with fio from a qemu 1.2 guest (virtio disk,cache=none), randread, with 4K block size on a small size of 1G (so it can be handle by the buffer cache on ceph cluster)

Quoted 6 lines

fio --filename=/dev/vdb -rw=randread --bs=4K --size=1000M --iodepth=40 --group_reporting --name=file1 --ioengine=libaio --direct=1

I can't get more than 5000 iops.

Have you tried increasing the iodepth?

sage

Quoted 39 lines

RBD cluster is :
---------------
3 nodes,with each node :
-6 x osd 15k drives (xfs), journal on tmpfs, 1 mon -cpu: 2x 4 cores intel xeon [email protected] rbd 0.53

ceph.conf

journal dio = false
filestore fiemap = false
filestore flusher = false
osd op threads = 24
osd disk threads = 24
filestore op threads = 6

kvm host is : 4 x 12 cores opteron
------------

During the bench:

on ceph nodes:
- cpu is around 10% used
- iostat show no disks activity on osds. (so I think that the 1G file is handle in the linux buffer)

on kvm host:

-cpu is around 20% used

I really don't see where is the bottleneck....

Any Ideas, hints ?

Quoted 6 lines

Quoted 6 lines

Regards,

Alexandre
--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected] More

majordomo

Quoted 6 lines

Quoted 3 lines

info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected] More

majordomo

Quoted 1 line

info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected] More majordomo info at
http://vger.kernel.org/majordomo-info.html

I do not really understand that network latency argument.
If one can get 40K iops with iSCSI, why can't I get the same with rados/ceph?
Note: network latency is the same in both cases
What do I miss?

I do not really understand that network latency argument.

If one can get 40K iops with iSCSI, why can't I get the same with rados/ceph?

Note: network latency is the same in both cases

What do I miss?

Quoted 162 lines

-----Original Message-----
From: [email protected] [mailto:ceph-devel- [email protected]] On Behalf Of Alexandre DERUMIER
Sent: Mittwoch, 31. Oktober 2012 18:27
To: Marcus Sorensen
Cc: Sage Weil; ceph-devel
Subject: Re: slow fio random read benchmark, need help

Thanks Marcus,

indeed gigabit ethernet.

note that my iscsi results (40k)was with multipath, so multiple gigabit links.

I have also done tests with a netapp array, with nfs, single link, I'm around 13000 iops

I will do more tests with multiples vms, from differents hosts, and with -- numjobs.

I'll keep you in touch,

Thanks for help,

Regards,

Alexandre

----- Mail original -----

De: "Marcus Sorensen" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "Sage Weil" <[email protected]>, "ceph-devel" <ceph- [email protected]>
Envoyé: Mercredi 31 Octobre 2012 18:08:11
Objet: Re: slow fio random read benchmark, need help

5000 is actually really good, if you ask me. Assuming everything is connected via gigabit. If you get 40k iops locally, you add the latency of tcp, as well as that of the ceph services and VM layer, and that's what you get. On my network I get about a .1ms round trip on gigabit over the same switch, which by definition can only do 10,000 iops. Then if you have storage on the other end capable of 40k iops, you add the latencies together (.1ms + .025ms) and you're at 8k iops.
Then add the small latency of the application servicing the io (NFS, Ceph, etc), and the latency introduced by your VM layer, and 5k sounds about right.

The good news is that you probably aren't taxing the storage, you can likely do many simultaneous tests from several VMs and get the same results.

You can try adding --numjobs to your fio to parallelize the specific test you're doing, or launching a second VM and doing the same test at the same time. This would be a good indicator if it's latency.

Quoted 4 lines · Alexandre DERUMIER

Quoted 1 line

Quoted 1 line

Have you tried increasing the iodepth?

Yes, I have try with 100 and 200, same results.

I have also try directly from the host, with /dev/rbd1, and I have same

result.

Quoted 6 lines

I have also try with 3 differents hosts, with differents cpus models.

(note: I can reach around 40.000 iops with same fio config on a zfs iscsi array)

My test ceph cluster nodes cpus are old (xeon E5420), but they are around

10% usage, so I think it's ok.

Quoted 74 lines

Do you have an idea if I can trace something ?

Thanks,

Alexandre

----- Mail original -----

De: "Sage Weil" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 16:57:05
Objet: Re: slow fio random read benchmark, need help

Quoted 7 lines · Alexandre DERUMIER

Hello,

I'm doing some tests with fio from a qemu 1.2 guest (virtio disk,cache=none), randread, with 4K block size on a small size of 1G (so it can be handle by the buffer cache on ceph cluster)

Quoted 6 lines

fio --filename=/dev/vdb -rw=randread --bs=4K --size=1000M --iodepth=40 --group_reporting --name=file1 --ioengine=libaio --direct=1

I can't get more than 5000 iops.

Have you tried increasing the iodepth?

sage

Quoted 39 lines

RBD cluster is :
---------------
3 nodes,with each node :
-6 x osd 15k drives (xfs), journal on tmpfs, 1 mon -cpu: 2x 4 cores intel xeon [email protected] rbd 0.53

ceph.conf

journal dio = false
filestore fiemap = false
filestore flusher = false
osd op threads = 24
osd disk threads = 24
filestore op threads = 6

kvm host is : 4 x 12 cores opteron
------------

During the bench:

on ceph nodes:
- cpu is around 10% used
- iostat show no disks activity on osds. (so I think that the 1G file is handle in the linux buffer)

on kvm host:

-cpu is around 20% used

I really don't see where is the bottleneck....

Any Ideas, hints ?

Quoted 6 lines

Quoted 6 lines

Regards,

Alexandre
--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected] More

majordomo

Quoted 6 lines

Quoted 3 lines

info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected] More

majordomo

Quoted 1 line

info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected] More majordomo info at
http://vger.kernel.org/majordomo-info.html

I'm not sure that latency addition is quite correct. Most use cases cases do multiple IOs at the same time, and good benchmarks tend to reflect that.

I suspect the IO limitations here are a result of QEMU's storage handling (or possibly our client layer) more than anything else — Josh can talk about that more than I can, though!
-Greg

Quoted 1 line

Same results with rbd kernel driver without QEMU involved.

I'm not sure that latency addition is quite correct. Most use cases cases do multiple IOs at the same time, and good benchmarks tend to reflect that.

I suspect the IO limitations here are a result of QEMU's storage handling (or possibly our client layer) more than anything else — Josh can talk about that more than I can, though!
-Greg

Quoted 1 line

Same results with rbd kernel driver without QEMU involved.

I'm not sure that latency addition is quite correct. Most use cases cases do multiple IOs at the same time, and good benchmarks tend to reflect that.

I suspect the IO limitations here are a result of QEMU's storage handling (or possibly our client layer) more than anything else — Josh can talk about that more than I can, though!
-Greg

Quoted 1 line

Same results with rbd kernel driver without QEMU involved.

I'm not sure that latency addition is quite correct. Most use cases cases do multiple IOs at the same time, and good benchmarks tend to reflect that.

I suspect the IO limitations here are a result of QEMU's storage handling (or possibly our client layer) more than anything else — Josh can talk about that more than I can, though!
-Greg

I'm not sure that latency addition is quite correct. Most use cases cases do multiple IOs at the same time, and good benchmarks tend to reflect that.

I suspect the IO limitations here are a result of QEMU's storage handling (or possibly our client layer) more than anything else — Josh can talk about that more than I can, though!
-Greg

Quoted 171 lines · Dietmar Maurer

I do not really understand that network latency argument.

If one can get 40K iops with iSCSI, why can't I get the same with rados/ceph?

Note: network latency is the same in both cases

What do I miss?

Quoted 162 lines

-----Original Message-----
From: [email protected] [mailto:ceph-devel- [email protected]] On Behalf Of Alexandre DERUMIER
Sent: Mittwoch, 31. Oktober 2012 18:27
To: Marcus Sorensen
Cc: Sage Weil; ceph-devel
Subject: Re: slow fio random read benchmark, need help

Thanks Marcus,

indeed gigabit ethernet.

note that my iscsi results (40k)was with multipath, so multiple gigabit links.

I have also done tests with a netapp array, with nfs, single link, I'm around 13000 iops

I will do more tests with multiples vms, from differents hosts, and with -- numjobs.

I'll keep you in touch,

Thanks for help,

Regards,

Alexandre

----- Mail original -----

De: "Marcus Sorensen" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "Sage Weil" <[email protected]>, "ceph-devel" <ceph- [email protected]>
Envoyé: Mercredi 31 Octobre 2012 18:08:11
Objet: Re: slow fio random read benchmark, need help

5000 is actually really good, if you ask me. Assuming everything is connected via gigabit. If you get 40k iops locally, you add the latency of tcp, as well as that of the ceph services and VM layer, and that's what you get. On my network I get about a .1ms round trip on gigabit over the same switch, which by definition can only do 10,000 iops. Then if you have storage on the other end capable of 40k iops, you add the latencies together (.1ms + .025ms) and you're at 8k iops.
Then add the small latency of the application servicing the io (NFS, Ceph, etc), and the latency introduced by your VM layer, and 5k sounds about right.

The good news is that you probably aren't taxing the storage, you can likely do many simultaneous tests from several VMs and get the same results.

You can try adding --numjobs to your fio to parallelize the specific test you're doing, or launching a second VM and doing the same test at the same time. This would be a good indicator if it's latency.

Quoted 4 lines · Alexandre DERUMIER

Quoted 1 line

Quoted 1 line

Have you tried increasing the iodepth?

Yes, I have try with 100 and 200, same results.

I have also try directly from the host, with /dev/rbd1, and I have same

result.

Quoted 6 lines

I have also try with 3 differents hosts, with differents cpus models.

(note: I can reach around 40.000 iops with same fio config on a zfs iscsi array)

My test ceph cluster nodes cpus are old (xeon E5420), but they are around

10% usage, so I think it's ok.

Quoted 74 lines

Do you have an idea if I can trace something ?

Thanks,

Alexandre

----- Mail original -----

De: "Sage Weil" <[email protected]>
À: "Alexandre DERUMIER" <[email protected]>
Cc: "ceph-devel" <[email protected]> Envoyé: Mercredi 31 Octobre 2012 16:57:05
Objet: Re: slow fio random read benchmark, need help

Quoted 7 lines · Alexandre DERUMIER

Hello,

I'm doing some tests with fio from a qemu 1.2 guest (virtio disk,cache=none), randread, with 4K block size on a small size of 1G (so it can be handle by the buffer cache on ceph cluster)

Quoted 6 lines

fio --filename=/dev/vdb -rw=randread --bs=4K --size=1000M --iodepth=40 --group_reporting --name=file1 --ioengine=libaio --direct=1

I can't get more than 5000 iops.

Have you tried increasing the iodepth?

sage

Quoted 39 lines

RBD cluster is :
---------------
3 nodes,with each node :
-6 x osd 15k drives (xfs), journal on tmpfs, 1 mon -cpu: 2x 4 cores intel xeon [email protected] rbd 0.53

ceph.conf

journal dio = false
filestore fiemap = false
filestore flusher = false
osd op threads = 24
osd disk threads = 24
filestore op threads = 6

kvm host is : 4 x 12 cores opteron
------------

During the bench:

on ceph nodes:
- cpu is around 10% used
- iostat show no disks activity on osds. (so I think that the 1G file is handle in the linux buffer)

on kvm host:

-cpu is around 20% used

I really don't see where is the bottleneck....

Any Ideas, hints ?

Quoted 6 lines

Quoted 6 lines

Regards,

Alexandre
--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected] More

majordomo

Quoted 6 lines

Quoted 3 lines

info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected] More

majordomo

Quoted 1 line

info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected] More majordomo info at
http://vger.kernel.org/majordomo-info.html

For the record, I'm not saying that it's the entire reason why the performance is lower (obviously since iscsi is better), I'm just saying that when you're talking about high iops, adding 100us (best case gigabit) to each request and response is significant

I always thought a distributed block storage could do such things faster (or at least as fast) than a single centralized store?

I have some customers with some huge databases (too big to be handle in the bufer), require a lot of ios. (around 10K).

I have redone tests with 4 guest in parallel, I get 4 x 5000iops, so it seem to scale ! (and cpu is very low on the ceph cluster).

So I'll try some tricks, like raid over multiple rbd devices, maybe it'll help.

Quoted 1 line

Did your RAID setup improve anything?

Quoted 1 line

Have you tried scaling past 4 guests in parallel?

I always thought a distributed block storage could do such things faster (or at least as fast) than a single centralized store?

I have some customers with some huge databases (too big to be handle in the bufer), require a lot of ios. (around 10K).

I have redone tests with 4 guest in parallel, I get 4 x 5000iops, so it seem to scale ! (and cpu is very low on the ceph cluster).

So I'll try some tricks, like raid over multiple rbd devices, maybe it'll help.

Call the probe function for the parent device.

Signed-off-by Alex Elder <[email protected]>

---
  drivers/block/rbd.c |   79
+++++++++++++++++++++++++++++++++++++++++++++++++--
  1 file changed, 76 insertions(+), 3 deletions(-)

drivers/block/rbd.c +2 -0

index 04062c1..8ef13f72 100644
@@ -222,6 +222,7 @@ struct rbd_device {
 
  	struct rbd_spec		*parent_spec;
  	u64			parent_overlap;
+	struct rbd_device	*parent;
 
  	/* protects updating the header */
  	struct rw_semaphore     header_rwsem;

@@ -255,6 +256,7 @@ static ssize_t rbd_add(struct bus_type *bus, const
 char *buf,
  		       size_t count);
  static ssize_t rbd_remove(struct bus_type *bus, const char *buf,
  			  size_t count);
+static int rbd_dev_probe(struct rbd_device *rbd_dev);
 
  static struct bus_attribute rbd_bus_attrs[] = {

__ATTR(add, S_IWUSR, NULL, rbd_add),

drivers/block/rbd.c +9 -1

@@ -378,6 +380,13 @@ out_opt:
  	return ERR_PTR(ret);
  }
 
+static struct rbd_client *__rbd_get_client(struct rbd_client *rbdc)
+{
+	kref_get(&rbdc->kref);
+
+	return rbdc;
+}
+
  /*
   * Find a ceph client with specific addr and configuration.  If
   * found, bump its reference count.

@@ -393,7 +402,8 @@ static struct rbd_client *rbd_client_find(struct
 ceph_options *ceph_opts)
  	spin_lock(&rbd_client_list_lock);
  	list_for_each_entry(client_node, &rbd_client_list, node) {
  		if (!ceph_compare_options(ceph_opts, client_node->client)) {
-			kref_get(&client_node->kref);
+			__rbd_get_client(client_node);
+
  			found = true;
  			break;

  		}

drivers/block/rbd.c +5 -0

@@ -3311,6 +3321,11 @@ static int rbd_dev_image_id(struct rbd_device
 *rbd_dev)
  	void *response;
  	void *p;
 
+	/* If we already have it we don't need to look it up */
+
+	if (rbd_dev->spec->image_id)
+		return 0;
+
  	/*
  	 * When probing a parent image, the image id is already

* known (and the image name likely is not). There's no

drivers/block/rbd.c +28 -0

@@ -3492,6 +3507,9 @@ out_err:
 
  static int rbd_dev_probe_finish(struct rbd_device *rbd_dev)
  {
+	struct rbd_device *parent = NULL;
+	struct rbd_spec *parent_spec = NULL;
+	struct rbd_client *rbdc = NULL;
  	int ret;
 
  	/* no need to lock here, as rbd_dev is not registered yet */

@@ -3536,6 +3554,31 @@ static int rbd_dev_probe_finish(struct rbd_device
 *rbd_dev)
  	 * At this point cleanup in the event of an error is the job
  	 * of the sysfs code (initiated by rbd_bus_del_dev()).
  	 */
+	/* Probe the parent if there is one */
+
+	if (rbd_dev->parent_spec) {
+		/*
+		 * We need to pass a reference to the client and the
+		 * parent spec when creating the parent rbd_dev.
+		 * Images related by parent/child relationships
+		 * always share both.
+		 */
+		parent_spec = rbd_spec_get(rbd_dev->parent_spec);
+		rbdc = __rbd_get_client(rbd_dev->rbd_client);
+
+		parent = rbd_dev_create(rbdc, parent_spec);
+		if (!parent) {
+			ret = -ENOMEM;
+			goto err_out_spec;
+		}
+		rbdc = NULL;		/* parent now owns reference */
+		parent_spec = NULL;	/* parent now owns reference */
+		ret = rbd_dev_probe(parent);
+		if (ret < 0)
+			goto err_out_parent;
+		rbd_dev->parent = parent;
+	}
+
  	down_write(&rbd_dev->header_rwsem);
  	ret = rbd_dev_snaps_register(rbd_dev);

  	up_write(&rbd_dev->header_rwsem);

drivers/block/rbd.c +32 -2

@@ -3554,6 +3597,12 @@ static int rbd_dev_probe_finish(struct rbd_device
 *rbd_dev)
  		(unsigned long long) rbd_dev->mapping.size);
 
  	return ret;
+
+err_out_parent:
+	rbd_dev_destroy(parent);
+err_out_spec:
+	rbd_spec_put(parent_spec);
+	rbd_put_client(rbdc);
  err_out_bus:
  	/* this will also clean up rest of rbd_dev stuff */

@@ -3717,6 +3766,12 @@ static void rbd_dev_release(struct device *dev)
  	module_put(THIS_MODULE);
  }
 
+static void __rbd_remove(struct rbd_device *rbd_dev)
+{
+	rbd_remove_all_snaps(rbd_dev);
+	rbd_bus_del_dev(rbd_dev);
+}
+
  static ssize_t rbd_remove(struct bus_type *bus,
  			  const char *buf,
  			  size_t count)

@@ -3743,8 +3798,26 @@ static ssize_t rbd_remove(struct bus_type *bus,
  		goto done;
  	}
 
-	rbd_remove_all_snaps(rbd_dev);
-	rbd_bus_del_dev(rbd_dev);
+	while (rbd_dev->parent_spec) {
+		struct rbd_device *first = rbd_dev;
+		struct rbd_device *second = first->parent;
+		struct rbd_device *third;
+
+		/*
+		 * Follow to the parent with no grandparent and
+		 * remove it.
+		 */
+		while (second && (third = second->parent)) {
+			first = second;
+			second = third;
+		}
+		__rbd_remove(second);
+		rbd_spec_put(first->parent_spec);
+		first->parent_spec = NULL;
+		first->parent_overlap = 0;
+		first->parent = NULL;
+	}
+	__rbd_remove(rbd_dev);
 
  done:
  	mutex_unlock(&ctl_mutex);

This all makes sense, but it reminds me of another issue we'll need to address:

http://www.tracker.newdream.net/issues/2533

We don't need to watch the header of a parent snapshot, since it's immutable and guaranteed not to be deleted out from under us. This avoids the bug referenced above. So I guess rbd_dev_probe{_finish} can take a parameter telling them whether to watch the header or not.

We should check whether multiple mapped rbds (without layering) hit this issue as well, and if so, default to not sharing the ceph_client until the bug is fixed.

Quoted 167 lines

Call the probe function for the parent device.

Signed-off-by Alex Elder <[email protected]>

---
  drivers/block/rbd.c |   79
+++++++++++++++++++++++++++++++++++++++++++++++++--
  1 file changed, 76 insertions(+), 3 deletions(-)

drivers/block/rbd.c +2 -0

index 04062c1..8ef13f72 100644
@@ -222,6 +222,7 @@ struct rbd_device {
 
      struct rbd_spec        *parent_spec;
      u64            parent_overlap;
+    struct rbd_device    *parent;
 
      /* protects updating the header */
      struct rw_semaphore     header_rwsem;

@@ -255,6 +256,7 @@ static ssize_t rbd_add(struct bus_type *bus, const
 char *buf,
                 size_t count);
  static ssize_t rbd_remove(struct bus_type *bus, const char *buf,
                size_t count);
+static int rbd_dev_probe(struct rbd_device *rbd_dev);
 
  static struct bus_attribute rbd_bus_attrs[] = {

      __ATTR(add, S_IWUSR, NULL, rbd_add),

drivers/block/rbd.c +9 -1

@@ -378,6 +380,13 @@ out_opt:
      return ERR_PTR(ret);
  }
 
+static struct rbd_client *__rbd_get_client(struct rbd_client *rbdc)
+{
+    kref_get(&rbdc->kref);
+
+    return rbdc;
+}
+
  /*
   * Find a ceph client with specific addr and configuration.  If
   * found, bump its reference count.

@@ -393,7 +402,8 @@ static struct rbd_client *rbd_client_find(struct
 ceph_options *ceph_opts)
      spin_lock(&rbd_client_list_lock);
      list_for_each_entry(client_node, &rbd_client_list, node) {
          if (!ceph_compare_options(ceph_opts, client_node->client)) {
-            kref_get(&client_node->kref);
+            __rbd_get_client(client_node);
+
              found = true;
              break;

          }

drivers/block/rbd.c +5 -0

@@ -3311,6 +3321,11 @@ static int rbd_dev_image_id(struct rbd_device
 *rbd_dev)
      void *response;
      void *p;
 
+    /* If we already have it we don't need to look it up */
+
+    if (rbd_dev->spec->image_id)
+        return 0;
+
      /*
       * When probing a parent image, the image id is already

* known (and the image name likely is not). There's no

drivers/block/rbd.c +28 -0

@@ -3492,6 +3507,9 @@ out_err:
 
  static int rbd_dev_probe_finish(struct rbd_device *rbd_dev)
  {
+    struct rbd_device *parent = NULL;
+    struct rbd_spec *parent_spec = NULL;
+    struct rbd_client *rbdc = NULL;
      int ret;
 
      /* no need to lock here, as rbd_dev is not registered yet */

@@ -3536,6 +3554,31 @@ static int rbd_dev_probe_finish(struct rbd_device
 *rbd_dev)
       * At this point cleanup in the event of an error is the job
       * of the sysfs code (initiated by rbd_bus_del_dev()).
       */
+    /* Probe the parent if there is one */
+
+    if (rbd_dev->parent_spec) {
+        /*
+         * We need to pass a reference to the client and the
+         * parent spec when creating the parent rbd_dev.
+         * Images related by parent/child relationships
+         * always share both.
+         */
+        parent_spec = rbd_spec_get(rbd_dev->parent_spec);
+        rbdc = __rbd_get_client(rbd_dev->rbd_client);
+
+        parent = rbd_dev_create(rbdc, parent_spec);
+        if (!parent) {
+            ret = -ENOMEM;
+            goto err_out_spec;
+        }
+        rbdc = NULL;        /* parent now owns reference */
+        parent_spec = NULL;    /* parent now owns reference */
+        ret = rbd_dev_probe(parent);
+        if (ret < 0)
+            goto err_out_parent;
+        rbd_dev->parent = parent;
+    }
+
      down_write(&rbd_dev->header_rwsem);
      ret = rbd_dev_snaps_register(rbd_dev);

      up_write(&rbd_dev->header_rwsem);

drivers/block/rbd.c +32 -2

@@ -3554,6 +3597,12 @@ static int rbd_dev_probe_finish(struct rbd_device
 *rbd_dev)
          (unsigned long long) rbd_dev->mapping.size);
 
      return ret;
+
+err_out_parent:
+    rbd_dev_destroy(parent);
+err_out_spec:
+    rbd_spec_put(parent_spec);
+    rbd_put_client(rbdc);
  err_out_bus:
      /* this will also clean up rest of rbd_dev stuff */

@@ -3717,6 +3766,12 @@ static void rbd_dev_release(struct device *dev)
      module_put(THIS_MODULE);
  }
 
+static void __rbd_remove(struct rbd_device *rbd_dev)
+{
+    rbd_remove_all_snaps(rbd_dev);
+    rbd_bus_del_dev(rbd_dev);
+}
+
  static ssize_t rbd_remove(struct bus_type *bus,
                const char *buf,
                size_t count)

@@ -3743,8 +3798,26 @@ static ssize_t rbd_remove(struct bus_type *bus,
          goto done;
      }
 
-    rbd_remove_all_snaps(rbd_dev);
-    rbd_bus_del_dev(rbd_dev);
+    while (rbd_dev->parent_spec) {
+        struct rbd_device *first = rbd_dev;
+        struct rbd_device *second = first->parent;
+        struct rbd_device *third;
+
+        /*
+         * Follow to the parent with no grandparent and
+         * remove it.
+         */
+        while (second && (third = second->parent)) {
+            first = second;
+            second = third;
+        }
+        __rbd_remove(second);
+        rbd_spec_put(first->parent_spec);
+        first->parent_spec = NULL;
+        first->parent_overlap = 0;
+        first->parent = NULL;
+    }
+    __rbd_remove(rbd_dev);
 
  done:
      mutex_unlock(&ctl_mutex);