rbd: stop using ceph_calc_raw_layout()

This series makes rbd no longer call ceph_calc_raw_layout(), and in doing so, also stop calling ceph_calc_file_object_mapping() for its requests. Apparently the call to the former was for the *other* side-effects it had (unrelated to the layout).

-Alex

[PATCH 1/4] rbd: pull in ceph_calc_raw_layout() [PATCH 2/4] rbd: open code rbd_calc_raw_layout() [PATCH 3/4] rbd: don't bother calculating file mapping [PATCH 4/4] rbd: use a common layout for each device

This is the first in a series of patches aimed at eliminating the use of ceph_calc_raw_layout() by rbd.

It simply pulls in a copy of that function and renames it rbd_calc_raw_layout().

---
 drivers/block/rbd.c |   36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

index e1094ff..810b58d 100644
@@ -1103,6 +1103,40 @@ static void rbd_layout_init(struct

ceph_file_layout *layout, u64 pool_id)
layout->fl_pg_pool = cpu_to_le32((u32) pool_id);
}

+int rbd_calc_raw_layout(struct ceph_file_layout *layout,
+ u64 off, u64 *plen, u64 *bno,
+ struct ceph_osd_request *req,
+ struct ceph_osd_req_op *op)
+{
+ u64 orig_len = *plen;
+ u64 objoff, objlen; /* extent in object */
+ int r;
+
+ /* object extent? */
+ r = ceph_calc_file_object_mapping(layout, off, orig_len, bno,
+ &objoff, &objlen);
+ if (r < 0)
+ return r;
+ if (objlen < orig_len) {
+ *plen = objlen;
+ dout(" skipping last %llu, final file extent %llu~%llu\n",
+ orig_len - *plen, off, *plen);
+ }
+
+ if (op->op == CEPH_OSD_OP_READ || op->op == CEPH_OSD_OP_WRITE) {
+ op->extent.offset = objoff;
+ op->extent.length = objlen;
+ }
+ req->r_num_pages = calc_pages_for(off, *plen);
+ req->r_page_alignment = off & ~PAGE_MASK;
+ if (op->op == CEPH_OSD_OP_WRITE)
+ op->payload_len = *plen;
+
+ dout("calc_layout bno=%llx %llu~%llu (%d pages)\n",
+ *bno, objoff, objlen, req->r_num_pages);
+ return 0;
+}
+
/*
* Send ceph osd request
*/
@@ -1169,7 +1203,7 @@ static int rbd_do_request(struct request *rq,
osd_req->r_oid_len = strlen(osd_req->r_oid);

rbd_layout_init(&osd_req->r_file_layout, rbd_dev->spec->pool_id);
- ret = ceph_calc_raw_layout(&osd_req->r_file_layout,
+ ret = rbd_calc_raw_layout(&osd_req->r_file_layout,
ofs, &len, &bno, osd_req, op);
rbd_assert(ret == 0);

1.7.9.5

This patch gets rid of rbd_calc_raw_layout() by simply open coding it in its one caller.

---
 drivers/block/rbd.c |   55
+++++++++++++++++----------------------------------
 1 file changed, 18 insertions(+), 37 deletions(-)

index 810b58d..1afe51f 100644
@@ -1034,7 +1034,7 @@ static struct ceph_osd_req_op

*rbd_create_rw_op(int opcode, u32 payload_len)
return NULL;
/*
* op extent offset and length will be set later on
- * in calc_raw_layout()
+ * after ceph_calc_file_object_mapping().
*/
op->op = opcode;
op->payload_len = payload_len;
@@ -1103,40 +1103,6 @@ static void rbd_layout_init(struct ceph_file_layout *layout, u64 pool_id)
layout->fl_pg_pool = cpu_to_le32((u32) pool_id);
}

-int rbd_calc_raw_layout(struct ceph_file_layout *layout,
- u64 off, u64 *plen, u64 *bno,
- struct ceph_osd_request *req,
- struct ceph_osd_req_op *op)
-{
- u64 orig_len = *plen;
- u64 objoff, objlen; /* extent in object */
- int r;
-
- /* object extent? */
- r = ceph_calc_file_object_mapping(layout, off, orig_len, bno,
- &objoff, &objlen);
- if (r < 0)
- return r;
- if (objlen < orig_len) {
- *plen = objlen;
- dout(" skipping last %llu, final file extent %llu~%llu\n",
- orig_len - *plen, off, *plen);
- }
-
- if (op->op == CEPH_OSD_OP_READ || op->op == CEPH_OSD_OP_WRITE) {
- op->extent.offset = objoff;
- op->extent.length = objlen;
- }
- req->r_num_pages = calc_pages_for(off, *plen);
- req->r_page_alignment = off & ~PAGE_MASK;
- if (op->op == CEPH_OSD_OP_WRITE)
- op->payload_len = *plen;
-
- dout("calc_layout bno=%llx %llu~%llu (%d pages)\n",
- *bno, objoff, objlen, req->r_num_pages);
- return 0;
-}
-
/*
* Send ceph osd request
*/
@@ -1160,6 +1126,8 @@ static int rbd_do_request(struct request *rq,
struct ceph_osd_request *osd_req;
int ret;
u64 bno;
+ u64 obj_off = 0;
+ u64 obj_len = 0;
struct timespec mtime = CURRENT_TIME;
struct rbd_request *rbd_req;
struct ceph_osd_client *osdc;
@@ -1203,9 +1171,22 @@ static int rbd_do_request(struct request *rq,
osd_req->r_oid_len = strlen(osd_req->r_oid);

rbd_layout_init(&osd_req->r_file_layout, rbd_dev->spec->pool_id);
- ret = rbd_calc_raw_layout(&osd_req->r_file_layout,
- ofs, &len, &bno, osd_req, op);
+ ret = ceph_calc_file_object_mapping(&osd_req->r_file_layout, ofs, len,
+ &bno, &obj_off, &obj_len);
rbd_assert(ret == 0);
+ if (obj_len < len) {
+ dout(" skipping last %llu, final file extent %llu~%llu\n",
+ len - obj_len, ofs, obj_len);
+ len = obj_len;
+ }
+ if (op->op == CEPH_OSD_OP_READ || op->op == CEPH_OSD_OP_WRITE) {
+ op->extent.offset = obj_off;
+ op->extent.length = obj_len;
+ if (op->op == CEPH_OSD_OP_WRITE)
+ op->payload_len = obj_len;
+ }
+ osd_req->r_num_pages = calc_pages_for(ofs, len);
+ osd_req->r_page_alignment = ofs & ~PAGE_MASK;

ceph_osdc_build_request(osd_req, ofs, len, 1, op,
snapc, snapid, &mtime);

1.7.9.5

When rbd_do_request() has a request to process it initializes a ceph file layout structure and uses it to compute offsets and limits for the range of the request using ceph_calc_file_object_mapping().

The layout used is fixed, and is based on RBD_MAX_OBJ_ORDER (30). It sets the layout's object size and stripe unit to be 1 GB (2^30), and sets the stripe count to be 1.

The job of ceph_calc_file_object_mapping() is to determine which of a sequence of objects will contain data covered by range, and within that object, at what offset the range starts. It also truncates the length of the range at the end of the selected object if necessary.

This is needed for ceph fs, but for rbd it really serves no purpose. It does its own blocking of images into objects, echo of which is (1 << obj_order) in size, and as a result it ignores the "bno" value returned by ceph_calc_file_object_mapping(). In addition, by the point a request has reached this function, it is already destined for a single rbd object, and its length will not exceed that object's extent. Because of this, and because the mapping will result in blocking up the range using an integer multiple of the image's object order, ceph_calc_file_object_mapping() will never change the offset or length values defined by the request.

In other words, this call is a big no-op for rbd data requests.

There is one exception. We read the header object using this function, and in that case we will not have already limited the request size. However, the header is a single object (not a file or rbd image), and should not be broken into pieces anyway. So in fact we should *not* be calling ceph_calc_file_object_mapping() when operating on the header object.

So...

Don't call ceph_calc_file_object_mapping() in rbd_do_request(), because useless for image data and incorrect to do sofor the image header.

---
 drivers/block/rbd.c |   18 ++++--------------
 1 file changed, 4 insertions(+), 14 deletions(-)

index 1afe51f..30a73ae 100644
@@ -1125,9 +1125,6 @@ static int rbd_do_request(struct request *rq,
 {
 	struct ceph_osd_request *osd_req;
 	int ret;
-	u64 bno;
-	u64 obj_off = 0;
-	u64 obj_len = 0;
 	struct timespec mtime = CURRENT_TIME;
 	struct rbd_request *rbd_req;
 	struct ceph_osd_client *osdc;

@@ -1171,19 +1168,12 @@ static int rbd_do_request(struct request *rq,
 	osd_req->r_oid_len = strlen(osd_req->r_oid);
 
 	rbd_layout_init(&osd_req->r_file_layout, rbd_dev->spec->pool_id);
-	ret = ceph_calc_file_object_mapping(&osd_req->r_file_layout, ofs, len,
-						&bno, &obj_off, &obj_len);
-	rbd_assert(ret == 0);
-	if (obj_len < len) {
-		dout(" skipping last %llu, final file extent %llu~%llu\n",
-		     len - obj_len, ofs, obj_len);
-		len = obj_len;
-	}
+
 	if (op->op == CEPH_OSD_OP_READ || op->op == CEPH_OSD_OP_WRITE) {
-		op->extent.offset = obj_off;
-		op->extent.length = obj_len;
+		op->extent.offset = ofs;
+		op->extent.length = len;
 		if (op->op == CEPH_OSD_OP_WRITE)
-			op->payload_len = obj_len;
+			op->payload_len = len;
 	}
 	osd_req->r_num_pages = calc_pages_for(ofs, len);
 	osd_req->r_page_alignment = ofs & ~PAGE_MASK;

1.7.9.5

Each osd message includes a layout structure, and for rbd it is always the same (at least for osd's in a given pool).

Initialize a layout structure when an rbd_dev gets created and just copy that into osd requests for the rbd image.

Replace an assertion that was done when initializing the layout structures with code that catches and handles anything that would trigger the assertion as soon as it is identified. This precludes that (bad) condition from ever occurring.

---
 drivers/block/rbd.c |   34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

index 30a73ae..fba0822 100644
@@ -235,6 +235,8 @@ struct rbd_device {
 
 	char			*header_name;
 
+	struct ceph_file_layout	layout;
+
 	struct ceph_osd_event   *watch_event;
 	struct ceph_osd_request *watch_request;
 

@@ -1093,16 +1095,6 @@ static void rbd_coll_end_req(struct rbd_request

*rbd_req,
ret, len);
}

-static void rbd_layout_init(struct ceph_file_layout *layout, u64 pool_id) -{
- memset(layout, 0, sizeof (*layout));
- layout->fl_stripe_unit = cpu_to_le32(1 << RBD_MAX_OBJ_ORDER);
- layout->fl_stripe_count = cpu_to_le32(1);
- layout->fl_object_size = cpu_to_le32(1 << RBD_MAX_OBJ_ORDER);
- rbd_assert(pool_id <= (u64) U32_MAX);
- layout->fl_pg_pool = cpu_to_le32((u32) pool_id);
-}
-
/*
* Send ceph osd request
*/
@@ -1167,7 +1159,7 @@ static int rbd_do_request(struct request *rq,
strncpy(osd_req->r_oid, object_name, sizeof(osd_req->r_oid)); osd_req->r_oid_len = strlen(osd_req->r_oid);

- rbd_layout_init(&osd_req->r_file_layout, rbd_dev->spec->pool_id);
+ osd_req->r_file_layout = rbd_dev->layout; /* struct */

if (op->op == CEPH_OSD_OP_READ || op->op == CEPH_OSD_OP_WRITE) {
op->extent.offset = ofs;
@@ -2266,6 +2258,13 @@ struct rbd_device *rbd_dev_create(struct rbd_client *rbdc,
rbd_dev->spec = spec;
rbd_dev->rbd_client = rbdc;

+ /* Initialize the layout used for all rbd requests */
+
+ rbd_dev->layout.fl_stripe_unit = cpu_to_le32(1 << RBD_MAX_OBJ_ORDER);
+ rbd_dev->layout.fl_stripe_count = cpu_to_le32(1);
+ rbd_dev->layout.fl_object_size = cpu_to_le32(1 << RBD_MAX_OBJ_ORDER);
+ rbd_dev->layout.fl_pg_pool = cpu_to_le32((u32) spec->pool_id);
+
return rbd_dev;
}

@@ -2520,6 +2519,12 @@ static int rbd_dev_v2_parent_info(struct rbd_device *rbd_dev)
if (parent_spec->pool_id == CEPH_NOPOOL)
goto out; /* No parent? No problem. */

+ /* The ceph file layout needs to fit pool id in 32 bits */
+
+ ret = -EIO;
+ if (WARN_ON(parent_spec->pool_id > (u64) U32_MAX))
+ goto out;
+
image_id = ceph_extract_encoded_string(&p, end, NULL, GFP_KERNEL); if (IS_ERR(image_id)) {
ret = PTR_ERR(image_id);
@@ -3648,6 +3653,13 @@ static ssize_t rbd_add(struct bus_type *bus,
if (spec->pool_id == CEPH_NOPOOL)
goto err_out_client;

+ /* The ceph file layout needs to fit pool id in 32 bits */
+
+ if (WARN_ON(spec->pool_id > (u64) U32_MAX)) {
+ rc = -EIO;
+ goto err_out_client;
+ }
+
rbd_dev = rbd_dev_create(rbdc, spec);
if (!rbd_dev)
goto err_out_client;

1.7.9.5