Skip to contentSkip to conversation

libceph: abstract setting message data info

PATCH In review Alex Elder 2 participants 6 messages Feb 25, 2013 – Feb 26, 2013

Subsystems (2)
lore
Share & tools
lore

Alex Elder

author
#0 lore raw

cover libceph: abstract setting message data info

This series makes the fields related to the data portion of a ceph message not get manipulated by code outside the ceph messenger. It implements some interface functions that can be used to assign data-related fields. Doing this will allow the way message data is managed to be changed independent of the users of the messenger module.

-Alex

[PATCH 1/4] libceph: distinguish page array and pagelist count [PATCH 2/4] libceph: set page alignment in start_request() [PATCH 3/4] libceph: isolate message page field manipulation [PATCH 4/4] libceph: isolate other message data fields

author
#1 lore raw

PATCH 1/4 libceph: distinguish page array and pagelist count

Use distinct fields for tracking the number of pages in a message's page array and in a message's page list. Currently only one or the other is used at a time, but that will be changing soon.

---
 fs/ceph/mds_client.c           |    4 ++--
 include/linux/ceph/messenger.h |    3 ++-
 net/ceph/messenger.c           |   14 ++++++++------
 net/ceph/osd_client.c          |    4 ++--
 4 files changed, 14 insertions(+), 11 deletions(-)
fs/ceph/mds_client.c +0 -0
index d958420..44435c2 100644
@@ -1719,7 +1719,7 @@ static struct ceph_msg

*create_request_message(struct ceph_mds_client *mdsc,
msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);

msg->pages = req->r_pages;
- msg->nr_pages = req->r_num_pages;
+ msg->page_count = req->r_num_pages;
msg->hdr.data_len = cpu_to_le32(req->r_data_len); msg->hdr.data_off = cpu_to_le16(0);

@@ -2600,10 +2600,10 @@ static void send_mds_reconnect(struct ceph_mds_client *mdsc,
}

reply->pagelist = pagelist;
+ reply->pagelist_count = calc_pages_for(0, pagelist->length);
if (recon_state.flock)
reply->hdr.version = cpu_to_le16(2);
reply->hdr.data_len = cpu_to_le32(pagelist->length);
- reply->nr_pages = calc_pages_for(0, pagelist->length);
ceph_con_send(&session->s_con, reply);

mutex_unlock(&session->s_mutex);

include/linux/ceph/messenger.h +2 -1
index 8297288..1b08349 100644
@@ -75,9 +75,10 @@ struct ceph_msg {
 	struct kvec front;              /* unaligned blobs of message */
 	struct ceph_buffer *middle;
 	struct page **pages;            /* data payload.  NOT OWNER. */
-	unsigned nr_pages;              /* size of page array */
+	unsigned page_count;		/* size of page array */
 	unsigned page_alignment;        /* io offset in first page */
 	struct ceph_pagelist *pagelist; /* instead of pages */
+	unsigned int pagelist_count;	/* number of pages in pagelist */
 
 	struct ceph_connection *con;
 	struct list_head list_head;
net/ceph/messenger.c +0 -0
index c06f940..9d8abb0 100644
@@ -813,7 +813,7 @@ static void prepare_write_message(struct

ceph_connection *con)
m, con->out_seq, le16_to_cpu(m->hdr.type), le32_to_cpu(m->hdr.front_len), le32_to_cpu(m->hdr.middle_len), le32_to_cpu(m->hdr.data_len),
- m->nr_pages);
+ m->page_count);
BUG_ON(le32_to_cpu(m->hdr.front_len) != m->front.iov_len);

/* tag + hdr + front + middle */
@@ -1072,7 +1072,7 @@ static int write_partial_msg_pages(struct ceph_connection *con)
const size_t trail_off = data_len - trail_len;

dout("write_partial_msg_pages %p msg %p page %d/%d offset %d\n",
- con, msg, con->out_msg_pos.page, msg->nr_pages,
+ con, msg, con->out_msg_pos.page, msg->page_count,
con->out_msg_pos.page_pos);

/*
@@ -2715,9 +2715,10 @@ struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
m->middle = NULL;

/* data */
- m->nr_pages = 0;
+ m->page_count = 0;
m->page_alignment = 0;
m->pages = NULL;
+ m->pagelist_count = 0;
m->pagelist = NULL;
#ifdef CONFIG_BLOCK
m->bio = NULL;
@@ -2890,13 +2891,14 @@ void ceph_msg_last_put(struct kref *kref)
ceph_buffer_put(m->middle);
m->middle = NULL;
}
- m->nr_pages = 0;
+ m->page_count = 0;
m->pages = NULL;

if (m->pagelist) {
ceph_pagelist_release(m->pagelist); kfree(m->pagelist); m->pagelist = NULL;
+ m->pagelist_count = 0;
}

m->trail = NULL;
@@ -2910,8 +2912,8 @@ EXPORT_SYMBOL(ceph_msg_last_put);

void ceph_msg_dump(struct ceph_msg *msg)
{
- pr_debug("msg_dump %p (front_max %d nr_pages %d)\n", msg,
- msg->front_max, msg->nr_pages);
+ pr_debug("msg_dump %p (front_max %d page_count %d)\n", msg,
+ msg->front_max, msg->page_count);
print_hex_dump(KERN_DEBUG, "header: ",
DUMP_PREFIX_OFFSET, 16, 1, &msg->hdr, sizeof(msg->hdr), true);

net/ceph/osd_client.c +0 -0
index 980911e..0a4d4a0 100644
@@ -1631,7 +1631,7 @@ int ceph_osdc_start_request(struct ceph_osd_client

*osdc,
int rc = 0;

req->r_request->pages = req->r_pages;
- req->r_request->nr_pages = req->r_num_pages;
+ req->r_request->page_count = req->r_num_pages;
#ifdef CONFIG_BLOCK
req->r_request->bio = req->r_bio;
#endif
@@ -1982,7 +1982,7 @@ static struct ceph_msg *get_reply(struct ceph_connection *con,
goto out;
}
m->pages = req->r_pages;
- m->nr_pages = req->r_num_pages;
+ m->page_count = req->r_num_pages;
m->page_alignment = req->r_page_alignment;
#ifdef CONFIG_BLOCK
m->bio = req->r_bio;

Signature · 1 line
1.7.9.5
author
#2 lore raw

PATCH 2/4 libceph: set page alignment in start_request()

In reply to Alex Elder

The page alignment field for a request is currently set in ceph_osdc_build_request(). It's not needed at that point nor do either of its callers need that value assigned at any point before they call ceph_osdc_start_request().

So move that assignment into ceph_osdc_start_request().

---
 net/ceph/osd_client.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
net/ceph/osd_client.c +0 -0
index 0a4d4a0..1bb2b59 100644
@@ -369,7 +369,6 @@ void ceph_osdc_build_request(struct ceph_osd_request

*req,
data_len += len;
}
req->r_request->hdr.data_len = cpu_to_le32(data_len);
- req->r_request->page_alignment = req->r_page_alignment;

BUG_ON(p > msg->front.iov_base + msg->front.iov_len); msg_size = p - msg->front.iov_base;
@@ -1632,6 +1631,7 @@ int ceph_osdc_start_request(struct ceph_osd_client *osdc,

req->r_request->pages = req->r_pages; req->r_request->page_count = req->r_num_pages;
+ req->r_request->page_alignment = req->r_page_alignment;
#ifdef CONFIG_BLOCK
req->r_request->bio = req->r_bio;
#endif

Signature · 1 line
1.7.9.5
author
#3 lore raw

PATCH 3/4 libceph: isolate message page field manipulation

In reply to Alex Elder

Define a function ceph_msg_data_set_pages(), which more clearly abstracts the assignment page-related fields for data in a ceph message structure. These fields should never be set more than once (add BUG_ON() calls to guarantee that). Use this new function in the osd client and mds client.

Rearrange the field order in a ceph_msg structure to group those that are used to define the possible data payloads.

---
 fs/ceph/mds_client.c           |    4 ++--
 include/linux/ceph/messenger.h |   22 +++++++++++++---------
 net/ceph/messenger.c           |   12 ++++++++++++
 net/ceph/osd_client.c          |   11 +++++------
 4 files changed, 32 insertions(+), 17 deletions(-)
fs/ceph/mds_client.c +0 -0
index 44435c2..d8842a1 100644
@@ -1718,8 +1718,8 @@ static struct ceph_msg

*create_request_message(struct ceph_mds_client *mdsc,
msg->front.iov_len = p - msg->front.iov_base; msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);

- msg->pages = req->r_pages;
- msg->page_count = req->r_num_pages;
+ ceph_msg_data_set_pages(msg, req->r_pages, req->r_num_pages, 0);
+
msg->hdr.data_len = cpu_to_le32(req->r_data_len); msg->hdr.data_off = cpu_to_le16(0);

include/linux/ceph/messenger.h +10 -9
index 1b08349..eeb28a0 100644
@@ -74,22 +74,23 @@ struct ceph_msg {
 	struct ceph_msg_footer footer;	/* footer */
 	struct kvec front;              /* unaligned blobs of message */
 	struct ceph_buffer *middle;
-	struct page **pages;            /* data payload.  NOT OWNER. */
-	unsigned page_count;		/* size of page array */
-	unsigned page_alignment;        /* io offset in first page */
+
+	struct page **pages;		/* data payload.  NOT OWNER. */
+	unsigned int page_alignment;	/* io offset in first page */
+	unsigned int page_count;	/* # pages in array or list */
 	struct ceph_pagelist *pagelist; /* instead of pages */
 	unsigned int pagelist_count;	/* number of pages in pagelist */
-
-	struct ceph_connection *con;
-	struct list_head list_head;
-
-	struct kref kref;
 #ifdef CONFIG_BLOCK
+	unsigned int bio_seg;		/* current bio segment */
 	struct bio  *bio;		/* instead of pages/pagelist */
 	struct bio  *bio_iter;		/* bio iterator */
-	unsigned int bio_seg;		/* current bio segment */
 #endif /* CONFIG_BLOCK */
 	struct ceph_pagelist *trail;	/* the trailing part of the data */
+
+	struct ceph_connection *con;
+	struct list_head list_head;
+
+	struct kref kref;
 	bool front_is_vmalloc;
 	bool more_to_follow;
 	bool needs_out_seq;
@@ -219,6 +220,9 @@ extern void ceph_msg_revoke_incoming(struct ceph_msg

*msg);

extern void ceph_con_keepalive(struct ceph_connection *con);

+extern void ceph_msg_data_set_pages(struct ceph_msg *msg, struct page **pages,
+ unsigned int page_count, size_t alignment);
+
extern struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
bool can_fail);
extern void ceph_msg_kfree(struct ceph_msg *m);

net/ceph/messenger.c +12 -0
index 9d8abb0..7328e1b 100644
@@ -2674,6 +2674,18 @@ void ceph_con_keepalive(struct ceph_connection *con)
 }
 EXPORT_SYMBOL(ceph_con_keepalive);
 
+void ceph_msg_data_set_pages(struct ceph_msg *msg, struct page **pages,
+		unsigned int page_count, size_t alignment)
+{
+	BUG_ON(msg->pages);
+	BUG_ON(msg->page_count);
+	BUG_ON(alignment >= PAGE_SIZE);
+
+	msg->pages = pages;
+	msg->page_count = page_count;
+	msg->page_alignment = alignment;
+}
+EXPORT_SYMBOL(ceph_msg_data_set_pages);
 
 /*
  * construct a new message with given type, size
net/ceph/osd_client.c +0 -0
index 1bb2b59..0f8351d 100644
@@ -1629,9 +1629,8 @@ int ceph_osdc_start_request(struct ceph_osd_client

*osdc,
{
int rc = 0;

- req->r_request->pages = req->r_pages;
- req->r_request->page_count = req->r_num_pages;
- req->r_request->page_alignment = req->r_page_alignment;
+ ceph_msg_data_set_pages(req->r_request, req->r_pages, req->r_num_pages,
+ req->r_page_alignment);
#ifdef CONFIG_BLOCK
req->r_request->bio = req->r_bio;
#endif
@@ -1981,9 +1980,9 @@ static struct ceph_msg *get_reply(struct ceph_connection *con,
m = NULL;
goto out;
}
- m->pages = req->r_pages;
- m->page_count = req->r_num_pages;
- m->page_alignment = req->r_page_alignment;
+
+ ceph_msg_data_set_pages(m, req->r_pages, req->r_num_pages,
+ req->r_page_alignment);
#ifdef CONFIG_BLOCK
m->bio = req->r_bio;
#endif

Signature · 1 line
1.7.9.5
author
#4 lore raw

PATCH 4/4 libceph: isolate other message data fields

In reply to Alex Elder

Define ceph_msg_data_set_pagelist(), ceph_msg_data_set_bio(), and ceph_msg_data_set_trail() to clearly abstract the assignment the remaining data-related fields in a ceph message structure. These fields should never be used more than once; add BUG_ON() calls to guarantee this. Use the new functions in the osd client and mds client.

---
 fs/ceph/mds_client.c           |    4 ++--
 include/linux/ceph/messenger.h |    6 ++++++
 net/ceph/messenger.c           |   28 ++++++++++++++++++++++++++++
 net/ceph/osd_client.c          |    6 +++---
 4 files changed, 39 insertions(+), 5 deletions(-)
fs/ceph/mds_client.c +0 -0
index d8842a1..c0eb44d 100644
@@ -2599,8 +2599,8 @@ static void send_mds_reconnect(struct

ceph_mds_client *mdsc,
goto fail;
}

- reply->pagelist = pagelist;
- reply->pagelist_count = calc_pages_for(0, pagelist->length);
+ ceph_msg_data_set_pagelist(reply, pagelist,
+ calc_pages_for(0, pagelist->length));
if (recon_state.flock)
reply->hdr.version = cpu_to_le16(2);
reply->hdr.data_len = cpu_to_le32(pagelist->length);

include/linux/ceph/messenger.h +0 -0
index eeb28a0..3cb3e8c 100644
@@ -222,6 +222,12 @@ extern void ceph_con_keepalive(struct

ceph_connection *con);

extern void ceph_msg_data_set_pages(struct ceph_msg *msg, struct page
**pages,
unsigned int page_count, size_t alignment);
+extern void ceph_msg_data_set_pagelist(struct ceph_msg *msg,
+ struct ceph_pagelist *pagelist,
+ unsigned int page_count);
+extern void ceph_msg_data_set_bio(struct ceph_msg *msg, struct bio *bio); +extern void ceph_msg_data_set_trail(struct ceph_msg *msg,
+ struct ceph_pagelist *trail);

extern struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
bool can_fail);

net/ceph/messenger.c +0 -0
index 7328e1b..82e3212 100644
@@ -2687,6 +2687,34 @@ void ceph_msg_data_set_pages(struct ceph_msg

*msg, struct page **pages,
}
EXPORT_SYMBOL(ceph_msg_data_set_pages);

+void ceph_msg_data_set_pagelist(struct ceph_msg *msg,
+ struct ceph_pagelist *pagelist,
+ unsigned int pagelist_count)
+{
+ BUG_ON(msg->pagelist);
+ BUG_ON(msg->pagelist_count);
+
+ msg->pagelist = pagelist;
+ msg->pagelist_count = pagelist_count;
+}
+EXPORT_SYMBOL(ceph_msg_data_set_pagelist);
+
+void ceph_msg_data_set_bio(struct ceph_msg *msg, struct bio *bio) +{
+ BUG_ON(msg->bio);
+
+ msg->bio = bio;
+}
+EXPORT_SYMBOL(ceph_msg_data_set_bio);
+
+void ceph_msg_data_set_trail(struct ceph_msg *msg, struct ceph_pagelist *trail)
+{
+ BUG_ON(msg->trail);
+
+ msg->trail = trail;
+}
+EXPORT_SYMBOL(ceph_msg_data_set_trail);
+
/*
* construct a new message with given type, size
* the new msg has a ref count of 1.

net/ceph/osd_client.c +0 -0
index 0f8351d..f984500 100644
@@ -1632,9 +1632,9 @@ int ceph_osdc_start_request(struct ceph_osd_client

*osdc,
ceph_msg_data_set_pages(req->r_request, req->r_pages, req->r_num_pages,
req->r_page_alignment);
#ifdef CONFIG_BLOCK
- req->r_request->bio = req->r_bio;
+ ceph_msg_data_set_bio(req->r_request, req->r_bio);
#endif
- req->r_request->trail = &req->r_trail;
+ ceph_msg_data_set_trail(req->r_request, &req->r_trail);

register_request(osdc, req);

@@ -1984,7 +1984,7 @@ static struct ceph_msg *get_reply(struct ceph_connection *con,
ceph_msg_data_set_pages(m, req->r_pages, req->r_num_pages,
req->r_page_alignment);
#ifdef CONFIG_BLOCK
- m->bio = req->r_bio;
+ ceph_msg_data_set_bio(m, req->r_bio);
#endif
}
*skip = 0;

Signature · 1 line
1.7.9.5
Reviewed-by on the v1 series
#5 lore raw
In reply to Alex Elder
Quoted 7 lines · Alex Elder

This series makes the fields related to the data portion of a ceph message not get manipulated by code outside the ceph messenger. It implements some interface functions that can be used to assign data-related fields. Doing this will allow the way message data is managed to be changed independent of the users of the messenger module.

Quoted 6 lines

-Alex

[PATCH 1/4] libceph: distinguish page array and pagelist count [PATCH 2/4] libceph: set page alignment in start_request() [PATCH 3/4] libceph: isolate message page field manipulation [PATCH 4/4] libceph: isolate other message data fields

These all look good.