libceph: abstract setting message data info
Subsystems (2)
cover libceph: abstract setting message data info
This series makes the fields related to the data portion of a ceph message not get manipulated by code outside the ceph messenger. It implements some interface functions that can be used to assign data-related fields. Doing this will allow the way message data is managed to be changed independent of the users of the messenger module.
-Alex
[PATCH 1/4] libceph: distinguish page array and pagelist count [PATCH 2/4] libceph: set page alignment in start_request() [PATCH 3/4] libceph: isolate message page field manipulation [PATCH 4/4] libceph: isolate other message data fields
PATCH 1/4 libceph: distinguish page array and pagelist count
Use distinct fields for tracking the number of pages in a message's page array and in a message's page list. Currently only one or the other is used at a time, but that will be changing soon.
--- fs/ceph/mds_client.c | 4 ++-- include/linux/ceph/messenger.h | 3 ++- net/ceph/messenger.c | 14 ++++++++------ net/ceph/osd_client.c | 4 ++-- 4 files changed, 14 insertions(+), 11 deletions(-)
fs/ceph/mds_client.c +0 -0
@@ -1719,7 +1719,7 @@ static struct ceph_msg
*create_request_message(struct ceph_mds_client *mdsc,
msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
msg->pages = req->r_pages;
- msg->nr_pages = req->r_num_pages;
+ msg->page_count = req->r_num_pages;
msg->hdr.data_len = cpu_to_le32(req->r_data_len); msg->hdr.data_off = cpu_to_le16(0);
@@ -2600,10 +2600,10 @@ static void send_mds_reconnect(struct ceph_mds_client *mdsc,
}
reply->pagelist = pagelist;
+ reply->pagelist_count = calc_pages_for(0, pagelist->length);
if (recon_state.flock)
reply->hdr.version = cpu_to_le16(2);
reply->hdr.data_len = cpu_to_le32(pagelist->length);
- reply->nr_pages = calc_pages_for(0, pagelist->length);
ceph_con_send(&session->s_con, reply);
mutex_unlock(&session->s_mutex);
include/linux/ceph/messenger.h +2 -1
@@ -75,9 +75,10 @@ struct ceph_msg {struct kvec front;/* unaligned blobs of message */ struct ceph_buffer *middle;struct page **pages;/* data payload. NOT OWNER. */ -unsigned nr_pages;/* size of page array */ +unsigned page_count;/* size of page array */ unsigned page_alignment;/* io offset in first page */ struct ceph_pagelist *pagelist;/* instead of pages */ +unsigned int pagelist_count;/* number of pages in pagelist */ struct ceph_connection *con;struct list_head list_head;
net/ceph/messenger.c +0 -0
@@ -813,7 +813,7 @@ static void prepare_write_message(struct
ceph_connection *con)
m, con->out_seq, le16_to_cpu(m->hdr.type), le32_to_cpu(m->hdr.front_len), le32_to_cpu(m->hdr.middle_len), le32_to_cpu(m->hdr.data_len),
- m->nr_pages);
+ m->page_count);
BUG_ON(le32_to_cpu(m->hdr.front_len) != m->front.iov_len);
/* tag + hdr + front + middle */
@@ -1072,7 +1072,7 @@ static int write_partial_msg_pages(struct ceph_connection *con)
const size_t trail_off = data_len - trail_len;
dout("write_partial_msg_pages %p msg %p page %d/%d offset %d\n",
- con, msg, con->out_msg_pos.page, msg->nr_pages,
+ con, msg, con->out_msg_pos.page, msg->page_count,
con->out_msg_pos.page_pos);
/*
@@ -2715,9 +2715,10 @@ struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
m->middle = NULL;
/* data */
- m->nr_pages = 0;
+ m->page_count = 0;
m->page_alignment = 0;
m->pages = NULL;
+ m->pagelist_count = 0;
m->pagelist = NULL;
#ifdef CONFIG_BLOCK
m->bio = NULL;
@@ -2890,13 +2891,14 @@ void ceph_msg_last_put(struct kref *kref)
ceph_buffer_put(m->middle);
m->middle = NULL;
}
- m->nr_pages = 0;
+ m->page_count = 0;
m->pages = NULL;
if (m->pagelist) {
ceph_pagelist_release(m->pagelist); kfree(m->pagelist); m->pagelist = NULL;
+ m->pagelist_count = 0;
}
m->trail = NULL;
@@ -2910,8 +2912,8 @@ EXPORT_SYMBOL(ceph_msg_last_put);
void ceph_msg_dump(struct ceph_msg *msg)
{
- pr_debug("msg_dump %p (front_max %d nr_pages %d)\n", msg,
- msg->front_max, msg->nr_pages);
+ pr_debug("msg_dump %p (front_max %d page_count %d)\n", msg,
+ msg->front_max, msg->page_count);
print_hex_dump(KERN_DEBUG, "header: ",
DUMP_PREFIX_OFFSET, 16, 1, &msg->hdr, sizeof(msg->hdr), true);
net/ceph/osd_client.c +0 -0
@@ -1631,7 +1631,7 @@ int ceph_osdc_start_request(struct ceph_osd_client
*osdc,
int rc = 0;
req->r_request->pages = req->r_pages;
- req->r_request->nr_pages = req->r_num_pages;
+ req->r_request->page_count = req->r_num_pages;
#ifdef CONFIG_BLOCK
req->r_request->bio = req->r_bio;
#endif
@@ -1982,7 +1982,7 @@ static struct ceph_msg *get_reply(struct ceph_connection *con,
goto out;
}
m->pages = req->r_pages;
- m->nr_pages = req->r_num_pages;
+ m->page_count = req->r_num_pages;
m->page_alignment = req->r_page_alignment;
#ifdef CONFIG_BLOCK
m->bio = req->r_bio;
Signature · 1 line
1.7.9.5
PATCH 2/4 libceph: set page alignment in start_request()
In reply to Alex ElderThe page alignment field for a request is currently set in ceph_osdc_build_request(). It's not needed at that point nor do either of its callers need that value assigned at any point before they call ceph_osdc_start_request().
So move that assignment into ceph_osdc_start_request().
--- net/ceph/osd_client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
net/ceph/osd_client.c +0 -0
@@ -369,7 +369,6 @@ void ceph_osdc_build_request(struct ceph_osd_request
*req,
data_len += len;
}
req->r_request->hdr.data_len = cpu_to_le32(data_len);
- req->r_request->page_alignment = req->r_page_alignment;
BUG_ON(p > msg->front.iov_base + msg->front.iov_len); msg_size = p - msg->front.iov_base;
@@ -1632,6 +1631,7 @@ int ceph_osdc_start_request(struct ceph_osd_client *osdc,
req->r_request->pages = req->r_pages; req->r_request->page_count = req->r_num_pages;
+ req->r_request->page_alignment = req->r_page_alignment;
#ifdef CONFIG_BLOCK
req->r_request->bio = req->r_bio;
#endif
Signature · 1 line
1.7.9.5
PATCH 3/4 libceph: isolate message page field manipulation
In reply to Alex ElderDefine a function ceph_msg_data_set_pages(), which more clearly abstracts the assignment page-related fields for data in a ceph message structure. These fields should never be set more than once (add BUG_ON() calls to guarantee that). Use this new function in the osd client and mds client.
Rearrange the field order in a ceph_msg structure to group those that are used to define the possible data payloads.
--- fs/ceph/mds_client.c | 4 ++-- include/linux/ceph/messenger.h | 22 +++++++++++++--------- net/ceph/messenger.c | 12 ++++++++++++ net/ceph/osd_client.c | 11 +++++------ 4 files changed, 32 insertions(+), 17 deletions(-)
fs/ceph/mds_client.c +0 -0
@@ -1718,8 +1718,8 @@ static struct ceph_msg
*create_request_message(struct ceph_mds_client *mdsc,
msg->front.iov_len = p - msg->front.iov_base; msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
- msg->pages = req->r_pages;
- msg->page_count = req->r_num_pages;
+ ceph_msg_data_set_pages(msg, req->r_pages, req->r_num_pages, 0);
+
msg->hdr.data_len = cpu_to_le32(req->r_data_len); msg->hdr.data_off = cpu_to_le16(0);
include/linux/ceph/messenger.h +10 -9
@@ -74,22 +74,23 @@ struct ceph_msg {struct ceph_msg_footer footer;/* footer */ struct kvec front;/* unaligned blobs of message */ struct ceph_buffer *middle; -struct page **pages;/* data payload. NOT OWNER. */ -unsigned page_count;/* size of page array */ -unsigned page_alignment;/* io offset in first page */ + +struct page **pages;/* data payload. NOT OWNER. */ +unsigned int page_alignment;/* io offset in first page */ +unsigned int page_count;/* # pages in array or list */ struct ceph_pagelist *pagelist;/* instead of pages */ unsigned int pagelist_count;/* number of pages in pagelist */ - -struct ceph_connection *con; -struct list_head list_head; - -struct kref kref;#ifdef CONFIG_BLOCK +unsigned int bio_seg;/* current bio segment */ struct bio *bio;/* instead of pages/pagelist */ struct bio *bio_iter;/* bio iterator */ -unsigned int bio_seg;/* current bio segment */ #endif /* CONFIG_BLOCK */ struct ceph_pagelist *trail;/* the trailing part of the data */ + +struct ceph_connection *con; +struct list_head list_head; + +struct kref kref;bool front_is_vmalloc;bool more_to_follow;bool needs_out_seq;
@@ -219,6 +220,9 @@ extern void ceph_msg_revoke_incoming(struct ceph_msg
*msg);
extern void ceph_con_keepalive(struct ceph_connection *con);
+extern void ceph_msg_data_set_pages(struct ceph_msg *msg, struct page **pages,
+ unsigned int page_count, size_t alignment);
+
extern struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
bool can_fail);
extern void ceph_msg_kfree(struct ceph_msg *m);
net/ceph/messenger.c +12 -0
@@ -2674,6 +2674,18 @@ void ceph_con_keepalive(struct ceph_connection *con) }EXPORT_SYMBOL (ceph_con_keepalive); +void ceph_msg_data_set_pages (struct ceph_msg *msg,struct page **pages, +unsigned int page_count,size_t alignment) +{ +BUG_ON (msg->pages); +BUG_ON (msg->page_count); +BUG_ON (alignment >=PAGE_SIZE ); + + msg->pages = pages; + msg->page_count = page_count; + msg->page_alignment = alignment; +} +EXPORT_SYMBOL (ceph_msg_data_set_pages);/* * construct a new message with given type, size
net/ceph/osd_client.c +0 -0
@@ -1629,9 +1629,8 @@ int ceph_osdc_start_request(struct ceph_osd_client
*osdc,
{
int rc = 0;
- req->r_request->pages = req->r_pages;
- req->r_request->page_count = req->r_num_pages;
- req->r_request->page_alignment = req->r_page_alignment;
+ ceph_msg_data_set_pages(req->r_request, req->r_pages, req->r_num_pages,
+ req->r_page_alignment);
#ifdef CONFIG_BLOCK
req->r_request->bio = req->r_bio;
#endif
@@ -1981,9 +1980,9 @@ static struct ceph_msg *get_reply(struct ceph_connection *con,
m = NULL;
goto out;
}
- m->pages = req->r_pages;
- m->page_count = req->r_num_pages;
- m->page_alignment = req->r_page_alignment;
+
+ ceph_msg_data_set_pages(m, req->r_pages, req->r_num_pages,
+ req->r_page_alignment);
#ifdef CONFIG_BLOCK
m->bio = req->r_bio;
#endif
Signature · 1 line
1.7.9.5
PATCH 4/4 libceph: isolate other message data fields
In reply to Alex ElderDefine ceph_msg_data_set_pagelist(), ceph_msg_data_set_bio(), and ceph_msg_data_set_trail() to clearly abstract the assignment the remaining data-related fields in a ceph message structure. These fields should never be used more than once; add BUG_ON() calls to guarantee this. Use the new functions in the osd client and mds client.
--- fs/ceph/mds_client.c | 4 ++-- include/linux/ceph/messenger.h | 6 ++++++ net/ceph/messenger.c | 28 ++++++++++++++++++++++++++++ net/ceph/osd_client.c | 6 +++--- 4 files changed, 39 insertions(+), 5 deletions(-)
fs/ceph/mds_client.c +0 -0
@@ -2599,8 +2599,8 @@ static void send_mds_reconnect(struct
ceph_mds_client *mdsc,
goto fail;
}
- reply->pagelist = pagelist;
- reply->pagelist_count = calc_pages_for(0, pagelist->length);
+ ceph_msg_data_set_pagelist(reply, pagelist,
+ calc_pages_for(0, pagelist->length));
if (recon_state.flock)
reply->hdr.version = cpu_to_le16(2);
reply->hdr.data_len = cpu_to_le32(pagelist->length);
include/linux/ceph/messenger.h +0 -0
@@ -222,6 +222,12 @@ extern void ceph_con_keepalive(struct
ceph_connection *con);
extern void ceph_msg_data_set_pages(struct ceph_msg *msg, struct page
**pages,
unsigned int page_count, size_t alignment);
+extern void ceph_msg_data_set_pagelist(struct ceph_msg *msg,
+ struct ceph_pagelist *pagelist,
+ unsigned int page_count);
+extern void ceph_msg_data_set_bio(struct ceph_msg *msg, struct bio *bio); +extern void ceph_msg_data_set_trail(struct ceph_msg *msg,
+ struct ceph_pagelist *trail);
extern struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags,
bool can_fail);
net/ceph/messenger.c +0 -0
@@ -2687,6 +2687,34 @@ void ceph_msg_data_set_pages(struct ceph_msg
*msg, struct page **pages,
}
EXPORT_SYMBOL(ceph_msg_data_set_pages);
+void ceph_msg_data_set_pagelist(struct ceph_msg *msg,
+ struct ceph_pagelist *pagelist,
+ unsigned int pagelist_count)
+{
+ BUG_ON(msg->pagelist);
+ BUG_ON(msg->pagelist_count);
+
+ msg->pagelist = pagelist;
+ msg->pagelist_count = pagelist_count;
+}
+EXPORT_SYMBOL(ceph_msg_data_set_pagelist);
+
+void ceph_msg_data_set_bio(struct ceph_msg *msg, struct bio *bio) +{
+ BUG_ON(msg->bio);
+
+ msg->bio = bio;
+}
+EXPORT_SYMBOL(ceph_msg_data_set_bio);
+
+void ceph_msg_data_set_trail(struct ceph_msg *msg, struct ceph_pagelist *trail)
+{
+ BUG_ON(msg->trail);
+
+ msg->trail = trail;
+}
+EXPORT_SYMBOL(ceph_msg_data_set_trail);
+
/*
* construct a new message with given type, size
* the new msg has a ref count of 1.
net/ceph/osd_client.c +0 -0
@@ -1632,9 +1632,9 @@ int ceph_osdc_start_request(struct ceph_osd_client
*osdc,
ceph_msg_data_set_pages(req->r_request, req->r_pages, req->r_num_pages,
req->r_page_alignment);
#ifdef CONFIG_BLOCK
- req->r_request->bio = req->r_bio;
+ ceph_msg_data_set_bio(req->r_request, req->r_bio);
#endif
- req->r_request->trail = &req->r_trail;
+ ceph_msg_data_set_trail(req->r_request, &req->r_trail);
register_request(osdc, req);
@@ -1984,7 +1984,7 @@ static struct ceph_msg *get_reply(struct ceph_connection *con,
ceph_msg_data_set_pages(m, req->r_pages, req->r_num_pages,
req->r_page_alignment);
#ifdef CONFIG_BLOCK
- m->bio = req->r_bio;
+ ceph_msg_data_set_bio(m, req->r_bio);
#endif
}
*skip = 0;
Signature · 1 line
1.7.9.5
Quoted 7 lines · Alex Elder
This series makes the fields related to the data portion of a ceph message not get manipulated by code outside the ceph messenger. It implements some interface functions that can be used to assign data-related fields. Doing this will allow the way message data is managed to be changed independent of the users of the messenger module.
Quoted 6 lines
-Alex
[PATCH 1/4] libceph: distinguish page array and pagelist count [PATCH 2/4] libceph: set page alignment in
start_request()[PATCH 3/4] libceph: isolate message page field manipulation [PATCH 4/4] libceph: isolate other message data fields
These all look good.