ceph-rbd: ceph RADOS block device
Subsystems (2)
v3 cover ceph-rbd: ceph RADOS block device
The following is our latest version of the rados block device. Any comments, questions, suggestions are more than welcome. We would like to have that ready for inclusion for the next window and it'd be nice having it reviewed by someone more familiar with the block/storage layer. Note that the actual block device implementation that we'd happy to get review for is located in the last patch of the series. All the preceding patches just build up the groundwork, and implement the infrastructure for it.
RBD allows striping of data across rados, the ceph distributed block layer, and is binary compatible with Christian Brunner's kvm-rbd implementation. Similarly to the Sheepdog project, it stripes the block device across 4MB (or other configurable size) objects stored by the distributed object store, and enjoys all the rados features (high availability, scalability, snapshots, etc.)
A use case for this device would be to have some kind of a local file system created on it, and use it with conjuction of kvm to do migration and other related stuff. Another option would be to use it as data devices for other distributed file systems (e.g., ocfs2, gfs2).
The device driver is is based on osdblk. Currently, it resides under the fs/ceph tree and does not exist as a separate module of its own, such that the ceph.ko module contains both the ceph file system and the rbd block device. Another option would be to have it as a separate module that resides under drivers/block, however, it would still depend on the ceph.ko module.
This version of the patch series includes many bug fixes. The major change is that it moves the snapshot administration functionality (e.g., snapshot creation, listing) to the osd side, and takes advantage of the osd extensible functionality. Also, snapshots can now be rolled back to a specific version.
Yehuda
---
Yehuda Sadeh (10):
ceph-rbd: lookup pool in osdmap by name
ceph-rbd: refactor osdc requests creation functions
ceph-rbd: messenger and osdc changes for rbd
ceph-rbd: enable creation of clients that don't need mds
ceph-rbd: refactor mount related functions, add helpers
ceph-rbd: generalize mon requests, add pool op support
ceph-rbd: some super.c changes for rbd
ceph-rbd: osdc support for osd call and rollback operations
ceph-rbd: sync common header and source files with server version
ceph-rbd: rados block device
fs/ceph/Kconfig | 10 + fs/ceph/Makefile | 2 +- fs/ceph/README | 1 + fs/ceph/ceph_fs.c | 50 +- fs/ceph/ceph_fs.h | 8 +- fs/ceph/ceph_strings.c | 1 + fs/ceph/debugfs.c | 11 +- fs/ceph/decode.h | 5 + fs/ceph/file.c | 46 ++ fs/ceph/messenger.c | 215 +++++- fs/ceph/messenger.h | 4 + fs/ceph/mon_client.c | 173 ++++- fs/ceph/mon_client.h | 5 + fs/ceph/osd_client.c | 353 ++++++++-- fs/ceph/osd_client.h | 66 ++ fs/ceph/osdmap.c | 13 + fs/ceph/osdmap.h | 2 + fs/ceph/pagelist.c | 2 +- fs/ceph/pagelist.h | 2 +- fs/ceph/rados.h | 8 + fs/ceph/rbd.c | 1835 ++++++++++++++++++++++++++++++++++++++++++++++++ fs/ceph/rbd.h | 8 + fs/ceph/rbd_types.h | 71 ++ fs/ceph/super.c | 194 ++++-- fs/ceph/super.h | 36 +- 25 files changed, 2918 insertions(+), 203 deletions(-) create mode 100644 fs/ceph/rbd.c create mode 100644 fs/ceph/rbd.h create mode 100644 fs/ceph/rbd_types.h
PATCH v3 1/10 ceph-rbd: lookup pool in osdmap by name
--- fs/ceph/osdmap.c | 13 +++++++++++++ fs/ceph/osdmap.h | 2 ++ 2 files changed, 15 insertions(+), 0 deletions(-)
fs/ceph/osdmap.c +13 -0
@@ -417,6 +417,19 @@ static struct ceph_pg_pool_info *__lookup_pg_pool(struct rb_root *root, int id)return NULL ; } +int ceph_pg_poolid_by_name (struct ceph_osdmap *map,const char *name) +{ +struct rb_node *rbp; + +for (rbp =rb_first (&map->pg_pools); rbp; rbp =rb_next (rbp)) { +struct ceph_pg_pool_info *pi = + rb_entry(rbp, struct ceph_pg_pool_info, node); +if (pi->name &&strcmp (pi->name, name) ==0 ) +return pi->id; + } +return -ENOENT ; +} +static void __remove_pg_pool (struct rb_root *root,struct ceph_pg_pool_info *pi) {rb_erase (&pi->node, root);
fs/ceph/osdmap.h +2 -0
@@ -125,4 +125,6 @@ extern int ceph_calc_pg_acting(struct ceph_osdmap *osdmap, struct ceph_pg pgid,extern int ceph_calc_pg_primary (struct ceph_osdmap *osdmap,struct ceph_pg pgid); +extern int ceph_pg_poolid_by_name (struct ceph_osdmap *map,const char *name); +#endif
Signature · 1 line
1.5.6.5
PATCH v3 2/10 ceph-rbd: refactor osdc requests creation functions
In reply to Yehuda SadehThe osd requests creation are being decoupled from the vino parameter, allowing clients using the osd to use other arbitrary object names that are not necessarily vino based. Also, calc_raw_layout now takes a snap id.
--- fs/ceph/osd_client.c | 187 ++++++++++++++++++++++++++++++++++--------------- fs/ceph/osd_client.h | 25 +++++++ 2 files changed, 155 insertions(+), 57 deletions(-)
fs/ceph/osd_client.c +130 -57
@@ -22,6 +22,35 @@ static int __kick_requests(struct ceph_osd_client *osdc,static void kick_requests (struct ceph_osd_client *osdc,struct ceph_osd *osd); +void ceph_calc_raw_layout (struct ceph_osd_client *osdc, +struct ceph_file_layout *layout, +u64 snapid, +u64 off,u64 len,u64 *bno, +struct ceph_osd_request *req) +{ +struct ceph_osd_request_head *reqhead = req->r_request->front.iov_base; +struct ceph_osd_op *op = (void *)(reqhead +1 ); +u64 orig_len = len; +u64 objoff, objlen;/* extent in object */ + + reqhead->snapid =cpu_to_le64 (snapid); + +/* object extent? */ +ceph_calc_file_object_mapping (layout, off, &len, bno, + &objoff, &objlen); +if (len < orig_len) +dout (" skipping last %llu, final file extent %llu~%llu\n" , + orig_len - len, off, len); + + op->extent.offset =cpu_to_le64 (objoff); + op->extent.length =cpu_to_le64 (objlen); + req->r_num_pages =calc_pages_for (off, len); + +dout ("calc_layout bno=%llx %llu~%llu (%d pages)\n" , + *bno, objoff, objlen, req->r_num_pages); + +} +/* * Implement client access to distributed object storage cluster. *
@@ -48,34 +77,17 @@ static void kick_requests(struct ceph_osd_client *osdc, struct ceph_osd *osd);* fill osd op in request message. */ static void calc_layout (struct ceph_osd_client *osdc, -struct ceph_vino vino,struct ceph_file_layout *layout, +struct ceph_vino vino, +struct ceph_file_layout *layout,u64 off,u64 *plen,struct ceph_osd_request *req) { -struct ceph_osd_request_head *reqhead = req->r_request->front.iov_base; -struct ceph_osd_op *op = (void *)(reqhead +1 ); -u64 orig_len = *plen; -u64 objoff, objlen;/* extent in object */ u64 bno; - reqhead->snapid =cpu_to_le64 (vino.snap); - -/* object extent? */ -ceph_calc_file_object_mapping (layout, off, plen, &bno, - &objoff, &objlen); -if (*plen < orig_len) -dout (" skipping last %llu, final file extent %llu~%llu\n" , - orig_len - *plen, off, *plen); +ceph_calc_raw_layout (osdc, layout, vino.snap, off, *plen, &bno, req);sprintf (req->r_oid,"%llx.%08llx" , vino.ino, bno); req->r_oid_len =strlen (req->r_oid); - - op->extent.offset =cpu_to_le64 (objoff); - op->extent.length =cpu_to_le64 (objlen); - req->r_num_pages =calc_pages_for (off, *plen); - -dout ("calc_layout %s (%d) %llu~%llu (%d pages)\n" , - req->r_oid, req->r_oid_len, objoff, objlen, req->r_num_pages); }/*
@@ -108,43 +120,34 @@ void ceph_osdc_release_request(struct kref *kref)kfree (req); } -/* -* build new request AND message, calculate layout, and adjust file -* extent as needed. -* -* if the file was recently truncated, we include information about its -* old and new size so that the object can be updated appropriately. (we -* avoid synchronously deleting truncated objects because it's slow.) -* -* if @do_sync, include a 'startsync' command so that the osd will flush -* data quickly. -*/ -struct ceph_osd_request *ceph_osdc_new_request (struct ceph_osd_client *osdc, -struct ceph_file_layout *layout, -struct ceph_vino vino, -u64 off,u64 *plen, -int opcode,int flags, +struct ceph_osd_request *ceph_osdc_alloc_request (struct ceph_osd_client *osdc, +int flags,struct ceph_snap_context *snapc,int do_sync, -u32 truncate_seq, -u64 truncate_size, -struct timespec *mtime, -bool use_mempool,int num_reply) +bool use_mempool, +gfp_t gfp_flags, +struct page **pages) {struct ceph_osd_request *req;struct ceph_msg *msg; -struct ceph_osd_request_head *head; -struct ceph_osd_op *op; -void *p;int num_op =1 + do_sync; -size_t msg_size =sizeof (*head) + num_op*sizeof (*op); -int i; +size_t msg_size =sizeof (struct ceph_osd_request_head ) + + num_op*sizeof (struct ceph_osd_op );if (use_mempool) { - req =mempool_alloc (osdc->req_mempool,GFP_NOFS ); + req =mempool_alloc (osdc->req_mempool, gfp_flags);memset (req,0 ,sizeof (*req)); }else { - req =kzalloc (sizeof (*req),GFP_NOFS ); + req =kzalloc (sizeof (*req), gfp_flags); + } +if (!req) +return NULL ; + +if (use_mempool) { + req =mempool_alloc (osdc->req_mempool, gfp_flags); +memset (req,0 ,sizeof (*req)); + }else { + req =kzalloc (sizeof (*req), gfp_flags); }if (req ==NULL )return NULL ;
@@ -164,7 +167,7 @@ struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *osdc, msg =ceph_msgpool_get (&osdc->msgpool_op_reply,0 ); else msg =ceph_msg_new (CEPH_MSG_OSD_OPREPLY , -OSD_OPREPLY_FRONT_LEN ,GFP_NOFS ); +OSD_OPREPLY_FRONT_LEN , gfp_flags);if (!msg) {ceph_osdc_put_request (req);return NULL ;
@@ -178,18 +181,48 @@ struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *osdc,if (use_mempool) msg =ceph_msgpool_get (&osdc->msgpool_op,0 );else - msg =ceph_msg_new (CEPH_MSG_OSD_OP , msg_size,GFP_NOFS ); + msg =ceph_msg_new (CEPH_MSG_OSD_OP , msg_size, gfp_flags);if (!msg) {ceph_osdc_put_request (req);return NULL ; } msg->hdr.type =cpu_to_le16 (CEPH_MSG_OSD_OP );memset (msg->front.iov_base,0 , msg->front.iov_len); + + req->r_request = msg; + req->r_pages = pages; + +return req; +} + +/* +* build new request AND message +* +*/ +void ceph_osdc_build_request (struct ceph_osd_request *req, +u64 off,u64 *plen, +int opcode, +struct ceph_snap_context *snapc, +int do_sync, +u32 truncate_seq, +u64 truncate_size, +struct timespec *mtime, +const char *oid, +int oid_len) +{ +struct ceph_msg *msg = req->r_request; +struct ceph_osd_request_head *head; +struct ceph_osd_op *op; +void *p; +int num_op =1 + do_sync; +size_t msg_size =sizeof (*head) + num_op*sizeof (*op); +int i; +int flags = req->r_flags; + head = msg->front.iov_base; op = (void *)(head +1 ); p = (void *)(op + num_op); - req->r_request = msg; req->r_snapc =ceph_get_snap_context (snapc); head->client_inc =cpu_to_le32 (1 );/* always, for now. */
@@ -199,10 +232,6 @@ struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *osdc, head->num_ops =cpu_to_le16 (num_op); op->op =cpu_to_le16 (opcode); -/* calculate max write size */ -calc_layout (osdc, vino, layout, off, plen, req); - req->r_file_layout = *layout;/* keep a copy */ -if (flags &CEPH_OSD_FLAG_WRITE ) { req->r_request->hdr.data_off =cpu_to_le16 (off); req->r_request->hdr.data_len =cpu_to_le32 (*plen);
@@ -212,9 +241,9 @@ struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *osdc, op->extent.truncate_seq =cpu_to_le32 (truncate_seq);/* fill in oid */ - head->object_len =cpu_to_le32 (req->r_oid_len); -memcpy (p, req->r_oid, req->r_oid_len); - p += req->r_oid_len; + head->object_len =cpu_to_le32 (oid_len); +memcpy (p, oid, oid_len); + p += oid_len;if (do_sync) { op++;
@@ -233,6 +262,50 @@ struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *osdc, msg_size = p - msg->front.iov_base; msg->front.iov_len = msg_size; msg->hdr.front_len =cpu_to_le32 (msg_size); +return ; +} + +/* +* build new request AND message, calculate layout, and adjust file +* extent as needed. +* +* if the file was recently truncated, we include information about its +* old and new size so that the object can be updated appropriately. (we +* avoid synchronously deleting truncated objects because it's slow.) +* +* if @do_sync, include a 'startsync' command so that the osd will flush +* data quickly. +*/ +struct ceph_osd_request *ceph_osdc_new_request (struct ceph_osd_client *osdc, +struct ceph_file_layout *layout, +struct ceph_vino vino, +u64 off,u64 *plen, +int opcode,int flags, +struct ceph_snap_context *snapc, +int do_sync, +u32 truncate_seq, +u64 truncate_size, +struct timespec *mtime, +bool use_mempool,int num_reply) +{ +struct ceph_osd_request *req = +ceph_osdc_alloc_request (osdc, flags, + snapc, do_sync, + use_mempool, +GFP_NOFS ,NULL ); +if (IS_ERR (req)) +return req; + +/* calculate max write size */ +calc_layout (osdc, vino, layout, off, plen, req); + req->r_file_layout = *layout;/* keep a copy */ + +ceph_osdc_build_request (req, off, plen, opcode, + snapc, do_sync, + truncate_seq, truncate_size, + mtime, + req->r_oid, req->r_oid_len); +return req; }
fs/ceph/osd_client.h +25 -0
@@ -119,6 +119,31 @@ extern void ceph_osdc_handle_reply(struct ceph_osd_client *osdc,extern void ceph_osdc_handle_map (struct ceph_osd_client *osdc,struct ceph_msg *msg); +extern void ceph_calc_raw_layout (struct ceph_osd_client *osdc, +struct ceph_file_layout *layout, +u64 snapid, +u64 off,u64 len,u64 *bno, +struct ceph_osd_request *req); + +extern struct ceph_osd_request *ceph_osdc_alloc_request (struct ceph_osd_client *osdc, +int flags, +struct ceph_snap_context *snapc, +int do_sync, +bool use_mempool, +gfp_t gfp_flags, +struct page **pages); + +extern void ceph_osdc_build_request (struct ceph_osd_request *req, +u64 off,u64 *plen, +int opcode, +struct ceph_snap_context *snapc, +int do_sync, +u32 truncate_seq, +u64 truncate_size, +struct timespec *mtime, +const char *oid, +int oid_len); + extern struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *, struct ceph_file_layout *layout, struct ceph_vino vino,
Signature · 1 line
1.5.6.5
PATCH v3 3/10 ceph-rbd: messenger and osdc changes for rbd
In reply to Yehuda SadehThe messenger can send/receive data in bio. This was added so that we wouldn't need to copy the data from the rados block device.
We can now have trailing variable sized data for osd ops. Also osd ops encoding is more modular.
--- fs/ceph/messenger.c | 215 ++++++++++++++++++++++++++++++++++++++++++-------- fs/ceph/messenger.h | 4 + fs/ceph/osd_client.c | 198 ++++++++++++++++++++++++++++++++++++--------- fs/ceph/osd_client.h | 58 +++++++++++--- fs/ceph/pagelist.c | 2 +- fs/ceph/pagelist.h | 2 +- 6 files changed, 393 insertions(+), 86 deletions(-)
fs/ceph/messenger.c +183 -32
@@ -9,6 +9,8 @@#include <linux/slab.h> #include <linux/socket.h> #include <linux/string.h> +#include <linux/bio.h> +#include <linux/blkdev.h> #include <net/tcp.h> #include "super.h"
@@ -539,8 +541,11 @@ static void prepare_write_message(struct ceph_connection *con)if (le32_to_cpu (m->hdr.data_len) >0 ) {/* initialize page iterator */ con->out_msg_pos.page =0 ; - con->out_msg_pos.page_pos = -le16_to_cpu (m->hdr.data_off) & ~PAGE_MASK ; +if (m->pages) + con->out_msg_pos.page_pos = +le16_to_cpu (m->hdr.data_off) & ~PAGE_MASK ; +else + con->out_msg_pos.page_pos =0 ; con->out_msg_pos.data_pos =0 ; con->out_msg_pos.did_page_crc =0 ; con->out_more =1 ;/* data + footer will follow */
@@ -722,6 +727,31 @@ out:return ret;/* done! */ } +#ifdef CONFIG_BLOCK +static void init_bio_iter (struct bio *bio,struct bio **iter,int *seg) +{ +if (!bio) { + *iter =NULL ; + *seg =0 ; +return ; + } + *iter = bio; + *seg = bio->bi_idx; +} + +static void iter_bio_next (struct bio **bio_iter,int *seg) +{ +if (*bio_iter ==NULL ) +return ; + +BUG_ON (*seg >= (*bio_iter)->bi_vcnt); + + (*seg)++; +if (*seg == (*bio_iter)->bi_vcnt) +init_bio_iter ((*bio_iter)->bi_next, bio_iter, seg); +} +#endif +/* * Write as much message data payload as we can. If we finish, queue * up the footer.
@@ -736,21 +766,45 @@ static int write_partial_msg_pages(struct ceph_connection *con)size_t len;int crc = con->msgr->nocrc;int ret; +int total_max_write; +int in_trail =0 ; +size_t trail_len = (msg->trail ? msg->trail->length :0 );dout ("write_partial_msg_pages %p msg %p page %d/%d offset %d\n" , con, con->out_msg, con->out_msg_pos.page, con->out_msg->nr_pages, con->out_msg_pos.page_pos); -while (con->out_msg_pos.page < con->out_msg->nr_pages) { +#ifdef CONFIG_BLOCK +if (msg->bio && !msg->bio_iter) +init_bio_iter (msg->bio, &msg->bio_iter, &msg->bio_seg); +#endif + +while (data_len > con->out_msg_pos.data_pos) {struct page *page =NULL ;void *kaddr =NULL ; +int max_write =PAGE_SIZE ; +int page_shift =0 ; + + total_max_write = data_len - trail_len - con->out_msg_pos.data_pos;/* * if we are calculating the data crc (the default), we need * to map the page. if our pages[] has been revoked, use the * zero page. */ -if (msg->pages) { + +/* have we reached the trail part of the data? */ +if (con->out_msg_pos.data_pos >= data_len - trail_len) { + in_trail =1 ; + + total_max_write = data_len - con->out_msg_pos.data_pos; + + page =list_first_entry (&msg->trail->head, + struct page, lru); +if (crc) + kaddr =kmap (page); + max_write =PAGE_SIZE ; + }else if (msg->pages) { page = msg->pages[con->out_msg_pos.page];if (crc) kaddr =kmap (page);
@@ -759,13 +813,25 @@ static int write_partial_msg_pages(struct ceph_connection *con) struct page, lru);if (crc) kaddr =kmap (page); +#ifdef CONFIG_BLOCK + }else if (msg->bio) { +struct bio_vec *bv; + + bv =bio_iovec_idx (msg->bio_iter, msg->bio_seg); + page = bv->bv_page; + page_shift = bv->bv_offset; +if (crc) + kaddr =kmap (page) + page_shift; + max_write = bv->bv_len; +#endif }else { page = con->msgr->zero_page;if (crc) kaddr =page_address (con->msgr->zero_page); } - len =min ((int )(PAGE_SIZE - con->out_msg_pos.page_pos), - (int )(data_len - con->out_msg_pos.data_pos)); + len =min_t (int, max_write - con->out_msg_pos.page_pos, + total_max_write); +if (crc && !con->out_msg_pos.did_page_crc) {void *base = kaddr + con->out_msg_pos.page_pos;u32 tmpcrc =le32_to_cpu (con->out_msg->footer.data_crc);
@@ -775,13 +841,14 @@ static int write_partial_msg_pages(struct ceph_connection *con)cpu_to_le32 (crc32c (tmpcrc, base, len)); con->out_msg_pos.did_page_crc =1 ; } - ret =kernel_sendpage (con->sock, page, - con->out_msg_pos.page_pos, len, + con->out_msg_pos.page_pos + page_shift, + len,MSG_DONTWAIT |MSG_NOSIGNAL |MSG_MORE ); -if (crc && (msg->pages || msg->pagelist)) +if (crc && + (msg->pages || msg->pagelist || msg->bio || in_trail) )kunmap (page);if (ret <=0 )
@@ -793,9 +860,16 @@ static int write_partial_msg_pages(struct ceph_connection *con) con->out_msg_pos.page_pos =0 ; con->out_msg_pos.page++; con->out_msg_pos.did_page_crc =0 ; -if (msg->pagelist) +if (in_trail) +list_move_tail (&page->lru, + &msg->trail->head); +else if (msg->pagelist)list_move_tail (&page->lru, &msg->pagelist->head); +#ifdef CONFIG_BLOCK + else if (msg->bio) + iter_bio_next(&msg->bio_iter, &msg->bio_seg); +#endif } }
@@ -1302,8 +1376,7 @@ static int read_partial_message_section(struct ceph_connection *con, struct kvec *section, unsigned int sec_len, u32 *crc) { - int left; -int ret; + int ret, left;BUG_ON (!section);
@@ -1326,13 +1399,83 @@ static int read_partial_message_section(struct ceph_connection *con,static struct ceph_msg *ceph_alloc_msg (struct ceph_connection *con,struct ceph_msg_header *hdr,int *skip); + + +static int read_partial_message_pages (struct ceph_connection *con, +struct page **pages, +unsigned data_len,int datacrc) +{ +void *p; +int ret; +int left; + + left =min ((int )(data_len - con->in_msg_pos.data_pos), + (int )(PAGE_SIZE - con->in_msg_pos.page_pos)); +/* (page) data */ +BUG_ON (pages ==NULL ); + p =kmap (pages[con->in_msg_pos.page]); + ret =ceph_tcp_recvmsg (con->sock, p + con->in_msg_pos.page_pos, + left); +if (ret >0 && datacrc) + con->in_data_crc = +crc32c (con->in_data_crc, + p + con->in_msg_pos.page_pos, ret); +kunmap (pages[con->in_msg_pos.page]); +if (ret <=0 ) +return ret; + con->in_msg_pos.data_pos += ret; + con->in_msg_pos.page_pos += ret; +if (con->in_msg_pos.page_pos ==PAGE_SIZE ) { + con->in_msg_pos.page_pos =0 ; + con->in_msg_pos.page++; + } + +return ret; +} + +#ifdef CONFIG_BLOCK +static int read_partial_message_bio (struct ceph_connection *con, +struct bio **bio_iter,int *bio_seg, +unsigned data_len,int datacrc) +{ +struct bio_vec *bv =bio_iovec_idx (*bio_iter, *bio_seg); +void *p; +int ret, left; + +if (IS_ERR (bv)) +return PTR_ERR (bv); + + left =min ((int )(data_len - con->in_msg_pos.data_pos), + (int )(bv->bv_len - con->in_msg_pos.page_pos)); + + p =kmap (bv->bv_page) + bv->bv_offset; + + ret =ceph_tcp_recvmsg (con->sock, p + con->in_msg_pos.page_pos, + left); +if (ret >0 && datacrc) + con->in_data_crc = +crc32c (con->in_data_crc, + p + con->in_msg_pos.page_pos, ret); +kunmap (bv->bv_page); +if (ret <=0 ) +return ret; + con->in_msg_pos.data_pos += ret; + con->in_msg_pos.page_pos += ret; +if (con->in_msg_pos.page_pos == bv->bv_len) { + con->in_msg_pos.page_pos =0 ; +iter_bio_next (bio_iter, bio_seg); + } + +return ret; +} +#endif +/* * read (part of) a message. */ static int read_partial_message (struct ceph_connection *con) {struct ceph_msg *m = con->in_msg; -void *p;int ret;int to, left;unsigned front_len, middle_len, data_len, data_off;
@@ -1417,7 +1560,10 @@ static int read_partial_message(struct ceph_connection *con) m->middle->vec.iov_len =0 ; con->in_msg_pos.page =0 ; - con->in_msg_pos.page_pos = data_off & ~PAGE_MASK ; +if (m->pages) + con->in_msg_pos.page_pos = data_off & ~PAGE_MASK ; +else + con->in_msg_pos.page_pos =0 ; con->in_msg_pos.data_pos =0 ; }
@@ -1434,27 +1580,29 @@ static int read_partial_message(struct ceph_connection *con)if (ret <=0 )return ret; } +#ifdef CONFIG_BLOCK +if (m->bio && !m->bio_iter) +init_bio_iter (m->bio, &m->bio_iter, &m->bio_seg); +#endif /* (page) data */ while (con->in_msg_pos.data_pos < data_len) { - left =min ((int )(data_len - con->in_msg_pos.data_pos), - (int )(PAGE_SIZE - con->in_msg_pos.page_pos)); -BUG_ON (m->pages ==NULL ); - p =kmap (m->pages[con->in_msg_pos.page]); - ret =ceph_tcp_recvmsg (con->sock, p + con->in_msg_pos.page_pos, - left); -if (ret >0 && datacrc) - con->in_data_crc = -crc32c (con->in_data_crc, - p + con->in_msg_pos.page_pos, ret); -kunmap (m->pages[con->in_msg_pos.page]); -if (ret <=0 ) -return ret; - con->in_msg_pos.data_pos += ret; - con->in_msg_pos.page_pos += ret; -if (con->in_msg_pos.page_pos ==PAGE_SIZE ) { - con->in_msg_pos.page_pos =0 ; - con->in_msg_pos.page++; +if (m->pages) { + ret =read_partial_message_pages (con, m->pages, + data_len, datacrc); +if (ret <=0 ) +return ret; +#ifdef CONFIG_BLOCK + }else if (m->bio) { + + ret =read_partial_message_bio (con, + &m->bio_iter, &m->bio_seg, + data_len, datacrc); +if (ret <=0 ) +return ret; +#endif + }else { +BUG_ON (1 ); } }
@@ -2130,6 +2278,9 @@ struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags) m->nr_pages =0 ; m->pages =NULL ; m->pagelist =NULL ; + m->bio =NULL ; + m->bio_iter =NULL ; + m->bio_seg =0 ;dout ("ceph_msg_new %p front %d\n" , m, front_len);return m;
fs/ceph/messenger.h +4 -0
@@ -82,6 +82,10 @@ struct ceph_msg {struct ceph_pagelist *pagelist;/* instead of pages */ struct list_head list_head;struct kref kref; +struct bio *bio;/* instead of pages/pagelist */ +struct bio *bio_iter;/* bio iterator */ +int bio_seg;/* current bio segment */ +struct ceph_pagelist *trail;/* the trailing part of the data */ bool front_is_vmalloc;bool more_to_follow;bool needs_out_seq;
fs/ceph/osd_client.c +158 -40
337 lines not on this page · raw message
fs/ceph/osd_client.h +46 -12
91 lines not on this page · raw message
fs/ceph/pagelist.c +1 -1
@@ -31,7 +31,7 @@ static int ceph_pagelist_addpage(struct ceph_pagelist *pl)return 0 ; } -int ceph_pagelist_append (struct ceph_pagelist *pl,void *buf,size_t len) +int ceph_pagelist_append (struct ceph_pagelist *pl,const void *buf,size_t len) {while (pl->room < len) {size_t bit = pl->room;
fs/ceph/pagelist.h +1 -1
@@ -19,7 +19,7 @@ static inline void ceph_pagelist_init(struct ceph_pagelist *pl) }extern int ceph_pagelist_release (struct ceph_pagelist *pl); -extern int ceph_pagelist_append (struct ceph_pagelist *pl,void *d,size_t l); +extern int ceph_pagelist_append (struct ceph_pagelist *pl,const void *d,size_t l);static inline int ceph_pagelist_encode_64 (struct ceph_pagelist *pl,u64 v) {
Signature · 1 line
1.5.6.5
PATCH v3 4/10 ceph-rbd: enable creation of clients that don't need mds
In reply to Yehuda SadehPreparing grounds for rbd that doesn't need mds client.
--- fs/ceph/debugfs.c | 11 ++++++++--- fs/ceph/mon_client.c | 3 ++- fs/ceph/super.c | 18 ++++++++++++------ fs/ceph/super.h | 2 ++ 4 files changed, 24 insertions(+), 10 deletions(-)
fs/ceph/debugfs.c +8 -3
@@ -440,9 +440,14 @@ int ceph_debugfs_client_init(struct ceph_client *client)if (!client->debugfs_congestion_kb) goto out; -sprintf (name,"../../bdi/%s" ,dev_name (client->sb->s_bdi->dev)); - client->debugfs_bdi =debugfs_create_symlink ("bdi" , client->debugfs_dir, - name); +if (client->backing_dev_info.dev) { +sprintf (name,"../../bdi/%s" , +dev_name (client->backing_dev_info.dev)); + client->debugfs_bdi = +debugfs_create_symlink ("bdi" , + client->debugfs_dir, + name); + }return 0 ;
fs/ceph/mon_client.c +2 -1
@@ -787,7 +787,8 @@ static void dispatch(struct ceph_connection *con, struct ceph_msg *msg)break ;case CEPH_MSG_MDS_MAP : -ceph_mdsc_handle_map (&monc->client->mdsc, msg); +if (monc->client->have_mdsc) +ceph_mdsc_handle_map (&monc->client->mdsc, msg);break ;case CEPH_MSG_OSD_MAP :
fs/ceph/super.c +12 -6
@@ -584,7 +584,8 @@ static void destroy_mount_args(struct ceph_mount_args *args)/* * create a fresh client instance */ -static struct ceph_client *ceph_create_client (struct ceph_mount_args *args) +struct ceph_client *ceph_create_client (struct ceph_mount_args *args, +int need_mdsc) {struct ceph_client *client;int err = -ENOMEM ;
@@ -639,9 +640,13 @@ static struct ceph_client *ceph_create_client(struct ceph_mount_args *args) err =ceph_osdc_init (&client->osdc, client);if (err <0 ) goto fail_monc; - err =ceph_mdsc_init (&client->mdsc, client); -if (err <0 ) - goto fail_osdc; +if (need_mdsc) { + err =ceph_mdsc_init (&client->mdsc, client); +if (err <0 ) + goto fail_osdc; + client->have_mdsc =1 ; + } +return client; fail_osdc:
@@ -668,7 +673,8 @@ static void ceph_destroy_client(struct ceph_client *client)dout ("destroy_client %p\n" , client);/* unmount */ -ceph_mdsc_stop (&client->mdsc); +if (client->have_mdsc) +ceph_mdsc_stop (&client->mdsc);ceph_osdc_stop (&client->osdc);/*
@@ -963,7 +969,7 @@ static int ceph_get_sb(struct file_system_type *fs_type, }/* create client (which we may/may not use) */ - client =ceph_create_client (args); + client =ceph_create_client (args,1 );if (IS_ERR (client)) { err =PTR_ERR (client); goto out_final;
fs/ceph/super.h +2 -0
@@ -139,6 +139,8 @@ struct ceph_client {int min_caps;/* min caps i added */ +int have_mdsc; +struct ceph_messenger *msgr;/* messenger instance */ struct ceph_mon_client monc;struct ceph_mds_client mdsc;
Signature · 1 line
1.5.6.5
PATCH v3 5/10 ceph-rbd: refactor mount related functions, add helpers
In reply to Yehuda SadehRemoved some functions' static declarations, separated mount operation to __open_session and open_root_dentry for clients that don't need the latter (rbd). Added other helper functions that will be used later in the rbd.
--- fs/ceph/file.c | 46 +++++++++++++++ fs/ceph/osd_client.h | 1 + fs/ceph/super.c | 149 +++++++++++++++++++++++++++++++++++++------------ fs/ceph/super.h | 27 ++++++++- 4 files changed, 183 insertions(+), 40 deletions(-)
fs/ceph/file.c +46 -0
@@ -363,6 +363,52 @@ static int copy_user_to_page_vector(struct page **pages,return len; } +int ceph_copy_to_page_vector (struct page **pages, +const char *data, +loff_t off,size_t len) +{ +int i =0 ; +size_t po = off & ~PAGE_CACHE_MASK ; +size_t left = len; +size_t l; + +while (left >0 ) { + l =min_t (size_t,PAGE_CACHE_SIZE -po, left); +memcpy (page_address (pages[i]) + po, data, l); + data += l; + left -= l; + po += l; +if (po ==PAGE_CACHE_SIZE ) { + po =0 ; + i++; + } + } +return len; +} + +int ceph_copy_from_page_vector (struct page **pages, +char *data, +loff_t off,size_t len) +{ +int i =0 ; +size_t po = off & ~PAGE_CACHE_MASK ; +size_t left = len; +size_t l; + +while (left >0 ) { + l =min_t (size_t,PAGE_CACHE_SIZE -po, left); +memcpy (data,page_address (pages[i]) + po, l); + data += l; + left -= l; + po += l; +if (po ==PAGE_CACHE_SIZE ) { + po =0 ; + i++; + } + } +return len; +} +/* * copy user data from a page vector into a user pointer */
fs/ceph/osd_client.h +1 -0
@@ -69,6 +69,7 @@ struct ceph_osd_request {struct list_head r_unsafe_item;struct inode *r_inode;/* for use by callbacks */ +void *r_priv;/* ditto */ char r_oid[40 ];/* object name */ int r_oid_len;
fs/ceph/super.c +112 -37
233 lines not on this page · raw message
fs/ceph/super.h +24 -3
@@ -48,14 +48,11 @@#define ceph_test_opt (client, opt) \ (!!((client)->mount_args->flags & CEPH_OPT_##opt)) -struct ceph_mount_args {int sb_flags;int flags;struct ceph_fsid fsid;struct ceph_entity_addr my_addr; -int num_mon; -struct ceph_entity_addr *mon_addr;int mount_timeout;int osd_idle_ttl;int osd_timeout;
@@ -67,6 +64,13 @@ struct ceph_mount_args {int cap_release_safety;int max_readdir;/* max readdir result (entires) */ int max_readdir_bytes;/* max readdir result (bytes) */ + +/* any type that can't be simply compared or doesn't need +need to be compared should go beyond this point, +ceph_compare_mount_args() should be updated accordingly */ +struct ceph_entity_addr *mon_addr;/* should be the first +pointer type of args */ +int num_mon;char *snapdir_name;/* default ".snap" */ char *name;char *secret;
@@ -739,6 +743,16 @@ extern struct kmem_cache *ceph_file_cachep;extern const char *ceph_msg_type_name (int type);extern int ceph_check_fsid (struct ceph_client *client,struct ceph_fsid *fsid); +extern struct ceph_mount_args *parse_mount_args (int flags,char *options, +const char *dev_name, +const char **path); +extern void ceph_destroy_mount_args (struct ceph_mount_args *args); +extern int ceph_compare_mount_args (struct ceph_mount_args *new_args, +struct ceph_client *client); +extern struct ceph_client *ceph_create_client (struct ceph_mount_args *args, +int need_mdsc); +extern void ceph_destroy_client (struct ceph_client *client); +extern int ceph_open_session (struct ceph_client *client);#define FSID_FORMAT "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-" \ "%02x%02x%02x%02x%02x%02x"
@@ -849,6 +863,13 @@ extern int ceph_mmap(struct file *file, struct vm_area_struct *vma);/* file.c */ extern const struct file_operations ceph_file_fops;extern const struct address_space_operations ceph_aops; +extern int ceph_copy_to_page_vector (struct page **pages, +const char *data, +loff_t off,size_t len); +extern int ceph_copy_from_page_vector (struct page **pages, +char *data, +loff_t off,size_t len); +extern struct page **ceph_alloc_page_vector (int num_pages,gfp_t flags);extern int ceph_open (struct inode *inode,struct file *file); extern struct dentry *ceph_lookup_open(struct inode *dir, struct dentry *dentry, struct nameidata *nd, int mode,
Signature · 1 line
1.5.6.5
PATCH v3 6/10 ceph-rbd: generalize mon requests, add pool op support
In reply to Yehuda SadehGeneralize the current statfs synchronous requests, and support pool_ops.
--- fs/ceph/mon_client.c | 170 +++++++++++++++++++++++++++++++++++++++++++++----- fs/ceph/mon_client.h | 5 ++ 2 files changed, 158 insertions(+), 17 deletions(-)
fs/ceph/mon_client.c +153 -17
@@ -349,7 +349,7 @@ out: }/* -* statfs +* generic requests (e.g., statfs, poolop) */ static struct ceph_mon_generic_request *__lookup_generic_req (struct ceph_mon_client *monc,u64 tid)
@@ -442,6 +442,35 @@ static struct ceph_msg *get_generic_reply(struct ceph_connection *con,return m; } +static int do_generic_request (struct ceph_mon_client *monc, +struct ceph_mon_generic_request *req) +{ +int err; + +/* register request */ +mutex_lock (&monc->mutex); + req->tid = ++monc->last_tid; + req->request->hdr.tid =cpu_to_le64 (req->tid); +__insert_generic_request (monc, req); + monc->num_generic_requests++; +ceph_con_send (monc->con,ceph_msg_get (req->request)); +mutex_unlock (&monc->mutex); + + err =wait_for_completion_interruptible (&req->completion); + +mutex_lock (&monc->mutex); +rb_erase (&req->node, &monc->generic_request_tree); + monc->num_generic_requests--; +mutex_unlock (&monc->mutex); + +if (!err) + err = req->result; +return err; +} + +/* +* statfs +*/ static void handle_statfs_reply (struct ceph_mon_client *monc,struct ceph_msg *msg) {
@@ -468,7 +497,7 @@ static void handle_statfs_reply(struct ceph_mon_client *monc,return ; bad: -pr_err ("corrupt generic reply, no tid\n" ); +pr_err ("corrupt generic reply, tid %llu\n" , tid);ceph_msg_dump (msg); }
@@ -487,6 +516,7 @@ int ceph_monc_do_statfs(struct ceph_mon_client *monc, struct ceph_statfs *buf)kref_init (&req->kref); req->buf = buf; + req->buf_len =sizeof (*buf);init_completion (&req->completion); err = -ENOMEM ;
@@ -504,33 +534,134 @@ int ceph_monc_do_statfs(struct ceph_mon_client *monc, struct ceph_statfs *buf) h->monhdr.session_mon_tid =0 ; h->fsid = monc->monmap->fsid; -/* register request */ -mutex_lock (&monc->mutex); - req->tid = ++monc->last_tid; - req->request->hdr.tid =cpu_to_le64 (req->tid); -__insert_generic_request (monc, req); - monc->num_generic_requests++; -mutex_unlock (&monc->mutex); + err =do_generic_request (monc, req); -/* send request and wait */ -ceph_con_send (monc->con,ceph_msg_get (req->request)); - err =wait_for_completion_interruptible (&req->completion); +out: +kref_put (&req->kref, release_generic_request); +return err; +} + +/* +* pool ops +*/ +static int get_poolop_reply_buf (const char *src,size_t src_len, +char *dst,size_t dst_len) +{ +u32 buf_len; + +if (src_len !=sizeof (u32) + dst_len) +return -EINVAL ; + + buf_len =le32_to_cpu (*(u32 *)src); +if (buf_len != dst_len) +return -EINVAL ; + +memcpy (dst, src +sizeof (u32), dst_len); +return 0 ; +} + +static void handle_poolop_reply (struct ceph_mon_client *monc, +struct ceph_msg *msg) +{ +struct ceph_mon_generic_request *req; +struct ceph_mon_poolop_reply *reply = msg->front.iov_base; +u64 tid =le64_to_cpu (msg->hdr.tid); + +if (msg->front.iov_len <sizeof (*reply)) + goto bad; +dout ("handle_poolop_reply %p tid %llu\n" , msg, tid);mutex_lock (&monc->mutex); -rb_erase (&req->node, &monc->generic_request_tree); - monc->num_generic_requests--; + req =__lookup_generic_req (monc, tid); +if (req) { +if (req->buf_len && +get_poolop_reply_buf (msg->front.iov_base +sizeof (*reply), + msg->front.iov_len -sizeof (*reply), + req->buf, req->buf_len) <0 ) { +mutex_unlock (&monc->mutex); + goto bad; + } + req->result =le32_to_cpu (reply->reply_code); +get_generic_request (req); + }mutex_unlock (&monc->mutex); +if (req) { +complete (&req->completion); +put_generic_request (req); + } +return ; -if (!err) - err = req->result; +bad: +pr_err ("corrupt generic reply, tid %llu\n" , tid); +ceph_msg_dump (msg); +} + +/* +* Do a synchronous pool op. +*/ +int ceph_monc_do_poolop (struct ceph_mon_client *monc,u32 op, +u32 pool,u64 snapid, +char *buf,int len) +{ +struct ceph_mon_generic_request *req; +struct ceph_mon_poolop *h; +int err; + + req =kzalloc (sizeof (*req),GFP_NOFS ); +if (!req) +return -ENOMEM ; + +kref_init (&req->kref); + req->buf = buf; + req->buf_len = len; +init_completion (&req->completion); + + err = -ENOMEM ; + req->request =ceph_msg_new (CEPH_MSG_POOLOP ,sizeof (*h),GFP_NOFS ); +if (!req->request) + goto out; + req->reply =ceph_msg_new (CEPH_MSG_POOLOP_REPLY ,1024 ,GFP_NOFS ); +if (!req->reply) + goto out; + +/* fill out request */ + req->request->hdr.version =cpu_to_le16 (2 ); + h = req->request->front.iov_base; + h->monhdr.have_version =0 ; + h->monhdr.session_mon =cpu_to_le16 (-1 ); + h->monhdr.session_mon_tid =0 ; + h->fsid = monc->monmap->fsid; + h->pool =cpu_to_le32 (pool); + h->op =cpu_to_le32 (op); + h->auid =0 ; + h->snapid =cpu_to_le64 (snapid); + h->name_len =0 ; + + err =do_generic_request (monc, req); out:kref_put (&req->kref, release_generic_request);return err; } +int ceph_monc_create_snapid (struct ceph_mon_client *monc, +u32 pool,u64 *snapid) +{ +return ceph_monc_do_poolop (monc,POOL_OP_CREATE_UNMANAGED_SNAP , + pool,0 , (char *)snapid,sizeof (*snapid)); + +} + +int ceph_monc_delete_snapid (struct ceph_mon_client *monc, +u32 pool,u64 snapid) +{ +return ceph_monc_do_poolop (monc,POOL_OP_CREATE_UNMANAGED_SNAP , + pool, snapid,0 ,0 ); + +} +/* -* Resend pending statfs requests. +* Resend pending generic requests. */ static void __resend_generic_request (struct ceph_mon_client *monc) {
@@ -782,6 +913,10 @@ static void dispatch(struct ceph_connection *con, struct ceph_msg *msg)handle_statfs_reply (monc, msg);break ; +case CEPH_MSG_POOLOP_REPLY : +handle_poolop_reply (monc, msg); +break ; +case CEPH_MSG_MON_MAP :ceph_monc_handle_map (monc, msg);break ;
@@ -820,6 +955,7 @@ static struct ceph_msg *mon_alloc_msg(struct ceph_connection *con,case CEPH_MSG_MON_SUBSCRIBE_ACK : m =ceph_msg_get (monc->m_subscribe_ack);break ; +case CEPH_MSG_POOLOP_REPLY :case CEPH_MSG_STATFS_REPLY :return get_generic_reply (con, hdr, skip);case CEPH_MSG_AUTH_REPLY :
fs/ceph/mon_client.h +5 -0
@@ -50,6 +50,7 @@ struct ceph_mon_generic_request {struct rb_node node;int result;void *buf; +int buf_len;struct completion completion;struct ceph_msg *request;/* original request */ struct ceph_msg *reply;/* and reply */
@@ -111,6 +112,10 @@ extern int ceph_monc_open_session(struct ceph_mon_client *monc);extern int ceph_monc_validate_auth (struct ceph_mon_client *monc); +extern int ceph_monc_create_snapid (struct ceph_mon_client *monc, +u32 pool,u64 *snapid); +extern int ceph_monc_delete_snapid (struct ceph_mon_client *monc, +u32 pool,u64 snapid);#endif
Signature · 1 line
1.5.6.5
PATCH v3 7/10 ceph-rbd: some super.c changes for rbd
In reply to Yehuda Sadeh - add 'snap' mount option
- add ceph_client_id helper
--- fs/ceph/super.c | 19 ++++++++++++++++++- fs/ceph/super.h | 2 ++ 2 files changed, 20 insertions(+), 1 deletions(-)
fs/ceph/super.c +18 -1
@@ -342,6 +342,7 @@ enum { Opt_snapdirname, Opt_name, Opt_secret, + Opt_snap, Opt_last_string,/* string args above */ Opt_ip,
@@ -374,6 +375,7 @@ static match_table_t arg_tokens = { {Opt_snapdirname, "snapdirname=%s"}, {Opt_name, "name=%s"}, {Opt_secret, "secret=%s"}, + {Opt_snap, "snap=%s"},/* string args above */ {Opt_ip, "ip=%s"}, {Opt_noshare, "noshare"},
@@ -508,6 +510,11 @@ struct ceph_mount_args *parse_mount_args(int flags, char *options, argstr[0 ].to-argstr[0 ].from, GFP_KERNEL);break ; +case Opt_snap: + args->snap =kstrndup (argstr[0 ].from, + argstr[0 ].to-argstr[0 ].from, +GFP_KERNEL ); +break ;/* misc */ case Opt_wsize:
@@ -582,6 +589,7 @@ void ceph_destroy_mount_args(struct ceph_mount_args *args)kfree (args->snapdir_name);kfree (args->name);kfree (args->secret); +kfree (args->snap);kfree (args); }
@@ -621,6 +629,10 @@ int ceph_compare_mount_args(struct ceph_mount_args *new_args,if (ret)return ret; + ret =strcmp_null (args1->snap, args2->snap); +if (ret) +return ret; +for (i =0 ; i < args1->num_mon; i++) {if (ceph_monmap_contains (client->monc.monmap, &args1->mon_addr[i]))
@@ -717,6 +729,11 @@ fail:return ERR_PTR (err); } +u64 ceph_client_id (struct ceph_client *client) +{ +return client->monc.auth->global_id; +} +void ceph_destroy_client (struct ceph_client *client) {dout ("destroy_client %p\n" , client);
@@ -767,7 +784,7 @@ int ceph_check_fsid(struct ceph_client *client, struct ceph_fsid *fsid) } }else {pr_info ("client%lld fsid " FSID_FORMAT "\n" , - client->monc.auth->global_id,PR_FSID (fsid)); +ceph_client_id (client),PR_FSID (fsid));memcpy (&client->fsid, fsid,sizeof (*fsid));ceph_debugfs_client_init (client); client->have_fsid = true;
fs/ceph/super.h +2 -0
@@ -74,6 +74,7 @@ struct ceph_mount_args {char *snapdir_name;/* default ".snap" */ char *name;char *secret; +char *snap;/* rbd snapshot */ };/*
@@ -751,6 +752,7 @@ extern int ceph_compare_mount_args(struct ceph_mount_args *new_args, struct ceph_client *client);extern struct ceph_client *ceph_create_client (struct ceph_mount_args *args,int need_mdsc); +extern u64 ceph_client_id (struct ceph_client *client);extern void ceph_destroy_client (struct ceph_client *client);extern int ceph_open_session (struct ceph_client *client);
Signature · 1 line
1.5.6.5
PATCH v3 8/10 ceph-rbd: osdc support for osd call and rollback operations
In reply to Yehuda SadehThis will be used for rbd snapshots administration.
--- fs/ceph/decode.h | 5 +++++ fs/ceph/osd_client.c | 22 ++++++++++++++++++++++ fs/ceph/osd_client.h | 6 ++++++ 3 files changed, 33 insertions(+), 0 deletions(-)
fs/ceph/decode.h +5 -0
@@ -189,6 +189,11 @@ static inline void ceph_encode_string(void **p, void *end,ceph_encode_need (p, end, n, bad); \ceph_encode_copy (p, pv, n); \ } while (0) +#define ceph_encode_string_safe (p, end, s, n, bad) \ + do { \ + ceph_encode_need(p, end, n, bad); \ + ceph_encode_string(p, end, s, n); \ + } while (0)#endif
fs/ceph/osd_client.c +22 -0
@@ -136,6 +136,7 @@ static int op_needs_trail(int op)case CEPH_OSD_OP_GETXATTR :case CEPH_OSD_OP_SETXATTR :case CEPH_OSD_OP_CMPXATTR : +case CEPH_OSD_OP_CALL :return 1 ;default :return 0 ;
@@ -286,6 +287,27 @@ static void osd_req_encode_op(struct ceph_osd_request *req, dst->payload_len =cpu_to_le32 (src->xattr.name_len + src->xattr.value_len);break ; +case CEPH_OSD_OP_CALL : +BUG_ON (!req->r_trail); + + dst->cls.class_len = src->cls.class_len; + dst->cls.method_len = src->cls.method_len; + dst->cls.indata_len =cpu_to_le32 (src->cls.indata_len); + +ceph_pagelist_append (req->r_trail, src->cls.class_name, + src->cls.class_len); +ceph_pagelist_append (req->r_trail, src->cls.method_name, + src->cls.method_len); +ceph_pagelist_append (req->r_trail, src->cls.indata, + src->cls.indata_len); + dst->payload_len =cpu_to_le32 (src->cls.class_len + + src->cls.method_len + + src->cls.indata_len); + +break ; +case CEPH_OSD_OP_ROLLBACK : + dst->snap.snapid =cpu_to_le64 (src->snap.snapid); +break ; } dst->payload_len =cpu_to_le32 (src->payload_len); }
fs/ceph/osd_client.h +6 -0
@@ -135,14 +135,20 @@ struct ceph_osd_req_op {__u8 cmp_mode;/* CEPH_OSD_CMPXATTR_MODE_* */ } xattr;struct { +const char *class_name;__u8 class_len; +const char *method_name;__u8 method_len;__u8 argc; +const char *indata;u32 indata_len; } cls;struct {u64 cookie, count; } pgls; +struct { +u64 snapid; + } snap; };u32 payload_len; };
Signature · 1 line
1.5.6.5
PATCH v3 9/10 ceph-rbd: sync common header and source files with server version
In reply to Yehuda Sadeh--- fs/ceph/ceph_fs.c | 50 +++++++++++++++++++++++------------------------ fs/ceph/ceph_fs.h | 8 +++--- fs/ceph/ceph_strings.c | 1 + fs/ceph/rados.h | 8 +++++++ 4 files changed, 37 insertions(+), 30 deletions(-)
fs/ceph/ceph_fs.c +24 -26
@@ -29,46 +29,44 @@ int ceph_file_layout_is_valid(const struct ceph_file_layout *layout)int ceph_flags_to_mode (int flags) { +int mode; +#ifdef O_DIRECTORY /* fixme */ if ((flags &O_DIRECTORY ) ==O_DIRECTORY )return CEPH_FILE_MODE_PIN ;#endif +if ((flags &O_APPEND ) ==O_APPEND ) + flags |=O_WRONLY ; + +if ((flags &O_ACCMODE ) ==O_RDWR ) + mode =CEPH_FILE_MODE_RDWR ; +else if ((flags &O_ACCMODE ) ==O_WRONLY ) + mode =CEPH_FILE_MODE_WR ; +else + mode =CEPH_FILE_MODE_RD ; +#ifdef O_LAZY if (flags &O_LAZY ) -return CEPH_FILE_MODE_LAZY ; + mode |=CEPH_FILE_MODE_LAZY ;#endif -if ((flags &O_APPEND ) ==O_APPEND ) - flags |=O_WRONLY ; - flags &=O_ACCMODE ; -if ((flags &O_RDWR ) ==O_RDWR ) -return CEPH_FILE_MODE_RDWR ; -if ((flags &O_WRONLY ) ==O_WRONLY ) -return CEPH_FILE_MODE_WR ; -return CEPH_FILE_MODE_RD ; +return mode; }int ceph_caps_for_mode (int mode) { -switch (mode) { -case CEPH_FILE_MODE_PIN : -return CEPH_CAP_PIN ; -case CEPH_FILE_MODE_RD : -return CEPH_CAP_PIN |CEPH_CAP_FILE_SHARED | +int caps =CEPH_CAP_PIN ; + +if (mode &CEPH_FILE_MODE_RD ) + caps |=CEPH_CAP_FILE_SHARED |CEPH_CAP_FILE_RD |CEPH_CAP_FILE_CACHE ; -case CEPH_FILE_MODE_RDWR : -return CEPH_CAP_PIN |CEPH_CAP_FILE_SHARED | -CEPH_CAP_FILE_EXCL | -CEPH_CAP_FILE_RD |CEPH_CAP_FILE_CACHE | -CEPH_CAP_FILE_WR |CEPH_CAP_FILE_BUFFER | -CEPH_CAP_AUTH_SHARED |CEPH_CAP_AUTH_EXCL | -CEPH_CAP_XATTR_SHARED |CEPH_CAP_XATTR_EXCL ; -case CEPH_FILE_MODE_WR : -return CEPH_CAP_PIN |CEPH_CAP_FILE_SHARED | -CEPH_CAP_FILE_EXCL | +if (mode &CEPH_FILE_MODE_WR ) + caps |=CEPH_CAP_FILE_EXCL |CEPH_CAP_FILE_WR |CEPH_CAP_FILE_BUFFER |CEPH_CAP_AUTH_SHARED |CEPH_CAP_AUTH_EXCL |CEPH_CAP_XATTR_SHARED |CEPH_CAP_XATTR_EXCL ; - } -return 0 ; +if (mode &CEPH_FILE_MODE_LAZY ) + caps |=CEPH_CAP_FILE_LAZYIO ; + +return caps; }
fs/ceph/ceph_fs.h +4 -4
@@ -55,16 +55,15 @@*/ #define CEPH_FEATURE_UID 1#define CEPH_FEATURE_NOSRCADDR 2 -#define CEPH_FEATURE_FLOCK 4#define CEPH_FEATURE_SUPPORTED_MON CEPH_FEATURE_UID|CEPH_FEATURE_NOSRCADDR#define CEPH_FEATURE_REQUIRED_MON CEPH_FEATURE_UID -#define CEPH_FEATURE_SUPPORTED_MDS CEPH_FEATURE_UID|CEPH_FEATURE_NOSRCADDR|CEPH_FEATURE_FLOCK +#define CEPH_FEATURE_SUPPORTED_MDS CEPH_FEATURE_UID|CEPH_FEATURE_NOSRCADDR#define CEPH_FEATURE_REQUIRED_MDS CEPH_FEATURE_UID#define CEPH_FEATURE_SUPPORTED_OSD CEPH_FEATURE_UID|CEPH_FEATURE_NOSRCADDR#define CEPH_FEATURE_REQUIRED_OSD CEPH_FEATURE_UID#define CEPH_FEATURE_SUPPORTED_CLIENT CEPH_FEATURE_NOSRCADDR -#define CEPH_FEATURE_REQUIRED_CLIENT CEPH_FEATURE_NOSRCADDR +#define CEPH_FEATURE_REQUIRED_CLIENT 0/*
@@ -563,7 +562,8 @@ int ceph_flags_to_mode(int flags); CEPH_CAP_FILE_EXCL)#define CEPH_CAP_ANY_WR (CEPH_CAP_ANY_EXCL | CEPH_CAP_ANY_FILE_WR)#define CEPH_CAP_ANY (CEPH_CAP_ANY_RD | CEPH_CAP_ANY_EXCL | \ - CEPH_CAP_ANY_FILE_WR | CEPH_CAP_PIN) + CEPH_CAP_ANY_FILE_WR | CEPH_CAP_FILE_LAZYIO | \ + CEPH_CAP_PIN)#define CEPH_CAP_LOCKS (CEPH_LOCK_IFILE | CEPH_LOCK_IAUTH | CEPH_LOCK_ILINK | \ CEPH_LOCK_IXATTR)
fs/ceph/ceph_strings.c +1 -0
@@ -28,6 +28,7 @@ const char *ceph_osd_op_name(int op)case CEPH_OSD_OP_TRUNCATE :return "truncate" ;case CEPH_OSD_OP_ZERO :return "zero" ;case CEPH_OSD_OP_WRITEFULL :return "writefull" ; +case CEPH_OSD_OP_ROLLBACK :return "rollback" ;case CEPH_OSD_OP_APPEND :return "append" ;case CEPH_OSD_OP_STARTSYNC :return "startsync" ;
fs/ceph/rados.h +8 -0
@@ -203,6 +203,7 @@ enum {CEPH_OSD_OP_TMAPGET =CEPH_OSD_OP_MODE_RD |CEPH_OSD_OP_TYPE_DATA |12 ,CEPH_OSD_OP_CREATE =CEPH_OSD_OP_MODE_WR |CEPH_OSD_OP_TYPE_DATA |13 , + CEPH_OSD_OP_ROLLBACK= CEPH_OSD_OP_MODE_WR | CEPH_OSD_OP_TYPE_DATA | 14,/** attrs **/ /* read */
@@ -272,6 +273,10 @@ static inline int ceph_osd_op_mode_modify(int op)return (op &CEPH_OSD_OP_MODE ) ==CEPH_OSD_OP_MODE_WR ; } +/* +* note that the following tmap stuff is also defined in the ceph librados.h +* any modification here needs to be updated there +*/ #define CEPH_OSD_TMAP_HDR 'h'#define CEPH_OSD_TMAP_SET 's'#define CEPH_OSD_TMAP_RM 'r'
@@ -350,6 +355,9 @@ struct ceph_osd_op {struct {__le64 cookie, count; } __attribute__ ((packed)) pgls; +struct { +__le64 snapid; + } __attribute__ ((packed)) snap; };__le32 payload_len; } __attribute__ ((packed));
Signature · 1 line
1.5.6.5
PATCH v3 10/10 ceph-rbd: rados block device
In reply to Yehuda SadehThis adds the rbd under fs/ceph. The rados block device is based on the drivers/block/osdblk.c. It allows mapping of rados (ther ceph block layer) objects over a block device. Each device has a metadata object, and data is being striped across different objects.
This includes adds snapshots capabilities to the ceph-rbd. The snapshots can be created per a single rbd-image, and the relevant snapshot information is being kept in the rbd header.
--- fs/ceph/Kconfig | 10 + fs/ceph/Makefile | 2 +- fs/ceph/README | 1 + fs/ceph/rbd.c | 1835 +++++++++++++++++++++++++++++++++++++++++++++++++++ fs/ceph/rbd.h | 8 + fs/ceph/rbd_types.h | 71 ++ fs/ceph/super.c | 8 + fs/ceph/super.h | 5 + 8 files changed, 1939 insertions(+), 1 deletions(-)
fs/ceph/Kconfig +10 -0
@@ -25,3 +25,13 @@ config CEPH_FS_PRETTYDEBUG If unsure, say N. +config CEPH_RBD + bool "Rados block device (RBD)" + depends on CEPH_FS + select CONFIG_BLOCK + default y + help + If you say Y here, ceph will include rbd, the RADOS block + device which stripes a block device over objects stored in + the Ceph distributed object store. +
fs/ceph/Makefile +1 -1
@@ -16,7 +16,7 @@ ceph-objs := super.o inode.o dir.o file.o addr.o ioctl.o \ auth.o auth_none.o \ crypto.o armor.o \ auth_x.o \ - ceph_fs.o ceph_strings.o ceph_hash.o ceph_frag.o + ceph_fs.o ceph_strings.o ceph_hash.o ceph_frag.o rbd.o else #Otherwise we were called directly from the command
fs/ceph/README +1 -0
@@ -7,6 +7,7 @@ src/include/ceph_fs.h fs/ceph/ceph_fs.h src/include/ceph_fs.cc fs/ceph/ceph_fs.c src/include/msgr.h fs/ceph/msgr.h src/include/rados.h fs/ceph/rados.h +src/include/rbd_types.h fs/ceph/rbd_types.h src/include/ceph_strings.cc fs/ceph/ceph_strings.c src/include/ceph_frag.h fs/ceph/ceph_frag.h src/include/ceph_frag.cc fs/ceph/ceph_frag.c
fs/ceph/rbd.c new file +1835 -0
1,835 lines not on this page · raw message
fs/ceph/rbd.h new file +8 -0
@@ -0,0 +1,8 @@ +#ifndef _FS_CEPH_RBD +#define _FS_CEPH_RBD + +extern void rbd_set_osdc (struct ceph_osd_client *o); +extern int __init rbd_init (void ); +extern void __exit rbd_exit(void); + +#endif
fs/ceph/rbd_types.h new file +71 -0
71 lines not on this page · raw message
fs/ceph/super.c +8 -0
@@ -18,6 +18,7 @@#include "super.h" #include "mon_client.h" #include "auth.h" +#include "rbd.h" /* * Ceph superblock operations
@@ -1151,8 +1152,14 @@ static int __init init_ceph(void)CEPH_MONC_PROTOCOL ,CEPH_MDSC_PROTOCOL ,CEPH_OSDC_PROTOCOL ,CEPH_OSDMAP_VERSION ,CEPH_OSDMAP_VERSION_EXT , CEPH_OSDMAP_INC_VERSION, CEPH_OSDMAP_INC_VERSION_EXT); + + ret =rbd_init (); +if (ret) + goto out_fs;return 0 ; +out_fs: +unregister_filesystem (&ceph_fs_type); out_icache:destroy_caches (); out_msgr:
@@ -1166,6 +1173,7 @@ out: static void __exit exit_ceph(void) {dout ("exit_ceph\n" ); +rbd_exit ();unregister_filesystem (&ceph_fs_type);ceph_caps_finalize ();destroy_caches ();
fs/ceph/super.h +5 -0
@@ -921,4 +921,9 @@ static inline struct inode *get_dentry_parent_inode(struct dentry *dentry)return NULL ; } +#ifndef CONFIG_CEPH_RBD +static inline int __init rbd_init (void ) {return 0 ; } +static inline void __exit rbd_exit(void) {} +#endif +#endif /* _FS_CEPH_SUPER_H */
Signature · 1 line
1.5.6.5