exofs RAID0 support
cover exofs RAID0 support
Submitted for review a striped IO support in exofs. Borrowing from pnfs-objects-layout, stripping can be optionally combined with mirroring.
The code was tested and passes tests. An associated patch to pnfs-objects-layout and pnfsd-exofs which enables striping export over pnfs, is also tested. (Not included in this patchset)
So without further ado:
[PATCH 1/8] exofs: debug print even less
[PATCH 2/8] exofs: Micro-optimize exofs_i_info
A couple of patches for next Kernel, not really related.
[PATCH 3/8] exofs: Recover in the case of read-passed-end-of-file [PATCH 4/8] exofs: Move layout related members to a layout structure [PATCH 5/8] exofs: unindent exofs_sbi_read
These are some preliminary preparations that stand by themselves
[PATCH 6/8] exofs: Define on-disk per-inode optional layout attribute [PATCH 7/8] exofs: RAID0 support
Finally the RAID0 support
[PATCH 8/8] exofs: convert io_state to use pages array instead of bio at input
And, a now requested change
Thank you for reviewing
Boaz
PATCH 1/8 exofs: debug print even less
* Last debug trimming left in some stupid print, remove them. Fixup some other prints
* Shift printing from inode.c to ios.c
* Add couple of prints when memory allocation fails.
--- fs/exofs/inode.c | 23 +++++++++++++---------- fs/exofs/ios.c | 38 ++++++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 16 deletions(-)
fs/exofs/inode.c +13 -10
@@ -193,7 +193,7 @@ static int __readpages_done(struct page_collect *pcol, bool do_unlock)else good_bytes = pcol->length - resid; -EXOFS_DBGMSG ("readpages_done(0x%lx) good_bytes=0x%llx" +EXOFS_DBGMSG2 ("readpages_done(0x%lx) good_bytes=0x%llx" " length=0x%lx nr_pages=%u\n" , pcol->inode->i_ino,_LLU (good_bytes), pcol->length, pcol->nr_pages);
@@ -222,7 +222,7 @@ static int __readpages_done(struct page_collect *pcol, bool do_unlock) }pcol_free (pcol); -EXOFS_DBGMSG ("readpages_done END\n" ); +EXOFS_DBGMSG2 ("readpages_done END\n" );return ret; }
@@ -290,7 +290,7 @@ static int read_exec(struct page_collect *pcol, bool is_sync)atomic_inc (&pcol->sbi->s_curr_pending); -EXOFS_DBGMSG ("read_exec obj=0x%llx start=0x%llx length=0x%lx\n" , +EXOFS_DBGMSG2 ("read_exec obj=0x%llx start=0x%llx length=0x%lx\n" , ios->obj.id,_LLU (ios->offset), pcol->length);/* pages ownership was passed to pcol_copy */
@@ -462,7 +462,7 @@ static void writepages_done(struct exofs_io_state *ios, void *p)else good_bytes = pcol->length - resid; -EXOFS_DBGMSG ("writepages_done(0x%lx) good_bytes=0x%llx" +EXOFS_DBGMSG2 ("writepages_done(0x%lx) good_bytes=0x%llx" " length=0x%lx nr_pages=%u\n" , pcol->inode->i_ino,_LLU (good_bytes), pcol->length, pcol->nr_pages);
@@ -490,7 +490,7 @@ static void writepages_done(struct exofs_io_state *ios, void *p)pcol_free (pcol);kfree (pcol); -EXOFS_DBGMSG ("writepages_done END\n" ); +EXOFS_DBGMSG2 ("writepages_done END\n" ); }static int write_exec (struct page_collect *pcol)
@@ -527,7 +527,7 @@ static int write_exec(struct page_collect *pcol) }atomic_inc (&pcol->sbi->s_curr_pending); -EXOFS_DBGMSG ("write_exec(0x%lx, 0x%llx) start=0x%llx length=0x%lx\n" , +EXOFS_DBGMSG2 ("write_exec(0x%lx, 0x%llx) start=0x%llx length=0x%lx\n" , pcol->inode->i_ino, pcol->pg_first,_LLU (ios->offset), pcol->length);/* pages ownership was passed to pcol_copy */
@@ -616,7 +616,7 @@ try_again: ret =pcol_add_page (pcol, page, len);if (unlikely (ret)) { -EXOFS_DBGMSG ("Failed pcol_add_page " +EXOFS_DBGMSG2 ("Failed pcol_add_page " "nr_pages=%u total_length=0x%lx\n" , pcol->nr_pages, pcol->length);
@@ -663,7 +663,7 @@ static int exofs_writepages(struct address_space *mapping,if (expected_pages <32L ) expected_pages =32L ; -EXOFS_DBGMSG ("inode(0x%lx) wbc->start=0x%llx wbc->end=0x%llx " +EXOFS_DBGMSG2 ("inode(0x%lx) wbc->start=0x%llx wbc->end=0x%llx " "nrpages=%lu start=0x%lx end=0x%lx expected_pages=%ld\n" , mapping->host->i_ino, wbc->range_start, wbc->range_end, mapping->nrpages, start, end, expected_pages);
@@ -1170,8 +1170,10 @@ static int exofs_update_inode(struct inode *inode, int do_sync)int ret; args =kzalloc (sizeof (*args),GFP_KERNEL ); -if (!args) +if (!args) { +EXOFS_DBGMSG ("Faild kzalloc of args\n" );return -ENOMEM ; + } fcb = &args->fcb;
@@ -1234,7 +1236,8 @@ static int exofs_update_inode(struct inode *inode, int do_sync) free_args:kfree (args); out: -EXOFS_DBGMSG ("ret=>%d\n" , ret); +EXOFS_DBGMSG ("(0x%lx) do_sync=%d ret=>%d\n" , + inode->i_ino, do_sync, ret);return ret; }
fs/exofs/ios.c +32 -6
@@ -26,6 +26,9 @@#include "exofs.h" +#define EXOFS_DBGMSG2(M...) do {} while (0) +/* #define EXOFS_DBGMSG2 EXOFS_DBGMSG */ +void exofs_make_credential (u8 cred_a[OSD_CAP_LEN ],const struct osd_obj_id *obj) {osd_sec_init_nosec_doall_caps (cred_a, obj, false, true);
@@ -73,6 +76,8 @@ int exofs_get_io_state(struct exofs_sb_info *sbi, struct exofs_io_state** pios)*/ ios =kzalloc (exofs_io_state_size (sbi->s_numdevs),GFP_KERNEL );if (unlikely (!ios)) { +EXOFS_DBGMSG ("Faild kzalloc bytes=%d\n" , +exofs_io_state_size (sbi->s_numdevs)); *pios =NULL ;return -ENOMEM ; }
@@ -276,6 +281,9 @@ int exofs_sbi_write(struct exofs_io_state *ios) bio =bio_kmalloc (GFP_KERNEL , ios->bio->bi_max_vecs);if (unlikely (!bio)) { +EXOFS_DBGMSG ( +"Faild to allocate BIO size=%u\n" , + ios->bio->bi_max_vecs); ret = -ENOMEM ; goto out; }
@@ -290,14 +298,21 @@ int exofs_sbi_write(struct exofs_io_state *ios)osd_req_write (or, &ios->obj, ios->offset, bio, ios->length); -/* EXOFS_DBGMSG("write sync=%d\n", sync);*/ +EXOFS_DBGMSG ("write(0x%llx) offset=0x%llx " +"length=0x%llx dev=%d\n" , +_LLU (ios->obj.id),_LLU (ios->offset), +_LLU (ios->length), i); }else if (ios->kern_buff) {osd_req_write_kern (or, &ios->obj, ios->offset, ios->kern_buff, ios->length); -/* EXOFS_DBGMSG("write_kern sync=%d\n", sync);*/ +EXOFS_DBGMSG2 ("write_kern(0x%llx) offset=0x%llx " +"length=0x%llx dev=%d\n" , +_LLU (ios->obj.id),_LLU (ios->offset), +_LLU (ios->length), i); }else {osd_req_set_attributes (or, &ios->obj); -/* EXOFS_DBGMSG("set_attributes sync=%d\n", sync);*/ +EXOFS_DBGMSG2 ("obj(0x%llx) set_attributes=%d dev=%d\n" , +_LLU (ios->obj.id), ios->out_attr_len, i); }if (ios->out_attr)
@@ -335,14 +350,25 @@ int exofs_sbi_read(struct exofs_io_state *ios)if (ios->bio) {osd_req_read (or, &ios->obj, ios->offset, ios->bio, ios->length); -/* EXOFS_DBGMSG("read sync=%d\n", sync);*/ +EXOFS_DBGMSG ("read(0x%llx) offset=0x%llx length=0x%llx" +" dev=%d\n" ,_LLU (ios->obj.id), +_LLU (ios->offset), +_LLU (ios->length), + first_dev); }else if (ios->kern_buff) {osd_req_read_kern (or, &ios->obj, ios->offset, ios->kern_buff, ios->length); -/* EXOFS_DBGMSG("read_kern sync=%d\n", sync);*/ +EXOFS_DBGMSG2 ("read_kern(0x%llx) offset=0x%llx " +"length=0x%llx dev=%d\n" , +_LLU (ios->obj.id), +_LLU (ios->offset), +_LLU (ios->length), + first_dev); }else {osd_req_get_attributes (or, &ios->obj); -/* EXOFS_DBGMSG("get_attributes sync=%d\n", sync);*/ +EXOFS_DBGMSG2 ("obj(0x%llx) get_attributes=%d dev=%d\n" , +_LLU (ios->obj.id), ios->in_attr_len, + first_dev); }if (ios->out_attr)
Signature · 1 line
1.6.6
PATCH 2/8 exofs: Micro-optimize exofs_i_info
In reply to Boaz Harroshoptimize the exofs_i_info struct usage by moving the embedded vfs_inode to be first. A compiler might optimize away an "add" operation with constant zero. (Which it cannot with other constants)
--- fs/exofs/exofs.h | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
fs/exofs/exofs.h +2 -2
@@ -78,13 +78,13 @@ struct exofs_sb_info {* our extension to the in-memory inode */ struct exofs_i_info { +struct inode vfs_inode;/* normal in-memory inode */ +wait_queue_head_t i_wq;/* wait queue for inode */ unsigned long i_flags;/* various atomic flags */ uint32_t i_data[EXOFS_IDATA ];/*short symlink names and device #s*/ uint32_t i_dir_start_lookup;/* which page to start lookup */ -wait_queue_head_t i_wq;/* wait queue for inode */ uint64_t i_commit_size;/* the object's written length */ uint8_t i_cred[OSD_CAP_LEN ];/* all-powerful credential */ -struct inode vfs_inode;/* normal in-memory inode */ };static inline osd_id exofs_oi_objno (struct exofs_i_info *oi)
Signature · 1 line
1.6.6
PATCH 3/8 exofs: Recover in the case of read-passed-end-of-file
In reply to Boaz HarroshIn check_io, implement the case of reading passed end of file, by clearing the pages and recover with no error. In a raid arrangement this can become a legitimate situation in case of holes in the file.
--- fs/exofs/ios.c | 36 ++++++++++++++++++++++++++++++------ 1 files changed, 30 insertions(+), 6 deletions(-)
fs/exofs/ios.c +30 -6
@@ -173,6 +173,21 @@ static int exofs_io_execute(struct exofs_io_state *ios)return ret; } +static void _clear_bio (struct bio *bio) +{ +struct bio_vec *bv; +unsigned i; + +__bio_for_each_segment (bv, bio, i,0 ) { +unsigned this_count = bv->bv_len; + +if (likely (PAGE_SIZE == this_count)) +clear_highpage (bv->bv_page); +else +zero_user (bv->bv_page, bv->bv_offset, this_count); + } +} +int exofs_check_io (struct exofs_io_state *ios,u64 *resid) {enum osd_err_priority acumulated_osd_err =0 ;
@@ -181,16 +196,25 @@ int exofs_check_io(struct exofs_io_state *ios, u64 *resid)for (i =0 ; i < ios->numdevs; i++) {struct osd_sense_info osi; -int ret =osd_req_decode_sense (ios->per_dev[i].or, &osi); +struct osd_request *or = ios->per_dev[i].or; +int ret; + +if (unlikely (!or)) +continue ; + ret =osd_req_decode_sense (or, &osi);if (likely (!ret))continue ; -if (unlikely (ret == -EFAULT )) { -EXOFS_DBGMSG ("%s: EFAULT Need page clear\n" , __func__); -/*FIXME: All the pages in this device range should: -* clear_highpage(page); -*/ +if (OSD_ERR_PRI_CLEAR_PAGES == osi.osd_err_pri) { +/* start read offset passed endof file */ +_clear_bio (ios->per_dev[i].bio); +EXOFS_DBGMSG ("start read offset passed end of file " +"offset=0x%llx, length=0x%llx\n" , +_LLU (ios->offset), +_LLU (ios->length)); + +continue ;/* we recovered */ }if (osi.osd_err_pri >= acumulated_osd_err) {
Signature · 1 line
1.6.6
PATCH 4/8 exofs: Move layout related members to a layout structure
In reply to Boaz Harrosh* Abstract away those members in exofs_sb_info that are related/needed by a layout into a new exofs_layout structure. Embed it in exofs_sb_info.
* At exofs_io_state receive/keep a pointer to an exofs_layout. No need for an exofs_sb_info pointer, all we need is at exofs_layout.
* Change any usage of above exofs_sb_info members to their new name.
--- fs/exofs/exofs.h | 24 ++++++++++++++++++------ fs/exofs/inode.c | 14 +++++++------- fs/exofs/ios.c | 36 +++++++++++++++++++----------------- fs/exofs/super.c | 51 +++++++++++++++++++++++++++------------------------ 4 files changed, 71 insertions(+), 54 deletions(-)
fs/exofs/exofs.h +18 -6
@@ -55,12 +55,18 @@/* u64 has problems with printk this will cast it to unsigned long long */ #define _LLU (x) (unsigned long long)(x) +struct exofs_layout { +osd_id s_pid;/* partition ID of file system*/ + +unsigned s_numdevs;/* Num of devices in array */ +struct osd_dev *s_ods[0 ];/* Variable length */ +}; +/* * our extension to the in-memory superblock */ struct exofs_sb_info {struct exofs_fscb s_fscb;/* Written often, pre-allocate*/ -osd_id s_pid;/* partition ID of file system*/ int s_timeout;/* timeout for OSD operations */ uint64_t s_nextid;/* highest object ID used */ uint32_t s_numfiles;/* number of files on fs */
@@ -69,9 +75,14 @@ struct exofs_sb_info {atomic_t s_curr_pending;/* number of pending commands */ uint8_t s_cred[OSD_CAP_LEN ];/* credential for the fscb */ -struct pnfs_osd_data_map data_map;/* Default raid to use */ -unsigned s_numdevs;/* Num of devices in array */ -struct osd_dev *s_ods[1 ];/* Variable length, minimum 1 */ +struct pnfs_osd_data_map data_map;/* Default raid to use +* FIXME: Needed ? +*/ +/* struct exofs_layout dir_layout;*/ /* Default dir layout */ +struct exofs_layout layout;/* Default files layout, +* contains the variable osd_dev +* array. Keep last */ +struct osd_dev *_min_one_dev[1 ];/* Place holder for one dev */ };/*
@@ -101,7 +112,7 @@ struct exofs_io_state {void *private;exofs_io_done_fn done; -struct exofs_sb_info *sbi; +struct exofs_layout *layout;struct osd_obj_id obj;u8 *cred;
@@ -189,7 +200,8 @@ void exofs_make_credential(u8 cred_a[OSD_CAP_LEN],int exofs_read_kern (struct osd_dev *od,u8 *cred,struct osd_obj_id *obj,u64 offset,void *p,unsigned length); -int exofs_get_io_state (struct exofs_sb_info *sbi,struct exofs_io_state ** ios); +int exofs_get_io_state (struct exofs_layout *layout, +struct exofs_io_state **ios);void exofs_put_io_state (struct exofs_io_state *ios);int exofs_check_io (struct exofs_io_state *ios,u64 *resid);
fs/exofs/inode.c +7 -7
@@ -65,7 +65,7 @@ static void _pcol_init(struct page_collect *pcol, unsigned expected_pages,/* Create master bios on first Q, later on cloning, each clone will be * allocated on it's destination Q */ - pcol->req_q =osd_request_queue (sbi->s_ods[0 ]); + pcol->req_q =osd_request_queue (sbi->layout.s_ods[0 ]); pcol->inode = inode; pcol->expected_pages = expected_pages;
@@ -99,7 +99,7 @@ static int pcol_try_alloc(struct page_collect *pcol) BIO_MAX_PAGES_KMALLOC);if (!pcol->ios) {/* First time allocate io_state */ -int ret =exofs_get_io_state (pcol->sbi, &pcol->ios); +int ret =exofs_get_io_state (&pcol->sbi->layout, &pcol->ios);if (ret)return ret;
@@ -872,7 +872,7 @@ static int exofs_get_inode(struct super_block *sb, struct exofs_i_info *oi,int ret; *obj_size = ~0 ; - ret =exofs_get_io_state (sbi, &ios); + ret =exofs_get_io_state (&sbi->layout, &ios);if (unlikely (ret)) {EXOFS_ERR ("%s: exofs_get_io_state failed.\n" , __func__);return ret;
@@ -1043,7 +1043,7 @@ static void create_done(struct exofs_io_state *ios, void *p)if (unlikely (ret)) {EXOFS_ERR ("object=0x%llx creation faild in pid=0x%llx" , -_LLU (exofs_oi_objno (oi)),_LLU (sbi->s_pid)); +_LLU (exofs_oi_objno (oi)),_LLU (sbi->layout.s_pid));/*TODO: When FS is corrupted creation can fail, object already * exist. Get rid of this asynchronous creation, if exist * increment the obj counter and try the next object. Until we
@@ -1104,7 +1104,7 @@ struct inode *exofs_new_inode(struct inode *dir, int mode)mark_inode_dirty (inode); - ret =exofs_get_io_state (sbi, &ios); + ret =exofs_get_io_state (&sbi->layout, &ios);if (unlikely (ret)) {EXOFS_ERR ("exofs_new_inode: exofs_get_io_state failed\n" );return ERR_PTR (ret);
@@ -1202,7 +1202,7 @@ static int exofs_update_inode(struct inode *inode, int do_sync) }else memcpy (fcb->i_data, oi->i_data,sizeof (fcb->i_data)); - ret =exofs_get_io_state (sbi, &ios); + ret =exofs_get_io_state (&sbi->layout, &ios);if (unlikely (ret)) {EXOFS_ERR ("%s: exofs_get_io_state failed.\n" , __func__); goto free_args;
@@ -1286,7 +1286,7 @@ void exofs_delete_inode(struct inode *inode)clear_inode (inode); - ret =exofs_get_io_state (sbi, &ios); + ret =exofs_get_io_state (&sbi->layout, &ios);if (unlikely (ret)) {EXOFS_ERR ("%s: exofs_get_io_state failed\n" , __func__);return ;
fs/exofs/ios.c +19 -17
@@ -67,23 +67,24 @@ out:return ret; } -int exofs_get_io_state (struct exofs_sb_info *sbi,struct exofs_io_state ** pios) +int exofs_get_io_state (struct exofs_layout *layout, +struct exofs_io_state **pios) {struct exofs_io_state *ios;/*TODO: Maybe use kmem_cach per sbi of size -* exofs_io_state_size(sbi->s_numdevs) +* exofs_io_state_size(layout->s_numdevs) */ - ios =kzalloc (exofs_io_state_size (sbi->s_numdevs),GFP_KERNEL ); + ios =kzalloc (exofs_io_state_size (layout->s_numdevs),GFP_KERNEL );if (unlikely (!ios)) {EXOFS_DBGMSG ("Faild kzalloc bytes=%d\n" , -exofs_io_state_size (sbi->s_numdevs)); +exofs_io_state_size (layout->s_numdevs)); *pios =NULL ;return -ENOMEM ; } - ios->sbi = sbi; - ios->obj.partition = sbi->s_pid; + ios->layout = layout; + ios->obj.partition = layout->s_pid; *pios = ios;return 0 ; }
@@ -238,10 +239,10 @@ int exofs_sbi_create(struct exofs_io_state *ios) {int i, ret; -for (i =0 ; i < ios->sbi->s_numdevs; i++) { +for (i =0 ; i < ios->layout->s_numdevs; i++) {struct osd_request *or; - or =osd_start_request (ios->sbi->s_ods[i],GFP_KERNEL ); + or =osd_start_request (ios->layout->s_ods[i],GFP_KERNEL );if (unlikely (!or)) {EXOFS_ERR ("%s: osd_start_request failed\n" , __func__); ret = -ENOMEM ;
@@ -262,10 +263,10 @@ int exofs_sbi_remove(struct exofs_io_state *ios) {int i, ret; -for (i =0 ; i < ios->sbi->s_numdevs; i++) { +for (i =0 ; i < ios->layout->s_numdevs; i++) {struct osd_request *or; - or =osd_start_request (ios->sbi->s_ods[i],GFP_KERNEL ); + or =osd_start_request (ios->layout->s_ods[i],GFP_KERNEL );if (unlikely (!or)) {EXOFS_ERR ("%s: osd_start_request failed\n" , __func__); ret = -ENOMEM ;
@@ -286,10 +287,10 @@ int exofs_sbi_write(struct exofs_io_state *ios) {int i, ret; -for (i =0 ; i < ios->sbi->s_numdevs; i++) { +for (i =0 ; i < ios->layout->s_numdevs; i++) {struct osd_request *or; - or =osd_start_request (ios->sbi->s_ods[i],GFP_KERNEL ); + or =osd_start_request (ios->layout->s_ods[i],GFP_KERNEL );if (unlikely (!or)) {EXOFS_ERR ("%s: osd_start_request failed\n" , __func__); ret = -ENOMEM ;
@@ -361,8 +362,9 @@ int exofs_sbi_read(struct exofs_io_state *ios)struct osd_request *or;unsigned first_dev = (unsigned )ios->obj.id; - first_dev %= ios->sbi->s_numdevs; - or =osd_start_request (ios->sbi->s_ods[first_dev],GFP_KERNEL ); + first_dev %= ios->layout->s_numdevs; + or =osd_start_request (ios->layout->s_ods[first_dev], +GFP_KERNEL );if (unlikely (!or)) {EXOFS_ERR ("%s: osd_start_request failed\n" , __func__); ret = -ENOMEM ;
@@ -438,7 +440,7 @@ int exofs_oi_truncate(struct exofs_i_info *oi, u64 size)__be64 newsize;int i, ret; -if (exofs_get_io_state (sbi, &ios)) +if (exofs_get_io_state (&sbi->layout, &ios))return -ENOMEM ; ios->obj.id =exofs_oi_objno (oi);
@@ -448,10 +450,10 @@ int exofs_oi_truncate(struct exofs_i_info *oi, u64 size) attr = g_attr_logical_length; attr.val_ptr = &newsize; -for (i =0 ; i < sbi->s_numdevs; i++) { +for (i =0 ; i < sbi->layout.s_numdevs; i++) {struct osd_request *or; - or =osd_start_request (sbi->s_ods[i],GFP_KERNEL ); + or =osd_start_request (sbi->layout.s_ods[i],GFP_KERNEL );if (unlikely (!or)) {EXOFS_ERR ("%s: osd_start_request failed\n" , __func__); ret = -ENOMEM ;
fs/exofs/super.c +27 -24
135 lines not on this page · raw message
Signature · 1 line
1.6.6
PATCH 5/8 exofs: unindent exofs_sbi_read
In reply to Boaz HarroshThe original idea was that a mirror read can be sub-divided to multiple devices. But this has very little gain and only at very large IOes so it's not going to be implemented soon.
--- fs/exofs/ios.c | 87 ++++++++++++++++++++++++------------------------------- 1 files changed, 38 insertions(+), 49 deletions(-)
fs/exofs/ios.c +38 -49
@@ -356,59 +356,48 @@ out:int exofs_sbi_read (struct exofs_io_state *ios) { -int i, ret; - -for (i =0 ; i <1 ; i++) { -struct osd_request *or; -unsigned first_dev = (unsigned )ios->obj.id; - - first_dev %= ios->layout->s_numdevs; - or =osd_start_request (ios->layout->s_ods[first_dev], -GFP_KERNEL ); -if (unlikely (!or)) { -EXOFS_ERR ("%s: osd_start_request failed\n" , __func__); - ret = -ENOMEM ; - goto out; - } - ios->per_dev[i].or = or; - ios->numdevs++; +struct osd_request *or; +struct exofs_per_dev_state *per_dev = &ios->per_dev[0 ]; +unsigned first_dev = (unsigned )ios->obj.id; -if (ios->bio) { -osd_req_read (or, &ios->obj, ios->offset, ios->bio, - ios->length); -EXOFS_DBGMSG ("read(0x%llx) offset=0x%llx length=0x%llx" -" dev=%d\n" ,_LLU (ios->obj.id), -_LLU (ios->offset), -_LLU (ios->length), - first_dev); - }else if (ios->kern_buff) { -osd_req_read_kern (or, &ios->obj, ios->offset, - ios->kern_buff, ios->length); -EXOFS_DBGMSG2 ("read_kern(0x%llx) offset=0x%llx " -"length=0x%llx dev=%d\n" , -_LLU (ios->obj.id), -_LLU (ios->offset), -_LLU (ios->length), - first_dev); - }else { -osd_req_get_attributes (or, &ios->obj); -EXOFS_DBGMSG2 ("obj(0x%llx) get_attributes=%d dev=%d\n" , -_LLU (ios->obj.id), ios->in_attr_len, - first_dev); - } + first_dev %= ios->layout->s_numdevs; + or =osd_start_request (ios->layout->s_ods[first_dev],GFP_KERNEL ); +if (unlikely (!or)) { +EXOFS_ERR ("%s: osd_start_request failed\n" , __func__); +return -ENOMEM ; + } + per_dev->or = or; + ios->numdevs++; + +if (ios->bio) { +osd_req_read (or, &ios->obj, ios->offset, ios->bio, ios->length); +EXOFS_DBGMSG ("read(0x%llx) offset=0x%llx length=0x%llx" +" dev=%d\n" ,_LLU (ios->obj.id), +_LLU (ios->offset),_LLU (ios->length), + first_dev); + }else if (ios->kern_buff) { +int ret =osd_req_read_kern (or, &ios->obj, ios->offset, + ios->kern_buff, ios->length); + +EXOFS_DBGMSG2 ("read_kern(0x%llx) offset=0x%llx " +"length=0x%llx dev=%d ret=>%d\n" , +_LLU (ios->obj.id),_LLU (ios->offset), +_LLU (ios->length), first_dev, ret); +if (unlikely (ret)) +return ret; + }else { +osd_req_get_attributes (or, &ios->obj); +EXOFS_DBGMSG2 ("obj(0x%llx) get_attributes=%d dev=%d\n" , +_LLU (ios->obj.id), ios->in_attr_len, first_dev); + } -if (ios->out_attr) -osd_req_add_set_attr_list (or, ios->out_attr, - ios->out_attr_len); +if (ios->out_attr) +osd_req_add_set_attr_list (or, ios->out_attr, ios->out_attr_len); -if (ios->in_attr) -osd_req_add_get_attr_list (or, ios->in_attr, - ios->in_attr_len); - } - ret =exofs_io_execute (ios); +if (ios->in_attr) +osd_req_add_get_attr_list (or, ios->in_attr, ios->in_attr_len); -out: -return ret; +return exofs_io_execute (ios); }int extract_attr_from_ios (struct exofs_io_state *ios,struct osd_attr *attr)
Signature · 1 line
1.6.6
PATCH 6/8 exofs: Define on-disk per-inode optional layout attribute
In reply to Boaz Harrosh* Layouts describe the way a file is spread on multiple devices. The layout information is stored in the objects attribute introduced in this patch.
* There can be multiple generating function for the layout. Currently defined:
- No attribute present - use below moving-window on global device table, all devices.
(This is the only one currently used in exofs)
- an obj_id generated moving window - the obj_id is a randomizing factor in the otherwise global map layout.
- An explicit layout stored, including a data_map and a device index list.
- More might be defined in future ...
* There are two attributes defined of the same structure:
A-data-files-layout - This layout is used by data-files. If present
at a directory, all files of that directory will be created with this layout.
A-meta-data-layout - This layout is used by a directory and other
meta-data information. Also inherited at creation of subdirectories.
* At creation time inodes are created with the layout specified above. A usermode utility may change the creation layout on a give directory or file. Which in the case of directories, will also apply to newly created files/subdirectories, children of that directory. In the simple unaltered case of a newly created exofs, no layout attributes are present, and all layouts adhere to the layout specified at the device-table.
* In case of a future file system loaded in an old exofs-driver. At iget(), the generating_function is inspected and if not supported will return an IO error to the application and the inode will not be loaded. So not to damage any data.
Note: After this patch we do not yet support any type of layout
only the RAID0 patch that enables striping at the super-block level will add support for RAID0 layouts above. This way we are past and future compatible and fully bisectable.
* Access to the device table is done by an accessor since it will change according to above information.
--- fs/exofs/common.h | 39 ++++++++++++++++++++++++++++++++++++ fs/exofs/exofs.h | 6 +++++ fs/exofs/inode.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++++---- fs/exofs/ios.c | 23 +++++++++++++++++---- 4 files changed, 114 insertions(+), 10 deletions(-)
fs/exofs/common.h +39 -0
@@ -55,6 +55,8 @@/* exofs Application specific page/attribute */ # define EXOFS_APAGE_FS_DATA (OSD_APAGE_APP_DEFINED_FIRST + 3)# define EXOFS_ATTR_INODE_DATA 1 +# define EXOFS_ATTR_INODE_FILE_LAYOUT 2 +# define EXOFS_ATTR_INODE_DIR_LAYOUT 3/* * The maximum number of files we can have is limited by the size of the
@@ -206,4 +208,41 @@ enum { (((name_len) + offsetof(struct exofs_dir_entry , name) + \EXOFS_DIR_ROUND ) & ~EXOFS_DIR_ROUND ) +/* +* The on-disk (optional) layout structure. +* sits in an EXOFS_ATTR_INODE_FILE_LAYOUT or EXOFS_ATTR_INODE_DIR_LAYOUT +* attribute, attached to any inode, usually to a directory. + */ + +enum exofs_inode_layout_gen_functions { +LAYOUT_MOVING_WINDOW =0 , +LAYOUT_IMPLICT =1 , +}; + +struct exofs_on_disk_inode_layout { +__le16 gen_func;/* One of enum exofs_inode_layout_gen_functions */ +__le16 pad; +union { +/* gen_func == LAYOUT_MOVING_WINDOW (default) */ +struct exofs_layout_sliding_window { +__le32 num_devices;/* first n devices in global-table*/ + } sliding_window __packed; + +/* gen_func == LAYOUT_IMPLICT */ +struct exofs_layout_implict_list { +struct exofs_dt_data_map data_map; +/* Variable array of size data_map.cb_num_comps. These +* are device indexes of the devices in the global table +*/ +__le32 dev_indexes[]; + } implict __packed; + }; +} __packed; + +static inline size_t exofs_on_disk_inode_layout_size (unsigned max_devs) +{ +return sizeof (struct exofs_on_disk_inode_layout ) + + max_devs *sizeof (__le32); +} +#endif /*ifndef __EXOFS_COM_H__*/
fs/exofs/exofs.h +6 -0
@@ -186,6 +186,12 @@ static inline struct exofs_i_info *exofs_i(struct inode *inode) }/* +* Given a layout, object_number and stripe_index return the associated global +* dev_index +*/ +unsigned exofs_layout_od_id (struct exofs_layout *layout, +osd_id obj_no,unsigned layout_index); +/* * Maximum count of links to a file */ #define EXOFS_LINK_MAX 32000
fs/exofs/inode.c +51 -5
87 lines not on this page · raw message
fs/exofs/ios.c +18 -5
@@ -107,6 +107,19 @@ void exofs_put_io_state(struct exofs_io_state *ios) } } +unsigned exofs_layout_od_id (struct exofs_layout *layout, +osd_id obj_no,unsigned layout_index) +{ +return layout_index; +} + +static inline struct osd_dev *exofs_ios_od (struct exofs_io_state *ios, +unsigned layout_index) +{ +return ios->layout->s_ods[ +exofs_layout_od_id (ios->layout, ios->obj.id, layout_index)]; +} +static void _sync_done (struct exofs_io_state *ios,void *p) {struct completion *waiting = p;
@@ -242,7 +255,7 @@ int exofs_sbi_create(struct exofs_io_state *ios)for (i =0 ; i < ios->layout->s_numdevs; i++) {struct osd_request *or; - or =osd_start_request (ios->layout->s_ods[i],GFP_KERNEL ); + or =osd_start_request (exofs_ios_od (ios, i),GFP_KERNEL );if (unlikely (!or)) {EXOFS_ERR ("%s: osd_start_request failed\n" , __func__); ret = -ENOMEM ;
@@ -266,7 +279,7 @@ int exofs_sbi_remove(struct exofs_io_state *ios)for (i =0 ; i < ios->layout->s_numdevs; i++) {struct osd_request *or; - or =osd_start_request (ios->layout->s_ods[i],GFP_KERNEL ); + or =osd_start_request (exofs_ios_od (ios, i),GFP_KERNEL );if (unlikely (!or)) {EXOFS_ERR ("%s: osd_start_request failed\n" , __func__); ret = -ENOMEM ;
@@ -290,7 +303,7 @@ int exofs_sbi_write(struct exofs_io_state *ios)for (i =0 ; i < ios->layout->s_numdevs; i++) {struct osd_request *or; - or =osd_start_request (ios->layout->s_ods[i],GFP_KERNEL ); + or =osd_start_request (exofs_ios_od (ios, i),GFP_KERNEL );if (unlikely (!or)) {EXOFS_ERR ("%s: osd_start_request failed\n" , __func__); ret = -ENOMEM ;
@@ -361,7 +374,7 @@ int exofs_sbi_read(struct exofs_io_state *ios)unsigned first_dev = (unsigned )ios->obj.id; first_dev %= ios->layout->s_numdevs; - or =osd_start_request (ios->layout->s_ods[first_dev],GFP_KERNEL ); + or =osd_start_request (exofs_ios_od (ios, first_dev),GFP_KERNEL );if (unlikely (!or)) {EXOFS_ERR ("%s: osd_start_request failed\n" , __func__);return -ENOMEM ;
@@ -442,7 +455,7 @@ int exofs_oi_truncate(struct exofs_i_info *oi, u64 size)for (i =0 ; i < sbi->layout.s_numdevs; i++) {struct osd_request *or; - or =osd_start_request (sbi->layout.s_ods[i],GFP_KERNEL ); + or =osd_start_request (exofs_ios_od (ios, i),GFP_KERNEL );if (unlikely (!or)) {EXOFS_ERR ("%s: osd_start_request failed\n" , __func__); ret = -ENOMEM ;
Signature · 1 line
1.6.6
PATCH 7/8 exofs: RAID0 support
In reply to Boaz HarroshWe now support striping over mirror devices. Including variable sized stripe_unit.
Some limits:
* stripe_unit must be a multiple of PAGE_SIZE
* stripe_unit * stripe_count is maximum upto 32-bit (4Gb)
Tested RAID0 over mirrors, RAID0 only, mirrors only. All check.
Design notes:
* I'm not using a vectored raid-engine mechanism yet. Following the pnfs-objects-layout data-map structure, "Mirror" is just a private case of "group_width" == 1, and RAID0 is a private case of "Mirrors" == 1. The performance lose of the general case over the particular special case optimization is totally negligible, also considering the extra code size.
* In general I added a prepare_stripes() stage that divides the to-be-io pages to the participating devices, the previous exofs_ios_write/read, now becomes _write/read_mirrors and a new write/read upper layer loops on all devices calling _write/read_mirrors. Effectively the prepare_stripes stage is the all secret.
Also truncate need fixing to accommodate for striping.
* In a RAID0 arrangement, in a regular usage scenario, if all inode layouts will start at the same device, the small files fill up the first device and the later devices stay empty, the farther the device the emptier it is.
To fix that, each inode will start at a different stripe_unit, according to it's obj_id modulus number-of-stripe-units. And will then span all stripe-units in the same incrementing order wrapping back to the beginning of the device table. We call it a stripe-units moving window.
Special consideration was taken to keep all devices in a mirror arrangement identical. So a broken osd-device could just be cloned from one of the mirrors and no FS scrubbing is needed. (We do that by rotating stripe-unit at a time and not a single device at a time.)
TODO:
We no longer verify object_length == inode->i_size in exofs_iget. (since i_size is stripped on multiple objects now).
I should introduce a multiple-device attribute reading, and use it in exofs_iget.
--- fs/exofs/exofs.h | 14 +++ fs/exofs/inode.c | 26 +---- fs/exofs/ios.c | 327 +++++++++++++++++++++++++++++++++++++++++++++--------- fs/exofs/super.c | 52 ++++++++- 4 files changed, 336 insertions(+), 83 deletions(-)
fs/exofs/exofs.h +14 -0
@@ -58,6 +58,17 @@struct exofs_layout {osd_id s_pid;/* partition ID of file system*/ +/* Our way of looking at the data_map */ +unsigned stripe_unit; +unsigned mirrors_p1; + +unsigned group_width;/* Without data integ components */ +unsigned data_integ;/* 0, 1, or 2 */ +unsigned group_depth; +unsigned group_count; + +enum exofs_inode_layout_gen_functions lay_func; +unsigned s_numdevs;/* Num of devices in array */ struct osd_dev *s_ods[0 ];/* Variable length */ };
@@ -133,6 +144,9 @@ struct exofs_io_state {struct exofs_per_dev_state {struct osd_request *or;struct bio *bio; +loff_t offset; +unsigned length; +unsigned dev; } per_dev[]; };
fs/exofs/inode.c +4 -22
@@ -869,18 +869,17 @@ static const struct osd_attr g_attr_inode_dir_layout = ATTR_DEF( 0);/* -* Read an inode from the OSD, and return it as is. We also return the size -* attribute in the 'obj_size' argument. +* Read the Linux inode info from the OSD, and return it as is. In exofs the +* inode info is in an application specific page/attribute of the osd-object. */ static int exofs_get_inode (struct super_block *sb,struct exofs_i_info *oi, -struct exofs_fcb *inode,uint64_t *obj_size) +struct exofs_fcb *inode) {struct exofs_sb_info *sbi = sb->s_fs_info;struct osd_attr attrs[] = { [0 ] = g_attr_inode_data, [1 ] = g_attr_inode_file_layout, [2 ] = g_attr_inode_dir_layout, - [3 ] = g_attr_logical_length, };struct exofs_io_state *ios;struct exofs_on_disk_inode_layout *layout;
@@ -944,15 +943,6 @@ static int exofs_get_inode(struct super_block *sb, struct exofs_i_info *oi, } } - *obj_size = ~0 ; - ret =extract_attr_from_ios (ios, &attrs[3 ]); -if (ret) { -EXOFS_ERR ("%s: extract_attr of logical_length failed\n" , - __func__); - goto out; - } - *obj_size =get_unaligned_be64 (attrs[3 ].val_ptr); - out:exofs_put_io_state (ios);return ret;
@@ -971,7 +961,6 @@ struct inode *exofs_iget(struct super_block *sb, unsigned long ino)struct exofs_i_info *oi;struct exofs_fcb fcb;struct inode *inode; -uint64_t obj_size;int ret; inode =iget_locked (sb, ino);
@@ -983,7 +972,7 @@ struct inode *exofs_iget(struct super_block *sb, unsigned long ino)__oi_init (oi);/* read the inode from the osd */ - ret =exofs_get_inode (sb, oi, &fcb, &obj_size); + ret =exofs_get_inode (sb, oi, &fcb);if (ret) goto bad_inode;
@@ -1004,13 +993,6 @@ struct inode *exofs_iget(struct super_block *sb, unsigned long ino) inode->i_blkbits =EXOFS_BLKSHIFT ; inode->i_generation =le32_to_cpu (fcb.i_generation); -if ((inode->i_size != obj_size) && - (!exofs_inode_is_fast_symlink (inode))) { -EXOFS_ERR ("WARNING: Size of inode=%llu != object=%llu\n" , - inode->i_size,_LLU (obj_size)); -/* FIXME: call exofs_inode_recovery() */ - } - oi->i_dir_start_lookup =0 ;if ((inode->i_nlink ==0 ) && (inode->i_mode ==0 )) {
fs/exofs/ios.c +273 -54
457 lines not on this page · raw message
fs/exofs/super.c +45 -7
71 lines not on this page · raw message
Signature · 1 line
1.6.6
PATCH 8/8 exofs: convert io_state to use pages array instead of bio at input
In reply to Boaz Harrosh* inode.c operations are full-pages based, and not actually true scatter-gather
* Lets us use more pages at once upto 512 (from 249) in 64 bit
* Brings us much much closer to be able to use exofs's io_state engine from objlayout driver. (Once I decide where to put the common code)
After RAID0 patch the outer (input) bio was never used as a bio, but was simply a page carrier into the raid engine. Even in the simple mirror/single-dev arrangement pages info was copied into a second bio. It is now easer to just pass a pages array into the io_state and prepare bio(s) once.
--- fs/exofs/exofs.h | 5 ++- fs/exofs/inode.c | 81 +++++++++++++++++++++++++++-------------------------- fs/exofs/ios.c | 46 +++++++++++++++++------------- 3 files changed, 71 insertions(+), 61 deletions(-)
fs/exofs/exofs.h +4 -1
@@ -131,7 +131,10 @@ struct exofs_io_state {loff_t offset;unsigned long length;void *kern_buff; -struct bio *bio; + +struct page **pages; +unsigned nr_pages; +unsigned pgbase;/* Attributes */ unsigned in_attr_len;
fs/exofs/inode.c +41 -40
224 lines not on this page · raw message
fs/exofs/ios.c +26 -20
106 lines not on this page · raw message
Signature · 1 line
1.6.6