exofs (was osdfs)

This patch ties all operation vectors into a file system superblock and registers the exofs file_system_type at module's load time.

* The file system control block (AKA on-disk superblock) resides in an object with a special ID (defined in common.h). Information included in the file system control block is used to fill the in-memory superblock structure at mount time. This object is created before the file system is used by mkexofs.c It contains information such as:
- The file system's magic number
- The next inode number to be allocated

Signed-off-by Boaz Harrosh <[email protected]>

---
 fs/exofs/Kbuild  |    2 +-
 fs/exofs/exofs.h |   30 ++++
 fs/exofs/inode.c |  195 +++++++++++++++++++++-
 fs/exofs/super.c |  502 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 727 insertions(+), 2 deletions(-)
 create mode 100644 fs/exofs/super.c

fs/exofs/Kbuild +1 -1

index 27c738c..e293cb9 100644
@@ -26,5 +26,5 @@ KBUILD_CPPFLAGS := -I$(OSD_INC) $(KBUILD_CPPFLAGS)
 
 endif
 
-exofs-objs := osd.o inode.o file.o symlink.o namei.o dir.o
+exofs-objs := osd.o inode.o file.o symlink.o namei.o dir.o super.o
 obj-$(CONFIG_EXOFS_FS) += exofs.o

fs/exofs/exofs.h +30 -0

index 7330b59..75c608d 100644
@@ -52,6 +52,17 @@
 #define _LLU(x) (unsigned long long)(x)
 
 /*
+ * struct to hold what we get from mount options
+ */
+struct exofs_mountopt {
+	const char *dev_name;
+	uint64_t pid;
+	int timeout;
+	bool mkfs;
+	int format; /*in Mbyte*/
+};
+
+/*
  * our extension to the in-memory superblock
  */
 struct exofs_sb_info {

@@ -110,6 +121,14 @@ static inline struct exofs_i_info *EXOFS_I(struct inode *inode)
 }
 
 /*
+ * ugly struct so that we can pass two arguments to update_inode's callback
+ */
+struct updatei_args {
+	struct exofs_sb_info	*sbi;
+	struct exofs_fcb	*fcb;
+};
+
+/*
  * Maximum count of links to a file
  */
 #define EXOFS_LINK_MAX           32000

@@ -188,12 +207,20 @@ void free_osd_req(struct osd_request *req);
 /* inode.c               */
 void exofs_truncate(struct inode *inode);
 extern struct inode *exofs_iget(struct super_block *, unsigned long);
+extern int exofs_write_inode(struct inode *, int);
+extern void exofs_delete_inode(struct inode *);
 struct inode *exofs_new_inode(struct inode *, int);
 int exofs_setattr(struct dentry *, struct iattr *);
 int exofs_write_begin(struct file *file, struct address_space *mapping,
 		loff_t pos, unsigned len, unsigned flags,
 		struct page **pagep, void **fsdata);
 
+/* super.c:              */
+#ifdef EXOFS_DEBUG
+void exofs_dprint_internal(char *str, ...);
+#endif
+extern void exofs_write_super(struct super_block *);
+
 /* dir.c:                */
 int exofs_add_link(struct dentry *, struct inode *);
 ino_t exofs_inode_by_name(struct inode *, struct dentry *);

@@ -223,6 +250,9 @@ extern struct address_space_operations exofs_aops;
 extern struct inode_operations exofs_dir_inode_operations;
 extern struct inode_operations exofs_special_inode_operations;
 
+/* super.c           */
+extern struct super_operations exofs_sops;
+
 /* symlink.c         */
 extern struct inode_operations exofs_symlink_inode_operations;
 extern struct inode_operations exofs_fast_symlink_inode_operations;

fs/exofs/inode.c +1 -0

index 25a562e..e24690b 100644
@@ -37,6 +37,7 @@
 #include "exofs.h"
 
 static int __readpage_filler(struct page *page, bool is_async_unlock);
+static int exofs_update_inode(struct inode *inode, int do_sync);
 
 /*
  * Test whether an inode is a fast symlink.

fs/exofs/inode.c +67 -0

+{
+	struct exofs_sb_info *sbi;
+	free_osd_req(req);
+	sbi = (struct exofs_sb_info *)p;
+	atomic_dec(&sbi->s_curr_pending);
+}
+
+/*
  * get_block_t - Fill in a buffer_head
  * An OSD takes care of block allocation so we just fake an allocation by
  * putting in the inode's sector_t in the buffer_head.

@@ -94,6 +107,62 @@ int exofs_write_begin_export(struct file *file, struct address_space *mapping,
 }
 
 /*
+ * Called when the refcount of an inode reaches zero.  We remove the object
+ * from the OSD here.  We make sure the object was created before we try and
+ * delete it.
+ */
+void exofs_delete_inode(struct inode *inode)
+{
+	struct exofs_i_info *oi = EXOFS_I(inode);
+	struct osd_request *req = NULL;
+	struct super_block *sb = inode->i_sb;
+	struct exofs_sb_info *sbi = sb->s_fs_info;
+	int ret;
+
+	truncate_inode_pages(&inode->i_data, 0);
+
+	if (is_bad_inode(inode))
+		goto no_delete;
+	mark_inode_dirty(inode);
+	exofs_update_inode(inode, inode_needs_sync(inode));
+
+	inode->i_size = 0;
+	if (inode->i_blocks)
+		exofs_truncate(inode);
+
+	clear_inode(inode);
+
+	req = prepare_osd_remove(sbi->s_dev, sbi->s_pid,
+				 inode->i_ino + EXOFS_OBJ_OFF);
+	if (!req) {
+		printk(KERN_ERR "ERROR: prepare_osd_remove failed\n");
+		return;
+	}
+
+	/* if we are deleting an obj that hasn't been created yet, wait */
+	if (!ObjCreated(oi)) {
+		if (!Obj2BCreated(oi))
+			BUG();
+		else
+			wait_event(oi->i_wq, ObjCreated(oi));
+	}
+
+	ret = exofs_async_op(req, delete_done, sbi, oi->i_cred);
+	if (ret) {
+		printk(KERN_ERR
+		       "ERROR: @exofs_delete_inode exofs_async_op failed\n");
+		free_osd_req(req);
+		return;
+	}
+	atomic_inc(&sbi->s_curr_pending);
+
+	return;
+
+no_delete:
+	clear_inode(inode);
+}
+
+/*
  * Callback function when writepage finishes.  Check for errors, unlock, clean
  * up, etc.
  */

@@ -610,6 +679,131 @@ bad_inode:
 }
 
 /*
+ * Callback function from exofs_update_inode().
+ */
+void updatei_done(struct osd_request *req, void *p)

fs/exofs/inode.c +34 -0

+}
+
+/*
+ * Write the inode to the OSD.  Just fill up the struct, and set the attribute
+ * synchronously or asynchronously depending on the do_sync flag.
+ */
+static int exofs_update_inode(struct inode *inode, int do_sync)
+{
+	struct exofs_i_info *oi = EXOFS_I(inode);
+	struct super_block *sb = inode->i_sb;
+	struct exofs_sb_info *sbi = sb->s_fs_info;
+	struct osd_request *req = NULL;
+	struct exofs_fcb *fcb = NULL;
+	int ret;
+	int n;
+
+	fcb = kmalloc(sizeof(struct exofs_fcb), GFP_KERNEL);
+	if (!fcb) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	fcb->i_mode = cpu_to_be16(inode->i_mode);
+	fcb->i_uid = cpu_to_be32(inode->i_uid);
+	fcb->i_gid = cpu_to_be32(inode->i_gid);
+	fcb->i_links_count = cpu_to_be16(inode->i_nlink);
+	fcb->i_ctime = cpu_to_be32(inode->i_ctime.tv_sec);
+	fcb->i_atime = cpu_to_be32(inode->i_atime.tv_sec);
+	fcb->i_mtime = cpu_to_be32(inode->i_mtime.tv_sec);
+	fcb->i_size = cpu_to_be64(oi->i_commit_size = i_size_read(inode));
+	fcb->i_generation = cpu_to_be32(inode->i_generation);
+
+	if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {
+		if (old_valid_dev(inode->i_rdev)) {

fs/exofs/inode.c +41 -0

+
+	req = prepare_osd_set_attr(sbi->s_dev, sbi->s_pid,
+				 (uint64_t) (inode->i_ino + EXOFS_OBJ_OFF));
+	if (!req) {
+		printk(KERN_ERR "ERROR: prepare set_attr failed.\n");
+		kfree(fcb);
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	prepare_set_attr_list_add_entry(req,
+					OSD_PAGE_NUM_IBM_UOBJ_FS_DATA,
+					OSD_ATTR_NUM_IBM_UOBJ_FS_DATA_INODE,
+					EXOFS_INO_ATTR_SIZE,
+					(unsigned char *)fcb);
+
+	if (!ObjCreated(oi)) {
+		if (!Obj2BCreated(oi))
+			BUG();
+		else
+			wait_event(oi->i_wq, ObjCreated(oi));
+	}
+
+	if (do_sync) {
+		ret = exofs_sync_op(req, sbi->s_timeout, oi->i_cred);
+		free_osd_req(req);
+		kfree(fcb);
+	} else {
+		struct updatei_args *args = NULL;
+
+		args = kmalloc(sizeof(struct updatei_args), GFP_KERNEL);
+		if (!args) {
+			kfree(fcb);
+			ret = -ENOMEM;
+			goto out;
+		}
+		args->sbi = sbi;
+		args->fcb = fcb;
+
+		ret = exofs_async_op(req, updatei_done, args, oi->i_cred);
+		if (ret) {

fs/exofs/inode.c +17 -1

+
+int exofs_write_inode(struct inode *inode, int wait)
+{
+	return exofs_update_inode(inode, wait);
+}
+
+int exofs_sync_inode(struct inode *inode)
+{
+	struct writeback_control wbc = {
+		.sync_mode = WB_SYNC_ALL,
+		.nr_to_write = 0,	/* sys_fsync did this */
+	};
+
+	return sync_inode(inode, &wbc);
+}
+
+/*
  * Set inode attributes - just call generic functions.
  */
 int exofs_setattr(struct dentry *dentry, struct iattr *iattr)

@@ -624,7 +818,6 @@ int exofs_setattr(struct dentry *dentry, struct iattr *iattr)
 	error = inode_setattr(inode, iattr);
 	return error;
 }
-
 /*
  * Callback function from exofs_new_inode().  The important thing is that we
  * set the ObjCreated flag so that other methods know that the object exists on

fs/exofs/super.c new file +197 -0

new file mode 100644
index 0000000..8ecf700
@@ -0,0 +1,502 @@
+/*
+ * Copyright (C) 2005, 2006
+ * Avishay Traeger ([email protected]) ([email protected])
+ * Copyright (C) 2005, 2006
+ * International Business Machines
+ *
+ * Copyrights for code taken from ext2:
+ *     Copyright (C) 1992, 1993, 1994, 1995
+ *     Remy Card ([email protected])
+ *     Laboratoire MASI - Institut Blaise Pascal
+ *     Universite Pierre et Marie Curie (Paris VI)
+ *     from
+ *     linux/fs/minix/inode.c
+ *     Copyright (C) 1991, 1992  Linus Torvalds
+ *
+ * This file is part of exofs.
+ *
+ * exofs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.  Since it is based on ext2, and the only
+ * valid version of GPL for the Linux kernel is version 2, the only valid
+ * version of GPL for exofs is version 2.
+ *
+ * exofs is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with exofs; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#include <linux/string.h>
+#include <linux/parser.h>
+#include <linux/vfs.h>
+#include <linux/random.h>
+
+#include "exofs.h"
+
+/******************************************************************************
+ * MOUNT OPTIONS
+ *****************************************************************************/
+
+/*
+ * exofs-specific mount-time options.
+ */
+enum { Opt_lun, Opt_tid, Opt_pid, Opt_to, Opt_mkfs, Opt_format, Opt_err };
+
+/*
+ * Our mount-time options.  These should ideally be 64-bit unsigned, but the
+ * kernel's parsing functions do not currently support that.  32-bit should be
+ * sufficient for most applications now.
+ */
+static match_table_t tokens = {
+	{Opt_pid, "pid=%u"},
+	{Opt_to, "to=%u"},
+	{Opt_err, NULL}
+};
+
+/*
+ * The main option parsing method.  Also makes sure that all of the mandatory
+ * mount options were set.
+ */
+static int parse_options(char *options, struct exofs_mountopt *opts)
+{
+	char *p;
+	substring_t args[MAX_OPT_ARGS];
+	int option;
+	int s_pid = 0;
+
+	EXOFS_DBGMSG("parse_options %s\n", options);
+	/* defaults */
+	memset(opts, 0, sizeof(*opts));
+	opts->timeout = BLK_DEFAULT_SG_TIMEOUT;
+
+	while ((p = strsep(&options, ",")) != NULL) {
+		int token;
+		if (!*p)
+			continue;
+
+		token = match_token(p, tokens, args);
+		switch (token) {
+		case Opt_pid:
+			if (match_int(&args[0], &option))
+				return -EINVAL;
+			if (option < 65536) {
+				EXOFS_ERR("Partition ID must be >= 65536");
+				return -EINVAL;
+			}
+			opts->pid = option;
+			s_pid = 1;
+			break;
+		case Opt_to:
+			if (match_int(&args[0], &option))
+				return -EINVAL;
+			if (option <= 0) {
+				EXOFS_ERR("Timout must be > 0");
+				return -EINVAL;
+			}
+			opts->timeout = option * HZ;
+			break;
+		}
+	}
+
+	if (!s_pid) {
+		EXOFS_ERR("Need to specify the following options:\n");
+		EXOFS_ERR("    -o pid=pid_no_to_use\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ * INODE CACHE
+ *****************************************************************************/
+
+/*
+ * Our inode cache.  Isn't it pretty?
+ */
+static struct kmem_cache *exofs_inode_cachep;
+
+/*
+ * Allocate an inode in the cache
+ */
+static struct inode *exofs_alloc_inode(struct super_block *sb)
+{
+	struct exofs_i_info *oi;
+
+	oi = kmem_cache_alloc(exofs_inode_cachep, GFP_KERNEL);
+	if (!oi)
+		return NULL;
+
+	oi->vfs_inode.i_version = 1;
+	return &oi->vfs_inode;
+}
+
+/*
+ * Remove an inode from the cache
+ */
+static void exofs_destroy_inode(struct inode *inode)
+{
+	kmem_cache_free(exofs_inode_cachep, EXOFS_I(inode));
+}
+
+/*
+ * Initialize the inode
+ */
+static void exofs_init_once(void *foo)
+{
+	struct exofs_i_info *oi = foo;
+
+	inode_init_once(&oi->vfs_inode);
+}
+
+/*
+ * Create and initialize the inode cache
+ */
+static int init_inodecache(void)
+{
+	exofs_inode_cachep = kmem_cache_create("exofs_inode_cache",
+					       sizeof(struct exofs_i_info),
+					       0, SLAB_RECLAIM_ACCOUNT,
+					       exofs_init_once);
+	if (exofs_inode_cachep == NULL)
+		return -ENOMEM;
+	return 0;
+}
+
+/*
+ * Destroy the inode cache
+ */
+static void destroy_inodecache(void)
+{
+	kmem_cache_destroy(exofs_inode_cachep);
+}
+
+/******************************************************************************
+ * SUPERBLOCK FUNCTIONS
+ *****************************************************************************/
+
+/*
+ * Write the superblock to the OSD
+ */
+void exofs_write_super(struct super_block *sb)
+{
+	struct exofs_sb_info *sbi;
+	struct exofs_fscb *fscb = NULL;
+	struct osd_request *req = NULL;
+
+	fscb = kzalloc(sizeof(struct exofs_fscb), GFP_KERNEL);
+	if (!fscb)
+		return;
+
+	lock_kernel();
+	sbi = sb->s_fs_info;

fs/exofs/super.c new file +48 -0

+	unlock_kernel();
+	kfree(fscb);
+}
+
+/*
+ * This function is called when the vfs is freeing the superblock.  We just
+ * need to free our own part.
+ */
+static void exofs_put_super(struct super_block *sb)
+{
+	int num_pend;
+	struct exofs_sb_info *sbi = sb->s_fs_info;
+
+	/* make sure there are no pending commands */
+	for (num_pend = atomic_read(&sbi->s_curr_pending); num_pend > 0;
+	     num_pend = atomic_read(&sbi->s_curr_pending)) {
+		wait_queue_head_t wq;
+		init_waitqueue_head(&wq);
+		wait_event_timeout(wq,
+				  (atomic_read(&sbi->s_curr_pending) == 0),
+				  msecs_to_jiffies(100));
+	}
+
+	osduld_put_device(sbi->s_dev);
+	kfree(sb->s_fs_info);
+	sb->s_fs_info = NULL;
+}
+
+/*
+ * Read the superblock from the OSD and fill in the fields
+ */
+static int exofs_fill_super(struct super_block *sb, void *data, int silent)
+{
+	struct inode *root;
+	struct exofs_mountopt *opts = data;
+	struct exofs_sb_info *sbi = NULL;    /*extended info                  */
+	struct exofs_fscb fscb;		     /*on-disk superblock info        */
+	struct osd_request *req = NULL;
+	int ret;
+
+	sbi = kzalloc(sizeof(*sbi), GFP_KERNEL);
+	if (!sbi)
+		return -ENOMEM;
+	sb->s_fs_info = sbi;
+
+	/* use mount options to fill superblock */
+	sbi->s_dev = osduld_path_lookup(opts->dev_name);
+	if (IS_ERR(sbi->s_dev)) {

fs/exofs/super.c new file +39 -0

+	strcpy(sb->s_id, "exofs");
+	sb->s_blocksize = EXOFS_BLKSIZE;
+	sb->s_blocksize_bits = EXOFS_BLKSHIFT;
+	atomic_set(&sbi->s_curr_pending, 0);
+	sb->s_bdev = NULL;
+	sb->s_dev = 0;
+
+	/* read data from on-disk superblock object */
+	make_credential(sbi->s_cred, sbi->s_pid, EXOFS_SUPER_ID);
+
+	req = prepare_osd_read(sbi->s_dev, sbi->s_pid, EXOFS_SUPER_ID,
+			       sizeof(struct exofs_fscb), 0, 0,
+			       (unsigned char *)(&fscb));
+	if (!req) {
+		if (!silent)
+			EXOFS_ERR("ERROR: could not prepare read request.\n");
+		ret = -ENOMEM;
+		goto free_sbi;
+	}
+
+	ret = exofs_sync_op(req, sbi->s_timeout, sbi->s_cred);
+	if (ret != 0) {
+		if (!silent)
+			EXOFS_ERR("ERROR: read super failed.\n");
+		ret = -EIO;
+		goto free_sbi;
+	}
+
+	sb->s_magic = fscb.s_magic;
+	sbi->s_nextid = fscb.s_nextid;
+	sbi->s_numfiles = fscb.s_numfiles;
+
+	/* make sure what we read from the object store is correct */
+	if (sb->s_magic != EXOFS_SUPER_MAGIC) {
+		if (!silent)
+			EXOFS_ERR("ERROR: Bad magic value\n");
+		ret = -EINVAL;
+		goto free_sbi;
+	}

fs/exofs/super.c new file +181 -0

+		ret = PTR_ERR(root);
+		goto free_sbi;
+	}
+	sb->s_root = d_alloc_root(root);
+	if (!sb->s_root) {
+		iput(root);
+		EXOFS_ERR("ERROR: get root inode failed\n");
+		ret = -ENOMEM;
+		goto free_sbi;
+	}
+
+	if (!S_ISDIR(root->i_mode)) {
+		dput(sb->s_root);
+		sb->s_root = NULL;
+		EXOFS_ERR("ERROR: corrupt root inode (mode = %hd)\n",
+		       root->i_mode);
+		ret = -EINVAL;
+		goto free_sbi;
+	}
+
+	ret = 0;
+out:
+	if (req)
+		free_osd_req(req);
+	return ret;
+
+free_sbi:
+	osduld_put_device(sbi->s_dev); /* NULL safe */
+	kfree(sbi);
+	goto out;
+}
+
+/*
+ * Set up the superblock (calls exofs_fill_super eventually)
+ */
+static int exofs_get_sb(struct file_system_type *type,
+			  int flags, const char *dev_name,
+			  void *data, struct vfsmount *mnt)
+{
+	struct exofs_mountopt opts;
+	int ret;
+
+	ret = parse_options((char *) data, &opts);
+	if (ret)
+		return ret;
+
+	opts.dev_name = dev_name;
+	return get_sb_nodev(type, flags, &opts, exofs_fill_super, mnt);
+}
+
+/*
+ * Return information about the file system state in the buffer.  This is used
+ * by the 'df' command, for example.
+ */
+static int exofs_statfs(struct dentry *dentry, struct kstatfs *buf)
+{
+	struct super_block *sb = dentry->d_sb;
+	struct exofs_sb_info *sbi = sb->s_fs_info;
+	uint8_t cred_a[OSD_CAP_LEN];
+	struct osd_request *req = NULL;
+	uint32_t page;
+	uint32_t attr;
+	uint16_t expected;
+	uint64_t capacity;
+	uint64_t used;
+	uint8_t *data;
+	int ret;
+
+	/* get used/capacity attributes */
+	make_credential(cred_a, sbi->s_pid, 0);
+
+	req = prepare_osd_get_attr(sbi->s_dev, sbi->s_pid, 0);
+	if (!req) {
+		EXOFS_ERR("ERROR: prepare get_attr failed.\n");
+		return -1;
+	}
+
+	prepare_get_attr_list_add_entry(req,
+			OSD_APAGE_PARTITION_QUOTAS,
+			OSD_ATTR_PQ_CAPACITY_QUOTA,
+			8);
+
+	prepare_get_attr_list_add_entry(req,
+			OSD_APAGE_PARTITION_INFORMATION,
+			OSD_ATTR_PI_USED_CAPACITY,
+			8);
+
+	ret = exofs_sync_op(req, sbi->s_timeout, cred_a);
+	if (ret)
+		goto out;
+
+	page = OSD_APAGE_PARTITION_QUOTAS;
+	attr = OSD_ATTR_PQ_CAPACITY_QUOTA;
+	expected = 8;
+	ret = extract_next_attr_from_req(req, &page, &attr, &expected, &data);
+	if (ret) {
+		EXOFS_ERR("ERROR: extract attr from req failed\n");
+		goto out;
+	}
+	capacity = be64_to_cpu(*((uint64_t *)data));
+
+	page = OSD_APAGE_PARTITION_INFORMATION;
+	attr = OSD_ATTR_PI_USED_CAPACITY;
+	expected = 8;
+	ret = extract_next_attr_from_req(req, &page, &attr, &expected, &data);
+	if (ret) {
+		EXOFS_ERR("ERROR: extract attr from req failed\n");
+		goto out;
+	}
+	used = be64_to_cpu(*((uint64_t *)data));
+
+	/* fill in the stats buffer */
+	buf->f_type = EXOFS_SUPER_MAGIC;
+	buf->f_bsize = EXOFS_BLKSIZE;
+	buf->f_blocks = (capacity >> EXOFS_BLKSHIFT);
+	buf->f_bfree = ((capacity - used) >> EXOFS_BLKSHIFT);
+	buf->f_bavail = buf->f_bfree;
+	buf->f_files = sbi->s_numfiles;
+	buf->f_ffree = EXOFS_MAX_ID - sbi->s_numfiles;
+	buf->f_namelen = EXOFS_NAME_LEN;
+out:
+	free_osd_req(req);
+
+	return ret;
+}
+
+struct super_operations exofs_sops = {
+	.alloc_inode    = exofs_alloc_inode,
+	.destroy_inode  = exofs_destroy_inode,
+	.write_inode    = exofs_write_inode,
+	.delete_inode   = exofs_delete_inode,
+	.put_super      = exofs_put_super,
+	.write_super    = exofs_write_super,
+	.statfs         = exofs_statfs,
+};
+
+/******************************************************************************
+ * INSMOD/RMMOD
+ *****************************************************************************/
+
+/*
+ * struct that describes this file system
+ */
+static struct file_system_type exofs_type = {
+	.owner          = THIS_MODULE,
+	.name           = "exofs",
+	.get_sb         = exofs_get_sb,
+	.kill_sb        = generic_shutdown_super,
+};
+
+static int __init init_exofs(void)
+{
+	int err;
+
+	err = init_inodecache();
+	if (err)
+		goto out;
+
+	err = register_filesystem(&exofs_type);
+	if (err)
+		goto out_d;
+
+	return 0;
+out_d:
+	destroy_inodecache();
+out:
+	return err;
+}
+
+static void __exit exit_exofs(void)
+{
+	unregister_filesystem(&exofs_type);
+	destroy_inodecache();
+}
+
+MODULE_AUTHOR("Avishay Traeger <[email protected]>");
+MODULE_DESCRIPTION("exofs");
+MODULE_LICENSE("GPL");
+
+module_init(init_exofs)
+module_exit(exit_exofs)

Signature · 1 line

1.6.0.1

Added some documentation in exofs.txt, as well as a BUGS file.

For further reading, operation instructions, example scripts and up to date infomation and code please see:
http://open-osd.org

Signed-off-by Boaz Harrosh <[email protected]>

+===============================================================================

Hi!

Quoted 7 lines

Added some documentation in exofs.txt, as well as a BUGS file.

For further reading, operation instructions, example scripts and up to date infomation and code please see:
http://open-osd.org

Signed-off-by Boaz Harrosh <[email protected]>

Documentation/filesystems/exofs.txt line 8

Quoted 8 lines

+===============================================================================

Documentation/filesystems/exofs.txt new file +7 -0

+WHAT IS EXOFS?
+===============================================================================
+
+exofs is a file system that uses an OSD and exports the API of a normal Linux
+file system. Users access exofs like any other local file system, and exofs
+will in turn issue commands to the local initiator.
+

Which tells me pretty much nothing. I guess it should explain what is OSD... there are way too many TLAs and ETLAs in FOSS world.
Pavel

Quoted 12 lines · Boaz Harrosh

This patch ties all operation vectors into a file system superblock and registers the exofs file_system_type at module's load time.

* The file system control block (AKA on-disk superblock) resides in an object with a special ID (defined in common.h). Information included in the file system control block is used to fill the in-memory superblock structure at mount time. This object is created before the file system is used by mkexofs.c It contains information such as:
- The file system's magic number
- The next inode number to be allocated

Quoted 1 line

Signed-off-by Boaz Harrosh <[email protected]>

Some minor comments below.

Quoted 1 line

---

Quoted 12 lines

fs/exofs/super.c new file +12 -0

+	sbi = kzalloc(sizeof(*sbi), GFP_KERNEL);
+	if (!sbi)
+		return -ENOMEM;
+	sb->s_fs_info = sbi;
+
+	/* use mount options to fill superblock */
+	sbi->s_dev = osduld_path_lookup(opts->dev_name);
+	if (IS_ERR(sbi->s_dev)) {
+		ret = PTR_ERR(sbi->s_dev);
+		sbi->s_dev = NULL;
+		goto free_sbi;
+	}

fs/exofs/super.c line 272

Quoted 6 lines

fs/exofs/super.c new file +6 -0

+
+	sbi->s_pid = opts->pid;
+	sbi->s_timeout = opts->timeout;
+
+	/* fill in some other data by hand */
+	memset(sb->s_id, 0, sizeof(sb->s_id));

wasn't it zeroed by kzalloc?

fs/exofs/super.c line 273

Quoted 1 line

fs/exofs/super.c new file +1 -0

+	strcpy(sb->s_id, "exofs");

We need a mechanism to prepare the file system (mkfs). I chose to implement that by means of a couple of mount-options. Because there is no user-mode API for committing OSD commands. And also, all this stuff is highly internal to the file system itself.

- Added two mount options mkfs=0/1,format=capacity_in_meg, so mkfs/format can be executed by kernel code just before mount. An mkexofs utility can now be implemented by means of a script that mounts and unmount the file system with proper options.

In this patch are all the osd infrastructure that will be used later by the file system.

Also the declarations of constants, on disk structures, and prototypes.

And the Kbuild+Kconfig files needed to build the exofs module.

...

fs/exofs/exofs.h new file +6 -0

+struct exofs_sb_info {
+	struct osd_dev	*s_dev;			/* returned by get_osd_dev    */
+	uint64_t	s_pid;			/* partition ID of file system*/
+	int		s_timeout;		/* timeout for OSD operations */
+	uint32_t	s_nextid;		/* highest object ID used     */
+	uint32_t	s_numfiles;		/* number of files on fs      */

fs/exofs/exofs.h new file +10 -0

+/*
+ * our extension to the in-memory inode
+ */
+struct exofs_i_info {
+	exofs_iflags_t i_flags;            /* various atomic flags            */
+	__le32	       i_data[EXOFS_IDATA];/*short symlink names and device #s*/
+	uint32_t       i_dir_start_lookup; /* which page to start lookup      */
+	wait_queue_head_t i_wq;            /* wait queue for inode            */
+	uint64_t       i_commit_size;      /* the object's written length     */
+	uint8_t        i_cred[OSD_CAP_LEN];/* all-powerful credential         */

fs/exofs/osd.c new file +12 -0

+{
+	struct osd_sense_info osi;
+	int ret = osd_req_decode_sense(req, &osi);
+
+	if (ret) { /* translate to Linux codes */
+		if (osi.additional_code == scsi_invalid_field_in_cdb) {
+			if (osi.cdb_field_offset == OSD_CFO_STARTING_BYTE)
+				ret = -EFAULT;
+			if (osi.cdb_field_offset == OSD_CFO_OBJECT_ID)
+				ret = -ENOENT;
+			else
+				ret = -EINVAL;

fs/exofs/osd.c new file +9 -0

+{
+	struct osd_obj_id obj = {
+		.partition = pid,
+		.id = oid
+	};
+
+	osd_sec_init_nosec_doall_caps(cred_a, &obj, false, true);
+}
+

...

implementation of the file_operations and inode_operations for regular data files.

Most file_operations are generic vfs implementations except:
- exofs_truncate will truncate the OSD object as well
- Generic file_fsync is not good for none_bd devices so open code it
- The default for .flush in Linux is todo nothing so call exofs_fsync on the file.

...

fs/exofs/file.c new file +3 -0

+	.truncate	= exofs_truncate,
+	.setattr	= exofs_setattr,
+};

OK Now we start to read and write from osd-objects, page-by-page. The page index is the object's offset.

...

fs/exofs/inode.c +32 -0

+	struct inode *inode = page->mapping->host;
+	struct exofs_sb_info *sbi = inode->i_sb->s_fs_info;
+
+	ret = check_ok(req);
+	free_osd_req(req);
+	atomic_dec(&sbi->s_curr_pending);
+
+	if (ret) {
+		if (ret == -ENOSPC)
+			set_bit(AS_ENOSPC, &page->mapping->flags);
+		else
+			set_bit(AS_EIO, &page->mapping->flags);
+
+		SetPageError(page);
+	}
+
+	end_page_writeback(page);
+	unlock_page(page);
+}
+
+/*
+ * Write a page to disk.  page->index gives us the page number.  The page is
+ * locked before this function is called.  We write asynchronously and then the
+ * callback function (writepage_done) is called.  We signify that the operation
+ * has completed by unlocking the page and calling end_page_writeback().
+ */
+static int exofs_writepage(struct page *page, struct writeback_control *wbc)
+{
+	struct inode *inode = page->mapping->host;
+	struct exofs_i_info *oi = EXOFS_I(inode);
+	loff_t i_size = i_size_read(inode);
+	unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;

fs/exofs/inode.c +24 -0

+		if (wbc->sync_mode == WB_SYNC_NONE) {
+			redirty_page_for_writepage(wbc, page);
+			unlock_page(page);
+			ret = 0;
+			goto out;
+		} else {
+			wait_event(oi->i_wq, ObjCreated(oi));
+		}
+	}
+
+	/* in this case, the page is within the limits of the file */
+	if (page->index < end_index)
+		goto do_it;
+
+	offset = i_size & (PAGE_CACHE_SIZE - 1);
+	len = offset;
+
+	/*in this case, the page is outside the limits (truncate in progress)*/
+	if (page->index >= end_index + 1 || !offset) {
+		unlock_page(page);
+		goto out;
+	}
+
+do_it:

fs/exofs/inode.c +20 -0

+                           
+	if (!req) {
+		printk(KERN_ERR "ERROR: writepage failed.\n");
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	oi->i_commit_size = min_t(uint64_t, oi->i_commit_size, len + start);
+
+	ret = exofs_async_op(req, writepage_done, (void *)page, oi->i_cred);
+	if (ret) {
+		free_osd_req(req);
+		goto fail;
+	}
+	atomic_inc(&sbi->s_curr_pending);
+out:
+	return ret;
+fail:
+	set_bit(AS_EIO, &page->mapping->flags);
+	end_page_writeback(page);

fs/exofs/inode.c +14 -0

+	struct inode *inode = page->mapping->host;
+	struct exofs_sb_info *sbi = inode->i_sb->s_fs_info;
+	int ret;
+
+	ret = check_ok(req);
+	free_osd_req(req);
+	atomic_dec(&sbi->s_curr_pending);
+
+	if (ret == 0) {
+
+		/* Everything is OK */
+		SetPageUptodate(page);
+		if (PageError(page))
+			ClearPageError(page);

fs/exofs/inode.c +19 -0

+		SetPageUptodate(page);
+		if (PageError(page))
+			ClearPageError(page);
+	} else /* Error */
+		SetPageError(page);
+
+	if (unlock)
+		unlock_page(page);
+
+	return ret;
+}
+
+void readpage_done(struct osd_request *req, void *p)
+{
+	__readpage_done(req, p, true);
+}
+
+/*
+ * Read a page from the OSD

fs/exofs/inode.c +5 -0

+	if (PageUptodate(page))
+		goto out;
+
+	if (page->index < end_index)
+		amount = PAGE_CACHE_SIZE;

fs/exofs/inode.c +47 -0

+	if (!req) {
+		printk(KERN_ERR "ERROR: readpage failed.\n");
+		ret = -ENOMEM;
+		unlock_page(page);
+		goto out;
+	}
+
+	atomic_inc(&sbi->s_curr_pending);
+	if (!is_async_unlock) {
+		exofs_sync_op(req, sbi->s_timeout, oi->i_cred);
+		ret = __readpage_done(req, page, false);
+	} else {
+		ret = exofs_async_op(req, readpage_done, page, oi->i_cred);
+		if (ret) {
+			free_osd_req(req);
+			unlock_page(page);
+			atomic_dec(&sbi->s_curr_pending);
+		}
+	}
+
+out:
+	return ret;
+}
+
+static int readpage_filler(struct page *page)
+{
+	int ret = __readpage_filler(page, true);
+
+	return ret;
+}
+
+/*
+ * We don't need the file
+ */
+static int exofs_readpage(struct file *file, struct page *page)
+{
+	return readpage_filler(page);
+}
+
+/*
+ * We don't need the data
+ */
+static int readpage_strip(void *data, struct page *page)
+{
+	return readpage_filler(page);
+}
+

fs/exofs/inode.c +7 -0

+	.readpage	= exofs_readpage,
+	.readpages	= exofs_readpages,
+	.writepage	= exofs_writepage,
+	.write_begin	= exofs_write_begin_export,
+	.write_end	= simple_write_end,
+	.writepages	= generic_writepages,
+};

implementation of directory and inode operations.

* A directory is treated as a file, and essentially contains a list of <file name, inode #> pairs for files that are found in that directory. The object IDs correspond to the files' inode numbers and are allocated using a 64bit incrementing global counter.
* Each file's control block (AKA on-disk inode) is stored in its object's attributes. This applies to both regular files and other types (directories, device files, symlinks, etc.).

...

 fs/exofs/dir.c   |  649 ++++++++++++++++++++++++++++++++++++++++++++++++++++++

+struct super_operations exofs_sops = {

Quoted 6 lines · On Tue, 16 Dec 2008 17:33:48 +0200 Boaz Harrosh <[email protected]> wrote:

We need a mechanism to prepare the file system (mkfs). I chose to implement that by means of a couple of mount-options. Because there is no user-mode API for committing OSD commands. And also, all this stuff is highly internal to the file system itself.

Quoted 4 lines

- Added two mount options mkfs=0/1,format=capacity_in_meg, so mkfs/format can be executed by kernel code just before mount. An mkexofs utility can now be implemented by means of a script that mounts and unmount the file system with proper options.

Doing mkfs in-kernel is unusual. I don't think the above description sufficiently helps the uninitiated understand why mkfs cannot be done in userspace as usual. Please flesh it out a bit.

What are the dependencies for this filesystem code? I assume that it depends on various block- and scsi-level patches? Which ones, and what is their status, and is this code even compileable without them?

Thanks.

fs/exofs/osd.c line 111

Quoted 7 lines

fs/exofs/osd.c new file +7 -0

+int prepare_get_attr_list_add_entry(struct osd_request *req,
+				    uint32_t page_num,
+				    uint32_t attr_num,
+				    uint32_t attr_len)
+{
+	struct osd_attr attr = {
+		.page = page_num,

Kernel developers expect a field called "page" to have type `struct page *'. osd_attr.page is thus designed to confuse.

Quoted 2 lines

...

Quoted 10 lines · On Tue, 16 Dec 2008 17:28:57 +0200 Boaz Harrosh <[email protected]> wrote:

implementation of directory and inode operations.

* A directory is treated as a file, and essentially contains a list of <file name, inode #> pairs for files that are found in that directory. The object IDs correspond to the files' inode numbers and are allocated using a 64bit incrementing global counter.
* Each file's control block (AKA on-disk inode) is stored in its object's attributes. This applies to both regular files and other types (directories, device files, symlinks, etc.).

Quoted 3 lines

...

 fs/exofs/dir.c   |  649 ++++++++++++++++++++++++++++++++++++++++++++++++++++++

yes, this does look rather ext2-like ;)

How long ago was the code cloned from ext2? iirc there have been a number of fairly subtle bugs fixed in ext2/dir.c over the past year or three. If the code was not quite recently cloned then I'd suggest that you spend a bit of time looking through the ext2 changelogs, see if there are any bugfixes which needs to be ported.

fs/exofs/inode.c int exofs_get_block(struct inode *inode, sector_t iblock, · line 186

Quoted 6 lines

fs/exofs/inode.c +6 -0

+
+	kaddr = page_address(page);
+
+	req = prepare_osd_write(sbi->s_dev, sbi->s_pid,
+			      inode->i_ino + EXOFS_OBJ_OFF, len, start, 0,
+			      kaddr);

Does prepare_osd_write() modify the memory at *kaddr? If so, does it do the needed flush_dcache_page()?

Quoted 6 lines

fs/exofs/inode.c +6 -0

+	} else if (ret == -EFAULT) {
+		char *kaddr;
+
+		/* In this case we were trying to read something that wasn't on
+		 * disk yet - return a page full of zeroes.  This should be OK,
+		 * because the object should be empty (if there was a write

fs/exofs/inode.c int exofs_get_block(struct inode *inode, sector_t iblock, · line 239

Quoted 4 lines

fs/exofs/inode.c +4 -0

+		 * before this read, the read would be waiting with the page
+		 * locked */
+		kaddr = page_address(page);
+		memset(kaddr, 0, PAGE_CACHE_SIZE);

There is I think a missing flsh_dcache_page() here. Use of the (somewhat misnamed) zero_user() would be an appropriate fix and cleanup.

fs/exofs/inode.c int exofs_get_block(struct inode *inode, sector_t iblock, · line 293

Quoted 8 lines

fs/exofs/inode.c +8 -0

+
+	/* this will be out of bounds, or doesn't exist yet */
+	if ((page->index >= end_index + 1) || !ObjCreated(oi) || !amount
+	    /*|| (i_start >= oi->i_commit_size)*/) {
+		kaddr = kmap_atomic(page, KM_USER0);
+		memset(kaddr, 0, PAGE_CACHE_SIZE);
+		flush_dcache_page(page);
+		kunmap_atomic(page, KM_USER0);

There's a flush_dcache_page() ;)

Could use clear_highpage() here.

Quoted 7 lines

fs/exofs/inode.c +7 -0

+		SetPageUptodate(page);
+		if (PageError(page))
+			ClearPageError(page);
+		if (is_async_unlock)
+			unlock_page(page);
+		goto out;
+	}

fs/exofs/inode.c int exofs_get_block(struct inode *inode, sector_t iblock, · line 306

Quoted 6 lines

fs/exofs/inode.c +6 -0

+
+	if (amount != PAGE_CACHE_SIZE) {
+		kaddr = kmap_atomic(page, KM_USER0);
+		memset(kaddr + amount, 0, PAGE_CACHE_SIZE - amount);
+		flush_dcache_page(page);
+		kunmap_atomic(page, KM_USER0);

Use zero_user()?

Quoted 11 lines · On Tue, 16 Dec 2008 17:17:25 +0200 Boaz Harrosh <[email protected]> wrote:

implementation of the file_operations and inode_operations for regular data files.

Most file_operations are generic vfs implementations except:
- exofs_truncate will truncate the OSD object as well
- Generic file_fsync is not good for none_bd devices so open code it
- The default for .flush in Linux is todo nothing so call exofs_fsync on the file.

...

fs/exofs/file.c line 51

Quoted 9 lines

fs/exofs/file.c new file +9 -0

+int exofs_file_fsync(struct file *filp, struct dentry *dentry, int datasync)
+{
+	int ret1, ret2;
+	struct address_space *mapping = filp->f_mapping;
+
+	ret1 = filemap_write_and_wait(mapping);
+	ret2 = file_fsync(filp, dentry, datasync);
+
+	return ret1 ? : ret2;

mutter. That gccism always makes me fall over dazed and confused. Maybe that's just me.

Did we really want to call file_fsync() if filemap_write_and_wait() failed?

Quoted 9 lines · Andrew Morton

Quoted 6 lines · On Tue, 16 Dec 2008 17:33:48 +0200 Boaz Harrosh <[email protected]> wrote:

We need a mechanism to prepare the file system (mkfs). I chose to implement that by means of a couple of mount-options. Because there is no user-mode API for committing OSD commands. And also, all this stuff is highly internal to the file system itself.

Quoted 8 lines

Quoted 4 lines

- Added two mount options mkfs=0/1,format=capacity_in_meg, so mkfs/format can be executed by kernel code just before mount. An mkexofs utility can now be implemented by means of a script that mounts and unmount the file system with proper options.

Doing mkfs in-kernel is unusual. I don't think the above description sufficiently helps the uninitiated understand why mkfs cannot be done in userspace as usual. Please flesh it out a bit.

There are a few main reasons.
- There is no user-mode API for initiating OSD commands. Such a subsystem would be hundredfold bigger then the mkfs code submitted. I think it would be hard and stupid to maintain a complex user-mode API just for creating a couple of objects and writing a couple of on disk structures.

- I intend to refactor the code further to make use of more super.c services, so to make this addition even smaller. Also future direction of raid over multiple objects will make even more kernel infrastructure needed which will need even more user-mode code duplication.
- I anticipate problems that are not yet addressed in this body of work but will be in the future, mainly that a single OSD-target (lun) can be shared by lots of FSs, and a single FS can span many OSD-targets. Some central management is much easier to do in Kernel.

Quoted 5 lines

What are the dependencies for this filesystem code? I assume that it depends on various block- and scsi-level patches? Which ones, and what is their status, and is this code even compileable without them?

This OSD-based file system is dependent on the open-osd initiator library code that I've submitted for inclusion for 2.6.29. It has been sitting in linux-next for a while now, and has not been receiving any comments for the last two updated patchsets I've sent to scsi-misc/lkml. However it has not yet been submitted into Jame's scsi-misc git tree, and James is the ultimate maintainer that should submit this work. I hope it will still be submitted into 2.6.29, as this code is totally self sufficient and does not endangers or changes any other Kernel subsystems. (All the needed ground work was already submitted to Linus since 2.6.26) So why should it not?

Quoted 9 lines · Andrew Morton

Quoted 6 lines · On Tue, 16 Dec 2008 17:33:48 +0200 Boaz Harrosh <[email protected]> wrote:

We need a mechanism to prepare the file system (mkfs). I chose to implement that by means of a couple of mount-options. Because there is no user-mode API for committing OSD commands. And also, all this stuff is highly internal to the file system itself.

Quoted 8 lines

Quoted 4 lines

- Added two mount options mkfs=0/1,format=capacity_in_meg, so mkfs/format can be executed by kernel code just before mount. An mkexofs utility can now be implemented by means of a script that mounts and unmount the file system with proper options.

Doing mkfs in-kernel is unusual. I don't think the above description sufficiently helps the uninitiated understand why mkfs cannot be done in userspace as usual. Please flesh it out a bit.

There are a few main reasons.
- There is no user-mode API for initiating OSD commands. Such a subsystem would be hundredfold bigger then the mkfs code submitted. I think it would be hard and stupid to maintain a complex user-mode API just for creating a couple of objects and writing a couple of on disk structures.
- I intend to refactor the code further to make use of more super.c services, so to make this addition even smaller. Also future direction of raid over multiple objects will make even more kernel infrastructure needed which will need even more user-mode code duplication.
- I anticipate problems that are not yet addressed in this body of work but will be in the future, mainly that a single OSD-target (lun) can be shared by lots of FSs, and a single FS can span many OSD-targets. Some central management is much easier to do in Kernel.

Quoted 5 lines

What are the dependencies for this filesystem code? I assume that it depends on various block- and scsi-level patches? Which ones, and what is their status, and is this code even compileable without them?

This OSD-based file system is dependent on the open-osd initiator library code that I've submitted for inclusion for 2.6.29. It has been sitting in linux-next for a while now, and has not been receiving any comments for the last two updated patchsets I've sent to scsi-misc/lkml. However it has not yet been submitted into Jame's scsi-misc git tree, and James is the ultimate maintainer that should submit this work. I hope it will still be submitted into 2.6.29, as this code is totally self sufficient and does not endangers or changes any other Kernel subsystems. (All the needed ground work was already submitted to Linus since 2.6.26) So why should it not?

Once the open-osd initiator library is accepted this file system could be accepted. I was hoping as a 2.6.30 time frame. (One Kernel after the open-osd library)

Quoted 1 line

Thanks.

Thank you dear Andrew for your most valuable input.

I will constify all the const needed code. will fix the global name space litter, will inline the macros and lower case the inlines. Will remove the typedefs.

I will reply to individual patches, I have a couple of questions. But all your comments are right and I will take care of them.

When, if, all is fixed, through which tree/maintainer can exofs be submitted?

Quoted 14 lines · Andrew Morton

fs/exofs/osd.c line 111

Quoted 7 lines

fs/exofs/osd.c new file +7 -0

+int prepare_get_attr_list_add_entry(struct osd_request *req,
+				    uint32_t page_num,
+				    uint32_t attr_num,
+				    uint32_t attr_len)
+{
+	struct osd_attr attr = {
+		.page = page_num,

Kernel developers expect a field called "page" to have type `struct page *'. osd_attr.page is thus designed to confuse.

Quoted 2 lines

...

Rant below (can be ignored):
This single fix will cause a massive change to the open-osd initiator patchset, (18 patches), and resubmission .I made the mistake because this name originates from a file that all naming conventions are taken from the OSD standard text. However this is no excuse for using a well known Kernel construct name. I will fix it. And will be more careful in the future.

Quoted 14 lines · Boaz Harrosh

Quoted 13 lines · Andrew Morton

Quoted 10 lines · On Tue, 16 Dec 2008 17:33:48 +0200 Boaz Harrosh <[email protected]> wrote:

We need a mechanism to prepare the file system (mkfs). I chose to implement that by means of a couple of mount-options. Because there is no user-mode API for committing OSD commands. And also, all this stuff is highly internal to the file system itself.

- Added two mount options mkfs=0/1,format=capacity_in_meg, so mkfs/format can be executed by kernel code just before mount. An mkexofs utility can now be implemented by means of a script that mounts and unmount the file system with proper options.

Quoted 8 lines

Quoted 3 lines

Doing mkfs in-kernel is unusual. I don't think the above description sufficiently helps the uninitiated understand why mkfs cannot be done in userspace as usual. Please flesh it out a bit.

There are a few main reasons.
- There is no user-mode API for initiating OSD commands. Such a subsystem would be hundredfold bigger then the mkfs code submitted. I think it would be hard and stupid to maintain a complex user-mode API just for creating a couple of objects and writing a couple of on disk structures.

This is really a reflection of the whole problem with the OSD paradigm.

In theory, a filesystem on OSD is a thin layer of metadata mapping objects to files. Get this right and the storage will manage things, like security and access and attributes (there's even a natural mapping to the VFS concept of extended attributes). Plus, the storage has enough information to manage persistence, backups and replication.

The real problem is that no-one has actually managed to come up with a useful VFS<->OSD mapping layer (even by extending or altering the VFS). Every filesystem that currently uses OSD has a separate direct OSD speaking interface (i.e. it slices out the block layer to do this and talks directly to the storage).

I suppose this could be taken to show that such a layer is impossibly complex, as you assert, but its lack is reflected in strange looking design decisions like in-kernel mkfs. It would also mean that there would be very little layered code sharing between ODS based filesystems.

Quoted 14 lines

- I intend to refactor the code further to make use of more super.c services, so to make this addition even smaller. Also future direction of raid over multiple objects will make even more kernel infrastructure needed which will need even more user-mode code duplication.
- I anticipate problems that are not yet addressed in this body of work but will be in the future, mainly that a single OSD-target (lun) can be shared by lots of FSs, and a single FS can span many OSD-targets. Some central management is much easier to do in Kernel.

Quoted 4 lines

What are the dependencies for this filesystem code? I assume that it depends on various block- and scsi-level patches? Which ones, and what is their status, and is this code even compileable without them?

Quoted 10 lines

This OSD-based file system is dependent on the open-osd initiator library code that I've submitted for inclusion for 2.6.29. It has been sitting in linux-next for a while now, and has not been receiving any comments for the last two updated patchsets I've sent to scsi-misc/lkml. However it has not yet been submitted into Jame's scsi-misc git tree, and James is the ultimate maintainer that should submit this work. I hope it will still be submitted into 2.6.29, as this code is totally self sufficient and does not endangers or changes any other Kernel subsystems. (All the needed ground work was already submitted to Linus since 2.6.26) So why should it not?

I don't like it mainly because it's not truly a useful general framework for others to build on. However, as argued above, there might not actually be such a useful framework, so as long as the only two consumers (you and Lustre) want an interface like this, I'll put it in.

James

_______________________________________________
osd-dev mailing list
[email protected]
http://mailman.open-osd.org/mailman/listinfo/osd-dev

Quoted 37 lines · James Bottomley

Quoted 14 lines · Boaz Harrosh

Quoted 13 lines · Andrew Morton

Quoted 10 lines · On Tue, 16 Dec 2008 17:33:48 +0200 Boaz Harrosh <[email protected]> wrote:

We need a mechanism to prepare the file system (mkfs). I chose to implement that by means of a couple of mount-options. Because there is no user-mode API for committing OSD commands. And also, all this stuff is highly internal to the file system itself.

- Added two mount options mkfs=0/1,format=capacity_in_meg, so mkfs/format can be executed by kernel code just before mount. An mkexofs utility can now be implemented by means of a script that mounts and unmount the file system with proper options.

Quoted 8 lines

Quoted 3 lines

Doing mkfs in-kernel is unusual. I don't think the above description sufficiently helps the uninitiated understand why mkfs cannot be done in userspace as usual. Please flesh it out a bit.

There are a few main reasons.
- There is no user-mode API for initiating OSD commands. Such a subsystem would be hundredfold bigger then the mkfs code submitted. I think it would be hard and stupid to maintain a complex user-mode API just for creating a couple of objects and writing a couple of on disk structures.

This is really a reflection of the whole problem with the OSD paradigm.

In theory, a filesystem on OSD is a thin layer of metadata mapping objects to files. Get this right and the storage will manage things, like security and access and attributes (there's even a natural mapping to the VFS concept of extended attributes). Plus, the storage has enough information to manage persistence, backups and replication.

The real problem is that no-one has actually managed to come up with a useful VFS<->OSD mapping layer (even by extending or altering the VFS). Every filesystem that currently uses OSD has a separate direct OSD speaking interface (i.e. it slices out the block layer to do this and talks directly to the storage).

Quoted 4 lines

I suppose this could be taken to show that such a layer is impossibly complex, as you assert, but its lack is reflected in strange looking design decisions like in-kernel mkfs. It would also mean that there would be very little layered code sharing between ODS based filesystems.

I think that we may need to gain some more experience to extract the commonalities of such file systems. Currently we came up with the lowest possible denominator the osd initiator library that deals with command formatting and execution, including attrs, sense status, and security.

To provide a higher level abstraction that would help with "administrative" tasks like mkfs and the like we already tossed an idea in the past - a file system that will represent the contents of an OSD in a namespace, for example: partition_id / object_id / {data, attrs / ..., ctl / ...}. Such a file system could provide a generic mapping which one could use to easily develop management applications for the OSD. That said, it's out of the scope of exofs which focuses mostly on the filesystem data and metadata paths.

Quoted 2 lines

Quoted 1 line

Boaz Harrosh <[email protected]> wrote:

When, if, all is fixed, through which tree/maintainer can exofs be submitted?

I can merge them. Or you can run a git tree of your own, add it to linux-next and ask Linus to pull it at the appropriate time.

Quoted 42 lines · Benny Halevy

Quoted 37 lines · James Bottomley

Quoted 14 lines · Boaz Harrosh

Quoted 13 lines · Andrew Morton

Quoted 10 lines · On Tue, 16 Dec 2008 17:33:48 +0200 Boaz Harrosh <[email protected]> wrote:

We need a mechanism to prepare the file system (mkfs). I chose to implement that by means of a couple of mount-options. Because there is no user-mode API for committing OSD commands. And also, all this stuff is highly internal to the file system itself.

- Added two mount options mkfs=0/1,format=capacity_in_meg, so mkfs/format can be executed by kernel code just before mount. An mkexofs utility can now be implemented by means of a script that mounts and unmount the file system with proper options.

Quoted 8 lines

Quoted 3 lines

Doing mkfs in-kernel is unusual. I don't think the above description sufficiently helps the uninitiated understand why mkfs cannot be done in userspace as usual. Please flesh it out a bit.

There are a few main reasons.
- There is no user-mode API for initiating OSD commands. Such a subsystem would be hundredfold bigger then the mkfs code submitted. I think it would be hard and stupid to maintain a complex user-mode API just for creating a couple of objects and writing a couple of on disk structures.

This is really a reflection of the whole problem with the OSD paradigm.

In theory, a filesystem on OSD is a thin layer of metadata mapping objects to files. Get this right and the storage will manage things, like security and access and attributes (there's even a natural mapping to the VFS concept of extended attributes). Plus, the storage has enough information to manage persistence, backups and replication.

The real problem is that no-one has actually managed to come up with a useful VFS<->OSD mapping layer (even by extending or altering the VFS). Every filesystem that currently uses OSD has a separate direct OSD speaking interface (i.e. it slices out the block layer to do this and talks directly to the storage).

Quoted 4 lines

I suppose this could be taken to show that such a layer is impossibly complex, as you assert, but its lack is reflected in strange looking design decisions like in-kernel mkfs. It would also mean that there would be very little layered code sharing between ODS based filesystems.

Quoted 5 lines

I think that we may need to gain some more experience to extract the commonalities of such file systems. Currently we came up with the lowest possible denominator the osd initiator library that deals with command formatting and execution, including attrs, sense status, and security.

Not putting words in James' mouth, but I definitely agree that the in-kernel mkfs raises a red flag or two. mkfs.ext3 for block-based filesystems has direct and intimate knowledge of ext3 filesystem structure, and it writes that information from userland directly to the block(s) necessary.

Similarly, mkfs for an object-based filesystem should be issuing SCSI commands to the OSD device from userland, AFAICS.

Quoted 10 lines

To provide a higher level abstraction that would help with "administrative" tasks like mkfs and the like we already tossed an idea in the past - a file system that will represent the contents of an OSD in a namespace, for example: partition_id / object_id / {data, attrs / ..., ctl / ...}. Such a file system could provide a generic mapping which one could use to easily develop management applications for the OSD. That said, it's out of the scope of exofs which focuses mostly on the filesystem data and metadata paths.

That's far too complex for what is necessary. Just issue SCSI commands from userland. We don't need an abstract interface specifically for low-level details. The VFS is that abstract interface; anything else should be low-level and purpose-built.

Jeff

Personally, I'm not sure if maintaining that intimate knowledge in a user space program is an ideal model with respect to keeping both in sync, avoiding code duplication, and dealing with upgrade issues (e.g. upgrading the kernel and not the user space utils)

Quoted 13 lines · On Wed, 31 Dec 2008 17:33:41 +0200 Boaz Harrosh <[email protected]> wrote:

Quoted 12 lines · Andrew Morton

fs/exofs/osd.c line 111

Quoted 7 lines

fs/exofs/osd.c new file +7 -0

+int prepare_get_attr_list_add_entry(struct osd_request *req,
+				    uint32_t page_num,
+				    uint32_t attr_num,
+				    uint32_t attr_len)
+{
+	struct osd_attr attr = {
+		.page = page_num,

Kernel developers expect a field called "page" to have type `struct page *'. osd_attr.page is thus designed to confuse.

Quoted 2 lines

...

Quoted 7 lines

Rant below (can be ignored):
This single fix will cause a massive change to the open-osd initiator patchset, (18 patches), and resubmission .I made the mistake because this name originates from a file that all naming conventions are taken from the OSD standard text. However this is no excuse for using a well known Kernel construct name. I will fix it. And will be more careful in the future.

The world wouldn't end if you left the code as-is. We've done worse things :)

Personally, I'm not sure if maintaining that intimate knowledge in a user space program is an ideal model with respect to keeping both in sync, avoiding code duplication, and dealing with upgrade issues (e.g. upgrading the kernel and not the user space utils)

Quoted 4 lines · James Bottomley

I don't like it mainly because it's not truly a useful general framework for others to build on. However, as argued above, there might not actually be such a useful framework, so as long as the only two consumers (you and Lustre) want an interface like this, I'll put it in.

Not to mention pnfs over objects which is coming up around the corner. The pnfs-obj layout driver will use the osd initiator library as well for distributed data I/O access (while the metadata server, to be based on exofs accesses the OSD for metadata and security ops too)

Quoted 5 lines · Benny Halevy

Quoted 4 lines · James Bottomley

I don't like it mainly because it's not truly a useful general framework for others to build on. However, as argued above, there might not actually be such a useful framework, so as long as the only two consumers (you and Lustre) want an interface like this, I'll put it in.

Quoted 4 lines

Not to mention pnfs over objects which is coming up around the corner. The pnfs-obj layout driver will use the osd initiator library as well for distributed data I/O access (while the metadata server, to be based on exofs accesses the OSD for metadata and security ops too)

What state is that project in right now?

--b.

Quoted 16 lines

In this patch are all the osd infrastructure that will be used later by the file system.

Also the declarations of constants, on disk structures, and prototypes.

And the Kbuild+Kconfig files needed to build the exofs module.

...

fs/exofs/exofs.h new file +6 -0

+struct exofs_sb_info {
+	struct osd_dev	*s_dev;			/* returned by get_osd_dev    */
+	uint64_t	s_pid;			/* partition ID of file system*/
+	int		s_timeout;		/* timeout for OSD operations */
+	uint32_t	s_nextid;		/* highest object ID used     */
+	uint32_t	s_numfiles;		/* number of files on fs      */

fs/exofs/exofs.h line 72

Quoted 10 lines

fs/exofs/exofs.h new file +10 -0

+	spinlock_t	s_next_gen_lock;	/* spinlock for gen # update  */
+	u32		s_next_generation;	/* next gen # to use          */
+	atomic_t	s_curr_pending;		/* number of pending commands */
+	uint8_t		s_cred[OSD_CAP_LEN];	/* all-powerful credential    */
+};
+
+/*
+ * our inode flags
+ */
+#ifdef ARCH_HAS_ATOMIC_UNSIGNED

This doesn't exist, and it would be fairly bad to introduce it. Please kill the ifdefs.

fs/exofs/exofs.h line 76

Quoted 4 lines

fs/exofs/exofs.h new file +4 -0

+typedef unsigned exofs_iflags_t;
+#else
+typedef unsigned long exofs_iflags_t;
+#endif

Then please kill the typedef altogether and replace it with `unsigned long' everywhere

Quoted 6 lines · Andrew Morton

Quoted 2 lines

Quoted 1 line

Boaz Harrosh <[email protected]> wrote:

When, if, all is fixed, through which tree/maintainer can exofs be submitted?

I can merge them. Or you can run a git tree of your own, add it to linux-next and ask Linus to pull it at the appropriate time.

Hi James

Andrew suggested that maybe I should push exofs file system directly to Linus as it is pretty orthogonal to any other work. Sitting in linux-next will quickly expose any advancements in VFS and will force me to keep the tree uptodate.

If that is so, and is accepted by Linus, would you rather that also the open-osd initiator library will be submitted through the same tree? The conflicts with scsi are very very narrow. The only real dependency is the ULD being a SCSI ULD. I will routinely ask your ACK on any scsi or ULD related patches. Which are very few. This way it will be easier to manage the dependencies between the OSD work, the OSD pNFS-Objects trees at pNFS project, and the pNFSD+EXOFS export. One less dependency.

[I already have such a public tree at git.open-osd.org for a while now]

Hi!

Quoted 29 lines

Quoted 16 lines

In this patch are all the osd infrastructure that will be used later by the file system.

Also the declarations of constants, on disk structures, and prototypes.

And the Kbuild+Kconfig files needed to build the exofs module.

...

fs/exofs/exofs.h new file +6 -0

+struct exofs_sb_info {
+	struct osd_dev	*s_dev;			/* returned by get_osd_dev    */
+	uint64_t	s_pid;			/* partition ID of file system*/
+	int		s_timeout;		/* timeout for OSD operations */
+	uint32_t	s_nextid;		/* highest object ID used     */
+	uint32_t	s_numfiles;		/* number of files on fs      */

fs/exofs/exofs.h line 72

Quoted 10 lines

fs/exofs/exofs.h new file +10 -0

+	spinlock_t	s_next_gen_lock;	/* spinlock for gen # update  */
+	u32		s_next_generation;	/* next gen # to use          */
+	atomic_t	s_curr_pending;		/* number of pending commands */
+	uint8_t		s_cred[OSD_CAP_LEN];	/* all-powerful credential    */
+};
+
+/*
+ * our inode flags
+ */
+#ifdef ARCH_HAS_ATOMIC_UNSIGNED

This doesn't exist, and it would be fairly bad to introduce it. Please kill the ifdefs.

Quoted 6 lines

fs/exofs/exofs.h line 76

Quoted 4 lines

fs/exofs/exofs.h new file +4 -0

+typedef unsigned exofs_iflags_t;
+#else
+typedef unsigned long exofs_iflags_t;
+#endif

Then please kill the typedef altogether and replace it with `unsigned long' everywhere

Hmmm.. .and at a note somewhere that we assume unsigned long to be atomic...?

Quoted 21 lines · Boaz Harrosh

Quoted 5 lines · Andrew Morton

Quoted 2 lines

Quoted 1 line

Boaz Harrosh <[email protected]> wrote:

When, if, all is fixed, through which tree/maintainer can exofs be submitted?

I can merge them. Or you can run a git tree of your own, add it to linux-next and ask Linus to pull it at the appropriate time.

Hi James

Andrew suggested that maybe I should push exofs file system directly to Linus as it is pretty orthogonal to any other work. Sitting in linux-next will quickly expose any advancements in VFS and will force me to keep the tree uptodate.

If that is so, and is accepted by Linus, would you rather that also the open-osd initiator library will be submitted through the same tree? The conflicts with scsi are very very narrow. The only real dependency is the ULD being a SCSI ULD. I will routinely ask your ACK on any scsi or ULD related patches. Which are very few. This way it will be easier to manage the dependencies between the OSD work, the OSD pNFS-Objects trees at pNFS project, and the pNFSD+EXOFS export. One less dependency.

Quoted 1 line

[I already have such a public tree at git.open-osd.org for a while now]

Since it's sitting in SCSI, at least the libosd piece belongs over the SCSI mailing list, so I think it makes sense to continue updating it via the SCSI tree.

What's the status of the major number request from LANANA. That's patch number one, and I haven't heard that they've confirmed the selection of 260 yet; or is LANANA now dead and it's who gets the major into the tree first?

James

Quoted 17 lines · Boaz Harrosh

Quoted 16 lines · Andrew Morton

Quoted 10 lines · On Tue, 16 Dec 2008 17:33:48 +0200 Boaz Harrosh <[email protected]> wrote:

We need a mechanism to prepare the file system (mkfs). I chose to implement that by means of a couple of mount-options. Because there is no user-mode API for committing OSD commands. And also, all this stuff is highly internal to the file system itself.

- Added two mount options mkfs=0/1,format=capacity_in_meg, so mkfs/format can be executed by kernel code just before mount. An mkexofs utility can now be implemented by means of a script that mounts and unmount the file system with proper options.

Doing mkfs in-kernel is unusual. I don't think the above description sufficiently helps the uninitiated understand why mkfs cannot be done in userspace as usual. Please flesh it out a bit.

Quoted 5 lines

There are a few main reasons.
- There is no user-mode API for initiating OSD commands. Such a subsystem would be hundredfold bigger then the mkfs code submitted. I think it would be hard and stupid to maintain a complex user-mode API just for creating a couple of objects and writing a couple of on disk structures.

This is really a reflection of the whole problem with the OSD paradigm.

In theory, a filesystem on OSD is a thin layer of metadata mapping objects to files. Get this right and the storage will manage things,

like security and access and attributes (there's even a natural mapping to the VFS concept of extended attributes). Plus, the storage has enough information to manage persistence, backups and replication.

The real problem is that no-one has actually managed to come up with a useful VFS<->OSD mapping layer (even by extending or altering the VFS). Every filesystem that currently uses OSD has a separate direct OSD speaking interface (i.e. it slices out the block layer to do this and talks directly to the storage).

I suppose this could be taken to show that such a layer is impossibly complex, as you assert, but its lack is reflected in strange looking design decisions like in-kernel mkfs. It would also mean that there would be very little layered code sharing between ODS based filesystems.

Quoted 14 lines

- I intend to refactor the code further to make use of more super.c services, so to make this addition even smaller. Also future direction of raid over multiple objects will make even more kernel infrastructure needed which will need even more user-mode code duplication.
- I anticipate problems that are not yet addressed in this body of work but will be in the future, mainly that a single OSD-target (lun) can be shared by lots of FSs, and a single FS can span many OSD-targets. Some central management is much easier to do in Kernel.

Quoted 4 lines

What are the dependencies for this filesystem code? I assume that it depends on various block- and scsi-level patches? Which ones, and what is their status, and is this code even compileable without them?

Quoted 10 lines

This OSD-based file system is dependent on the open-osd initiator library code that I've submitted for inclusion for 2.6.29. It has been sitting in linux-next for a while now, and has not been receiving any comments for the last two updated patchsets I've sent to scsi-misc/lkml. However it has not yet been submitted into Jame's scsi-misc git tree, and James is the ultimate maintainer that should submit this work. I hope it will still be submitted into 2.6.29, as this code is totally self sufficient and does not endangers or changes any other Kernel subsystems. (All the needed ground work was already submitted to Linus since 2.6.26) So why should it not?

I don't like it mainly because it's not truly a useful general framework for others to build on. However, as argued above, there might not actually be such a useful framework, so as long as the only two consumers (you and Lustre) want an interface like this, I'll put it in.

James

User-mode Interface is another matter. There are some ideas and some already implemented.
[Hosted on open-osd.org
see: http://git.open-osd.org/gitweb.cgi?p=osc-osd/.git;a=summary
look inside the osd-initiator directory]
And I have a toy interface that adds no new entries into the Kernel in the form of an OSDVFS module, that will let you access the raw OSD device through the VFS name-space.

The lack of any user-mode API is just the lack of any current need/priority, or that I'm the only one working on OSD. But nothing that could not be solved in two weeks of pragmatic work. Surly it's not a paradigm problem.

Quoted 38 lines · Pavel Machek

Hi!

Quoted 35 lines

Quoted 16 lines

In this patch are all the osd infrastructure that will be used later by the file system.

Also the declarations of constants, on disk structures, and prototypes.

And the Kbuild+Kconfig files needed to build the exofs module.

...

fs/exofs/exofs.h new file +6 -0

+struct exofs_sb_info {
+	struct osd_dev	*s_dev;			/* returned by get_osd_dev    */
+	uint64_t	s_pid;			/* partition ID of file system*/
+	int		s_timeout;		/* timeout for OSD operations */
+	uint32_t	s_nextid;		/* highest object ID used     */
+	uint32_t	s_numfiles;		/* number of files on fs      */

fs/exofs/exofs.h line 72

Quoted 10 lines

fs/exofs/exofs.h new file +10 -0

+	spinlock_t	s_next_gen_lock;	/* spinlock for gen # update  */
+	u32		s_next_generation;	/* next gen # to use          */
+	atomic_t	s_curr_pending;		/* number of pending commands */
+	uint8_t		s_cred[OSD_CAP_LEN];	/* all-powerful credential    */
+};
+
+/*
+ * our inode flags
+ */
+#ifdef ARCH_HAS_ATOMIC_UNSIGNED

This doesn't exist, and it would be fairly bad to introduce it. Please kill the ifdefs.

fs/exofs/exofs.h line 76

Quoted 4 lines

fs/exofs/exofs.h new file +4 -0

+typedef unsigned exofs_iflags_t;
+#else
+typedef unsigned long exofs_iflags_t;
+#endif

Then please kill the typedef altogether and replace it with `unsigned long' everywhere

Quoted 2 lines

Hmmm.. .and at a note somewhere that we assume unsigned long to be atomic...?

I think I'll just use unsigned. It's more then enough I'm not using more then 3 bits for now. Is unsigned workable for all ARCHs?

Quoted 13 lines · Boaz Harrosh

Quoted 12 lines · Pavel Machek

Hi!

Quoted 10 lines

Quoted 1 line

+#ifdef ARCH_HAS_ATOMIC_UNSIGNED

This doesn't exist, and it would be fairly bad to introduce it. Please kill the ifdefs.

fs/exofs/exofs.h line 76

Quoted 4 lines

fs/exofs/exofs.h new file +4 -0

+typedef unsigned exofs_iflags_t;
+#else
+typedef unsigned long exofs_iflags_t;
+#endif

Then please kill the typedef altogether and replace it with `unsigned long' everywhere

Quoted 4 lines

Quoted 2 lines

Hmmm.. .and at a note somewhere that we assume unsigned long to be atomic...?

I think I'll just use unsigned. It's more then enough I'm not using more then 3 bits for now. Is unsigned workable for all ARCHs?

/*
* our extension to the in-memory inode
*/
struct exofs_i_info {
unsigned long i_flags; /* various atomic flags */

/*
* our inode flags
*/
#define OBJ_2BCREATED 0 /* object will be created soon*/
#define OBJ_CREATED 1 /* object has been created on the osd*/

static inline int obj_2bcreated(struct exofs_i_info *oi)

{
	return test_bit(OBJ_2BCREATED, &(oi->i_flags));
}

static inline void set_obj_2bcreated(struct exofs_i_info *oi)

{
	set_bit(OBJ_2BCREATED, &(oi->i_flags));
}

static inline int obj_created(struct exofs_i_info *oi)

{
	return test_bit(OBJ_CREATED, &(oi->i_flags));
}

static inline void set_obj_created(struct exofs_i_info *oi)

{
	set_bit(OBJ_CREATED, &(oi->i_flags));
}

Please just use atomic_t.

(see "atomics: document that linux expects certain atomic behaviour" thread for discussion)
Pavel

Quoted 14 lines · Pavel Machek

Quoted 13 lines · Boaz Harrosh

Quoted 12 lines · Pavel Machek

Hi!

Quoted 10 lines

Quoted 1 line

+#ifdef ARCH_HAS_ATOMIC_UNSIGNED

This doesn't exist, and it would be fairly bad to introduce it. Please kill the ifdefs.

fs/exofs/exofs.h line 76

Quoted 4 lines

fs/exofs/exofs.h new file +4 -0

+typedef unsigned exofs_iflags_t;
+#else
+typedef unsigned long exofs_iflags_t;
+#endif

Then please kill the typedef altogether and replace it with `unsigned long' everywhere

Quoted 4 lines

Quoted 4 lines

Quoted 2 lines

Hmmm.. .and at a note somewhere that we assume unsigned long to be atomic...?

I think I'll just use unsigned. It's more then enough I'm not using more then 3 bits for now. Is unsigned workable for all ARCHs?

<added>

Quoted 5 lines

/*
* our extension to the in-memory inode
*/
struct exofs_i_info {
unsigned long i_flags; /* various atomic flags */

<snip>

Quoted 18 lines

/*
* our inode flags
*/
#define OBJ_2BCREATED 0 /* object will be created soon*/
#define OBJ_CREATED 1 /* object has been created on the osd*/

static inline int obj_2bcreated(struct exofs_i_info *oi)

{
	return test_bit(OBJ_2BCREATED, &(oi->i_flags));
}

static inline void set_obj_2bcreated(struct exofs_i_info *oi)

{
	set_bit(OBJ_2BCREATED, &(oi->i_flags));
}

static inline int obj_created(struct exofs_i_info *oi)

Quoted 8 lines

{
	return test_bit(OBJ_CREATED, &(oi->i_flags));
}

static inline void set_obj_created(struct exofs_i_info *oi)

{
	set_bit(OBJ_CREATED, &(oi->i_flags));
}

</added>

Quoted 6 lines

Please just use atomic_t.

(see "atomics: document that linux expects certain atomic behaviour" thread for discussion)
Pavel

I have a problem with this. The context of i_flags is to be used with set_bit() and test_bit(). In some ARCHs like x86_64 they take an "unsigned long *" in most others they take a "void *" and cast internally to a "u32 *". (for x86_64 I must use "unsigned long", anything else warns)

I think if I declare "unsigned long" but only use 32 bits flags then I should be in the clear with ALL archs, I'll see if that works once this code sits in linux-next. (That's real ugly I think)

Is set_bit() and test_bit() should only be used from arch/ code? What can regular kernel code use?

I don't like it mainly because it's not truly a useful general framework for others to build on. However, as argued above, there might not actually be such a useful framework, so as long as the only two consumers (you and Lustre) want an interface like this, I'll put it in.

In this patch are all the osd infrastructure that will be used later by the file system.

Also the declarations of constants, on disk structures, and prototypes.

And the Kbuild+Kconfig files needed to build the exofs module.

Signed-off-by Boaz Harrosh <[email protected]>

---
 fs/exofs/Kbuild   |   30 +++++
 fs/exofs/Kconfig  |   13 ++
 fs/exofs/common.h |  154 ++++++++++++++++++++++++
 fs/exofs/exofs.h  |  183 +++++++++++++++++++++++++++++
 fs/exofs/osd.c    |  334 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 714 insertions(+), 0 deletions(-)
 create mode 100644 fs/exofs/Kbuild
 create mode 100644 fs/exofs/Kconfig
 create mode 100644 fs/exofs/common.h
 create mode 100644 fs/exofs/exofs.h
 create mode 100644 fs/exofs/osd.c

fs/exofs/Kbuild new file +30 -0

new file mode 100644
index 0000000..fd3351e
@@ -0,0 +1,30 @@
+#
+# Kbuild for the EXOFS module
+#
+# Copyright (C) 2008 Panasas Inc.  All rights reserved.
+#
+# Authors:
+#   Boaz Harrosh <[email protected]>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2
+#
+# Kbuild - Gets included from the Kernels Makefile and build system
+#
+
+ifneq ($(OSD_INC),)
+# we are built out-of-tree Kconfigure everything as on
+
+CONFIG_EXOFS_FS=m
+ccflags += -DCONFIG_EXOFS_FS -DCONFIG_EXOFS_FS_MODULE
+# ccflags += -DCONFIG_EXOFS_DEBUG
+
+# if we are built out-of-tree and the hosting kernel has OSD headers
+# then "ccflags-y +=" will not pick the out-off-tree headers. Only by doing
+# this it will work. This might break in future kernels
+KBUILD_CPPFLAGS := -I$(OSD_INC) $(KBUILD_CPPFLAGS)
+
+endif
+
+exofs-objs := osd.o
+obj-$(CONFIG_EXOFS_FS) += exofs.o

fs/exofs/Kconfig new file +13 -0

new file mode 100644
index 0000000..86194b2
@@ -0,0 +1,13 @@
+config EXOFS_FS
+	tristate "exofs: OSD based file system support"
+	depends on SCSI_OSD_ULD
+	help
+	  EXOFS is a file system that uses an OSD storage device,
+	  as its backing storage.
+
+# Debugging-related stuff
+config EXOFS_DEBUG
+	bool "Enable debugging"
+	depends on EXOFS_FS
+	help
+	  This option enables EXOFS debug prints.

fs/exofs/common.h new file +108 -0

new file mode 100644
index 0000000..9a165b3
@@ -0,0 +1,154 @@
+/*
+ * Copyright (C) 2005, 2006
+ * Avishay Traeger ([email protected]) ([email protected])
+ * Copyright (C) 2005, 2006
+ * International Business Machines
+ *
+ * Copyrights for code taken from ext2:
+ *     Copyright (C) 1992, 1993, 1994, 1995
+ *     Remy Card ([email protected])
+ *     Laboratoire MASI - Institut Blaise Pascal
+ *     Universite Pierre et Marie Curie (Paris VI)
+ *     from
+ *     linux/fs/minix/inode.c
+ *     Copyright (C) 1991, 1992  Linus Torvalds
+ *
+ * This file is part of exofs.
+ *
+ * exofs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.  Since it is based on ext2, and the only
+ * valid version of GPL for the Linux kernel is version 2, the only valid
+ * version of GPL for exofs is version 2.
+ *
+ * exofs is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with exofs; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#ifndef __EXOFS_COM_H__
+#define __EXOFS_COM_H__
+
+#include <linux/types.h>
+#include <linux/timex.h>
+
+#include <scsi/osd_attributes.h>
+#include <scsi/osd_initiator.h>
+#include <scsi/osd_sec.h>
+
+/****************************************************************************
+ * Object ID related defines
+ * NOTE: inode# = object ID - EXOFS_OBJ_OFF
+ ****************************************************************************/
+#define EXOFS_OBJ_OFF	0x10000	/* offset for objects */
+#define EXOFS_SUPER_ID	0x10000	/* object ID for on-disk superblock */
+#define EXOFS_BM_ID	0x10001	/* object ID for ID bitmap */
+#define EXOFS_ROOT_ID	0x10002	/* object ID for root directory */
+#define EXOFS_TEST_ID	0x10003	/* object ID for test object */
+
+/* exofs Application specific page/attribute */
+#ifndef OSD_PAGE_NUM_IBM_UOBJ_FS_DATA
+# define OSD_PAGE_NUM_IBM_UOBJ_FS_DATA	   (OSD_APAGE_APP_DEFINED_FIRST + 3)
+# define OSD_ATTR_NUM_IBM_UOBJ_FS_DATA_INODE 1
+#endif
+
+/*
+ * The maximum number of files we can have is limited by the size of the
+ * inode number.  This is the largest object ID that the file system supports.
+ * Object IDs 0, 1, and 2 are always in use (see above defines).
+ */
+enum {
+	EXOFS_UINT64_MAX = (~0LL),
+	EXOFS_MAX_INO_ID = (sizeof(ino_t) * 8 == 64) ? EXOFS_UINT64_MAX :
+					(1LL << (sizeof(ino_t) * 8 - 1)),
+	EXOFS_MAX_ID	 = (EXOFS_MAX_INO_ID - 1 - EXOFS_OBJ_OFF),
+};
+
+/****************************************************************************
+ * Misc.
+ ****************************************************************************/
+#define EXOFS_BLKSHIFT	12
+#define EXOFS_BLKSIZE	(1UL << EXOFS_BLKSHIFT)
+
+/****************************************************************************
+ * superblock-related things
+ ****************************************************************************/
+#define EXOFS_SUPER_MAGIC	0x5DF5
+
+/*
+ * The file system control block - stored in an object's data (mainly, the one
+ * with ID EXOFS_SUPER_ID).  This is where the in-memory superblock is stored
+ * on disk.  Right now it just has a magic value, which is basically a sanity
+ * check on our ability to communicate with the object store.
+ */
+struct exofs_fscb {
+	uint32_t  s_nextid;	/* Highest object ID used */
+	uint32_t  s_numfiles;	/* Number of files on fs */
+	uint16_t  s_magic;	/* Magic signature */
+	uint16_t  s_newfs;	/* Non-zero if this is a new fs */
+};
+
+/****************************************************************************
+ * inode-related things
+ ****************************************************************************/
+#define EXOFS_IDATA		5
+
+/*
+ * The file control block - stored in an object's attributes.  This is where
+ * the in-memory inode is stored on disk.
+ */
+struct exofs_fcb {
+	uint64_t  i_size;		/* Size of the file */
+	uint16_t  i_mode;         	/* File mode */
+	uint16_t  i_links_count;  	/* Links count */

fs/exofs/common.h new file +36 -0

+
+#define EXOFS_INO_ATTR_SIZE	sizeof(struct exofs_fcb)
+
+/****************************************************************************
+ * dentry-related things
+ ****************************************************************************/
+#define EXOFS_NAME_LEN	255
+
+/*
+ * The on-disk directory entry
+ */
+struct exofs_dir_entry {
+	uint32_t	inode;			/* inode number           */
+	uint16_t	rec_len;		/* directory entry length */
+	uint8_t		name_len;		/* name length            */
+	uint8_t		file_type;		/* umm...file type        */
+	char		name[EXOFS_NAME_LEN];	/* file name              */
+};
+
+enum {
+	EXOFS_FT_UNKNOWN,
+	EXOFS_FT_REG_FILE,
+	EXOFS_FT_DIR,
+	EXOFS_FT_CHRDEV,
+	EXOFS_FT_BLKDEV,
+	EXOFS_FT_FIFO,
+	EXOFS_FT_SOCK,
+	EXOFS_FT_SYMLINK,
+	EXOFS_FT_MAX
+};
+
+#define EXOFS_DIR_PAD			4
+#define EXOFS_DIR_ROUND			(EXOFS_DIR_PAD - 1)
+#define EXOFS_DIR_REC_LEN(name_len)	(((name_len) + 8 + EXOFS_DIR_ROUND) & \
+					 ~EXOFS_DIR_ROUND)
+#endif

fs/exofs/exofs.h new file +183 -0

new file mode 100644
index 0000000..8534450
@@ -0,0 +1,183 @@
+/*
+ * Copyright (C) 2005, 2006
+ * Avishay Traeger ([email protected]) ([email protected])
+ * Copyright (C) 2005, 2006
+ * International Business Machines
+ *
+ * Copyrights for code taken from ext2:
+ *     Copyright (C) 1992, 1993, 1994, 1995
+ *     Remy Card ([email protected])
+ *     Laboratoire MASI - Institut Blaise Pascal
+ *     Universite Pierre et Marie Curie (Paris VI)
+ *     from
+ *     linux/fs/minix/inode.c
+ *     Copyright (C) 1991, 1992  Linus Torvalds
+ *
+ * This file is part of exofs.
+ *
+ * exofs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.  Since it is based on ext2, and the only
+ * valid version of GPL for the Linux kernel is version 2, the only valid
+ * version of GPL for exofs is version 2.
+ *
+ * exofs is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with exofs; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#include <linux/fs.h>
+#include <linux/time.h>
+#include "common.h"
+
+#ifndef __EXOFS_H__
+#define __EXOFS_H__
+
+#define EXOFS_ERR(fmt, a...) printk(KERN_ERR "exofs: " fmt, ##a)
+
+#ifdef CONFIG_EXOFS_DEBUG
+#define EXOFS_DBGMSG(fmt, a...) \
+	printk(KERN_NOTICE "exofs @%s:%d: " fmt, __func__, __LINE__, ##a)
+#else
+#define EXOFS_DBGMSG(fmt, a...) \
+	do {} while (0)
+#endif
+
+/* u64 has problems with printk this will cast it to unsigned long long */
+#define _LLU(x) (unsigned long long)(x)
+
+/*
+ * our extension to the in-memory superblock
+ */
+struct exofs_sb_info {
+	struct osd_dev	*s_dev;			/* returned by get_osd_dev    */
+	uint64_t	s_pid;			/* partition ID of file system*/
+	int		s_timeout;		/* timeout for OSD operations */
+	uint32_t	s_nextid;		/* highest object ID used     */
+	uint32_t	s_numfiles;		/* number of files on fs      */
+	spinlock_t	s_next_gen_lock;	/* spinlock for gen # update  */
+	u32		s_next_generation;	/* next gen # to use          */
+	atomic_t	s_curr_pending;		/* number of pending commands */
+	uint8_t		s_cred[OSD_CAP_LEN];	/* all-powerful credential    */
+};
+
+/*
+ * our inode flags
+ */
+#ifdef ARCH_HAS_ATOMIC_UNSIGNED
+typedef unsigned exofs_iflags_t;
+#else
+typedef unsigned long exofs_iflags_t;
+#endif
+
+#define OBJ_2BCREATED	0	/* object will be created soon*/
+#define OBJ_CREATED	1	/* object has been created on the osd*/
+
+#define Obj2BCreated(oi) \
+	test_bit(OBJ_2BCREATED, &(oi->i_flags))
+#define SetObj2BCreated(oi) \
+	set_bit(OBJ_2BCREATED, &(oi->i_flags))
+
+#define ObjCreated(oi) \
+	test_bit(OBJ_CREATED, &(oi->i_flags))
+#define SetObjCreated(oi) \
+	set_bit(OBJ_CREATED, &(oi->i_flags))
+
+/*
+ * our extension to the in-memory inode
+ */
+struct exofs_i_info {
+	exofs_iflags_t i_flags;            /* various atomic flags            */
+	__le32	       i_data[EXOFS_IDATA];/*short symlink names and device #s*/
+	uint32_t       i_dir_start_lookup; /* which page to start lookup      */
+	wait_queue_head_t i_wq;            /* wait queue for inode            */
+	uint64_t       i_commit_size;      /* the object's written length     */
+	uint8_t        i_cred[OSD_CAP_LEN];/* all-powerful credential         */
+	struct inode   vfs_inode;          /* normal in-memory inode          */
+};
+
+/*
+ * get to our inode from the vfs inode
+ */
+static inline struct exofs_i_info *EXOFS_I(struct inode *inode)
+{
+	return container_of(inode, struct exofs_i_info, vfs_inode);
+}
+
+/*************************
+ * function declarations *
+ *************************/
+/* osd.c                 */
+void make_credential(uint8_t[], uint64_t, uint64_t);
+int check_ok(struct osd_request *);
+int exofs_sync_op(struct osd_request *, int, uint8_t *);
+int exofs_async_op(struct osd_request *, osd_req_done_fn *, void *, char *);
+
+int prepare_get_attr_list_add_entry(struct osd_request *req,
+				    uint32_t page_num,
+				    uint32_t attr_num,
+				    uint32_t attr_len);
+int prepare_set_attr_list_add_entry(struct osd_request *req,
+				    uint32_t page_num,
+				    uint32_t attr_num,
+				    uint16_t attr_len,
+				    const unsigned char *attr_val);
+int extract_next_attr_from_req(struct osd_request *req,
+			       uint32_t *page_num, uint32_t *attr_num,
+			       uint16_t *attr_len, uint8_t **attr_val);
+struct osd_request *prepare_osd_format_lun(struct osd_dev *dev,
+					   uint64_t formatted_capacity);
+struct osd_request *prepare_osd_create_partition(struct osd_dev *dev,
+						 uint64_t requested_id);
+struct osd_request *prepare_osd_remove_partition(struct osd_dev *dev,
+						 uint64_t requested_id);
+struct osd_request *prepare_osd_create(struct osd_dev *dev,
+				       uint64_t part_id,
+				       uint64_t requested_id);
+struct osd_request *prepare_osd_remove(struct osd_dev *dev,
+				       uint64_t part_id,
+				       uint64_t obj_id);
+struct osd_request *prepare_osd_set_attr(struct osd_dev *dev,
+					 uint64_t part_id,
+					 uint64_t obj_id);
+struct osd_request *prepare_osd_get_attr(struct osd_dev *dev,
+					 uint64_t part_id,
+					 uint64_t obj_id);
+struct osd_request *prepare_osd_read(struct osd_dev *dev,
+				     uint64_t part_id,
+				     uint64_t obj_id,
+				     uint64_t length,
+				     uint64_t offset,
+				     int cmd_data_use_sg,
+				     unsigned char *cmd_data);
+struct osd_request *prepare_osd_write(struct osd_dev *dev,
+				      uint64_t part_id,
+				      uint64_t obj_id,
+				      uint64_t length,
+				      uint64_t offset,
+				      int cmd_data_use_sg,
+				      const unsigned char *cmd_data);
+struct osd_request *prepare_osd_list(struct osd_dev *dev,
+				     uint64_t part_id,
+				     uint32_t list_id,
+				     uint64_t alloc_len,
+				     uint64_t initial_obj_id,
+				     int use_sg,
+				     void *data);
+int extract_list_from_req(struct osd_request *req,
+			  uint64_t *total_matches_p,
+			  uint64_t *num_ids_retrieved_p,
+			  uint64_t *list_of_ids_p[],
+			  int      *is_list_of_partitions_p,
+			  int      *list_isnt_up_to_date_p,
+			  uint64_t *continuation_tag_p,
+			  uint32_t *list_id_for_more_p);
+
+void free_osd_req(struct osd_request *req);
+
+#endif

fs/exofs/osd.c new file +334 -0

new file mode 100644
index 0000000..3859d3e
@@ -0,0 +1,334 @@
+/*
+ * Copyright (C) 2005, 2006
+ * Avishay Traeger ([email protected]) ([email protected])
+ * Copyright (C) 2005, 2006
+ * International Business Machines
+ *
+ * This file is part of exofs.
+ *
+ * exofs is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.  Since it is based on ext2, and the only
+ * valid version of GPL for the Linux kernel is version 2, the only valid
+ * version of GPL for exofs is version 2.
+ *
+ * exofs is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with exofs; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#include <scsi/scsi_device.h>
+#include <scsi/osd_sense.h>
+
+#include "exofs.h"
+
+int check_ok(struct osd_request *req)
+{
+	struct osd_sense_info osi;
+	int ret = osd_req_decode_sense(req, &osi);
+
+	if (ret) { /* translate to Linux codes */
+		if (osi.additional_code == scsi_invalid_field_in_cdb) {
+			if (osi.cdb_field_offset == OSD_CFO_STARTING_BYTE)
+				ret = -EFAULT;
+			if (osi.cdb_field_offset == OSD_CFO_OBJECT_ID)
+				ret = -ENOENT;
+			else
+				ret = -EINVAL;
+		} else if (osi.additional_code == osd_quota_error)
+			ret = -ENOSPC;
+		else
+			ret = -EIO;
+	}
+
+	return ret;
+}
+
+void make_credential(uint8_t cred_a[OSD_CAP_LEN], uint64_t pid, uint64_t oid)
+{
+	struct osd_obj_id obj = {
+		.partition = pid,
+		.id = oid
+	};
+
+	osd_sec_init_nosec_doall_caps(cred_a, &obj, false, true);
+}
+
+/*
+ * Perform a synchronous OSD operation.
+ */
+int exofs_sync_op(struct osd_request *req, int timeout, uint8_t *credential)
+{
+	int ret;
+
+	req->timeout = timeout;
+	ret = osd_finalize_request(req, 0, credential, NULL);
+	if (ret) {
+		EXOFS_DBGMSG("Faild to osd_finalize_request() => %d\n", ret);
+		return ret;
+	}
+
+	ret = osd_execute_request(req);
+
+	if (ret)
+		EXOFS_DBGMSG("osd_execute_request() => %d\n", ret);
+	/* osd_req_decode_sense(or, ret); */
+	return ret;
+}
+
+/*
+ * Perform an asynchronous OSD operation.
+ */
+int exofs_async_op(struct osd_request *req, osd_req_done_fn *async_done,
+		   void *caller_context, char *credential)
+{
+	int ret;
+
+	ret = osd_finalize_request(req, 0, credential, NULL);
+	if (ret) {
+		EXOFS_DBGMSG("Faild to osd_finalize_request() => %d\n", ret);
+		return ret;
+	}
+
+	ret = osd_execute_request_async(req, async_done, caller_context);
+
+	if (ret)
+		EXOFS_DBGMSG("osd_execute_request_async() => %d\n", ret);
+	return ret;
+}
+
+int prepare_get_attr_list_add_entry(struct osd_request *req,
+				    uint32_t page_num,
+				    uint32_t attr_num,
+				    uint32_t attr_len)
+{
+	struct osd_attr attr = {
+		.page = page_num,
+		.attr_id = attr_num,
+		.len = attr_len,
+	};
+
+	return osd_req_add_get_attr_list(req, &attr, 1);
+}
+
+int prepare_set_attr_list_add_entry(struct osd_request *req,
+				    uint32_t page_num,
+				    uint32_t attr_num,
+				    uint16_t attr_len,
+				    const unsigned char *attr_val)
+{
+	struct osd_attr attr = {
+		.page = page_num,
+		.attr_id = attr_num,
+		.len = attr_len,
+		.val_ptr = (u8 *)attr_val,
+	};
+
+	return osd_req_add_set_attr_list(req, &attr, 1);
+}
+
+int extract_next_attr_from_req(struct osd_request *req,
+	uint32_t *page_num, uint32_t *attr_num,
+	uint16_t *attr_len, uint8_t **attr_val)
+{
+	struct osd_attr attr = {.page = 0}; /* start with zeros */
+	void *iter = NULL;
+	int nelem;
+
+	do {
+		nelem = 1;
+		osd_req_decode_get_attr_list(req, &attr, &nelem, &iter);
+		if ((attr.page == *page_num) && (attr.attr_id == *attr_num)) {
+			*attr_len = attr.len;
+			*attr_val = attr.val_ptr;
+			return 0;
+		}
+	} while (iter);
+
+	return -EIO;
+}
+
+struct osd_request *prepare_osd_format_lun(struct osd_dev *dev,
+					   uint64_t formatted_capacity)
+{
+	struct osd_request *or = osd_start_request(dev, GFP_KERNEL);
+
+	if (!or)
+		return NULL;
+
+	osd_req_format(or, formatted_capacity);
+
+	return or;
+}
+
+struct osd_request *prepare_osd_create_partition(struct osd_dev *dev,
+						 uint64_t requested_id)
+{
+	struct osd_request *or = osd_start_request(dev, GFP_KERNEL);
+
+	if (!or)
+		return NULL;
+
+	osd_req_create_partition(or, requested_id);
+
+	return or;
+}
+
+struct osd_request *prepare_osd_remove_partition(struct osd_dev *dev,
+						 uint64_t requested_id)
+{
+	struct osd_request *or = osd_start_request(dev, GFP_KERNEL);
+
+	if (!or)
+		return NULL;
+
+	osd_req_remove_partition(or, requested_id);
+
+	return or;
+}
+
+struct osd_request *prepare_osd_create(struct osd_dev *dev,
+				     uint64_t part_id,
+				     uint64_t requested_id)
+{
+	struct osd_obj_id obj = {
+		.partition = part_id,
+		.id = requested_id
+	};
+	struct osd_request *or = osd_start_request(dev, GFP_KERNEL);
+
+	if (!or)
+		return NULL;
+
+	osd_req_create_object(or, &obj);
+
+	return or;
+}
+
+struct osd_request *prepare_osd_remove(struct osd_dev *dev,
+				     uint64_t part_id,
+				     uint64_t obj_id)
+{
+	struct osd_obj_id obj = {
+		.partition = part_id,
+		.id = obj_id
+	};
+	struct osd_request *or = osd_start_request(dev, GFP_KERNEL);
+
+	if (!or)
+		return NULL;
+
+	osd_req_remove_object(or, &obj);
+
+	return or;
+}
+
+struct osd_request *prepare_osd_set_attr(struct osd_dev *dev,
+				       uint64_t part_id,
+				       uint64_t obj_id)
+{
+	struct osd_obj_id obj = {
+		.partition = part_id,
+		.id = obj_id
+	};
+	struct osd_request *or = osd_start_request(dev, GFP_KERNEL);
+
+	if (!or)
+		return NULL;
+
+	osd_req_set_attributes(or, &obj);
+
+	return or;
+}
+
+struct osd_request *prepare_osd_get_attr(struct osd_dev *dev,
+				       uint64_t part_id,
+				       uint64_t obj_id)
+{
+	struct osd_obj_id obj = {
+		.partition = part_id,
+		.id = obj_id
+	};
+	struct osd_request *or = osd_start_request(dev, GFP_KERNEL);
+
+	if (!or)
+		return NULL;
+
+	osd_req_get_attributes(or, &obj);
+
+	return or;
+}
+
+struct osd_request *prepare_osd_read(struct osd_dev *dev,
+				   uint64_t part_id,
+				   uint64_t obj_id,
+				   uint64_t length,
+				   uint64_t offset,
+				   int cmd_data_use_sg,
+				   unsigned char *cmd_data)
+{
+	struct osd_obj_id obj = {
+		.partition = part_id,
+		.id = obj_id
+	};
+	struct osd_request *or = osd_start_request(dev, GFP_KERNEL);
+	struct request_queue *req_q = dev->scsi_device->request_queue;
+	struct bio *bio;
+
+	if (!or)
+		return NULL;
+
+	BUG_ON(cmd_data_use_sg);
+	bio = bio_map_kern(req_q, cmd_data, length, or->alloc_flags);
+	if (!bio) {
+		osd_end_request(or);
+		return NULL;
+	}
+
+	osd_req_read(or, &obj, bio, offset);
+	EXOFS_DBGMSG("osd_req_read(p=%llX, ob=%llX, l=%llu, of=%llu)\n",
+		_LLU(part_id), _LLU(obj_id), _LLU(length), _LLU(offset));
+	return or;
+}
+
+struct osd_request *prepare_osd_write(struct osd_dev *dev,
+				    uint64_t part_id,
+				    uint64_t obj_id,
+				    uint64_t length,
+				    uint64_t offset,
+				    int cmd_data_use_sg,
+				    const unsigned char *cmd_data)
+{
+	struct osd_obj_id obj = {
+		.partition = part_id,
+		.id = obj_id
+	};
+	struct osd_request *or = osd_start_request(dev, GFP_KERNEL);
+	struct request_queue *req_q = dev->scsi_device->request_queue;
+	struct bio *bio;
+
+	if (!or)
+		return NULL;
+
+	BUG_ON(cmd_data_use_sg);
+	bio = bio_map_kern(req_q, (u8 *)cmd_data, length, or->alloc_flags);
+	if (!bio) {
+		osd_end_request(or);
+		return NULL;
+	}
+
+	osd_req_write(or, &obj, bio, offset);
+	EXOFS_DBGMSG("osd_req_write(p=%llX, ob=%llX, l=%llu, of=%llu)\n",
+		_LLU(part_id), _LLU(obj_id), _LLU(length), _LLU(offset));
+	return or;
+}
+
+void free_osd_req(struct osd_request *req)
+{
+	osd_end_request(req);
+}