exofs & pnfsd-exofs assorted changes

index 21b1c71..474d99f 100644
@@ -66,9 +66,6 @@ struct exofs_sb_info {
 	u32		s_next_generation;	/* next gen # to use          */
 	atomic_t	s_curr_pending;		/* number of pending commands */
 
-	struct pnfs_osd_data_map data_map;	/* Default raid to use
-						 * FIXME: Needed ?
-						 */
 	struct ore_layout	layout;		/* Default files layout       */
 	struct ore_comp one_comp;		/* id & cred of partition id=0*/
 	struct ore_components comps;		/* comps for the partition    */

93 lines not on this page · raw message

index c5c5e00..0ac4931 100644
@@ -34,6 +34,8 @@ struct ore_comp {
 
 struct ore_layout {
 	/* Our way of looking at the data_map */
+	enum pnfs_osd_raid_algorithm4
+		 raid_algorithm;
 	unsigned stripe_unit;
 	unsigned mirrors_p1;
 

index 07bc5e2..10b9adb 100644
@@ -73,11 +73,16 @@ err:
 	return status;
 }
 
-static unsigned exofs_layout_od_id(struct ore_layout *layout,
-				   struct ore_components *comps,
-				   osd_id obj_no, unsigned layout_index)
+void ore_layout_2_pnfs_layout(struct pnfs_osd_layout *pl,
+			      const struct ore_layout *ol)
 {
-	return (layout_index + obj_no * layout->mirrors_p1) % comps->numdevs;
+	pl->olo_map.odm_num_comps = ol->group_width * ol->mirrors_p1 *
+				    ol->group_count;
+	pl->olo_map.odm_stripe_unit = ol->stripe_unit;
+	pl->olo_map.odm_group_width = ol->group_width;
+	pl->olo_map.odm_group_depth = ol->group_depth;
+	pl->olo_map.odm_mirror_cnt = ol->mirrors_p1 - 1;
+	pl->olo_map.odm_raid_algorithm = ol->raid_algorithm;
 }
 
 static enum nfsstat4 exofs_layout_get(

@@ -88,9 +93,6 @@ static enum nfsstat4 exofs_layout_get(
 {
 	struct exofs_i_info *oi = exofs_i(inode);
 	struct exofs_sb_info *sbi = inode->i_sb->s_fs_info;
-	struct ore_layout *el = &sbi->layout;
-	struct ore_components *ec = &sbi->comps;
-	struct pnfs_osd_object_cred *creds = NULL;
 	struct pnfs_osd_layout layout;
 	__be32 *start;
 	unsigned i;

@@ -110,29 +112,25 @@ static enum nfsstat4 exofs_layout_get(
 	}
 
 	/* Fill in a pnfs_osd_layout struct */
-	layout.olo_map = sbi->data_map;
+	ore_layout_2_pnfs_layout(&layout, &sbi->layout);
+
 	layout.olo_comps_index = 0;
-	layout.olo_num_comps = ec->numdevs;
-	layout.olo_comps = creds;
+	layout.olo_num_comps = layout.olo_map.odm_num_comps;
 
 	nfserr = pnfs_osd_xdr_encode_layout_hdr(xdr, &layout);
 	if (unlikely(nfserr))
 		goto out;
 
 	/* Encode layout components */
-	for (i = 0; i < ec->numdevs; i++) {
+	for (i = 0; i <  layout.olo_num_comps; i++) {
 		struct pnfs_osd_object_cred cred;
-		struct osd_obj_id oid = {
-			.partition = sbi->one_comp.obj.partition,
-			.id = exofs_oi_objno(oi)
-		};
-		unsigned dev = exofs_layout_od_id(el, ec, oid.id, i);
+		unsigned sbi_dev = oi->comps.ods - sbi->comps.ods + i;
 
 		set_dev_id(&cred.oc_object_id.oid_device_id, args->lg_sbid,
-			   dev);
-		cred.oc_object_id.oid_partition_id = oid.partition;
-		cred.oc_object_id.oid_object_id = oid.id;
-		cred.oc_osd_version = osd_dev_is_ver1(ec->ods[dev]) ?
+			   sbi_dev);
+		cred.oc_object_id.oid_partition_id = oi->one_comp.obj.partition;
+		cred.oc_object_id.oid_object_id = oi->one_comp.obj.id;
+		cred.oc_osd_version = osd_dev_is_ver1(oi->comps.ods[i]) ?
 						PNFS_OSD_VERSION_1 :
 						PNFS_OSD_VERSION_2;
 		cred.oc_cap_key_sec = PNFS_OSD_CAP_KEY_SEC_NONE;

@@ -141,10 +139,15 @@ static enum nfsstat4 exofs_layout_get(
 		cred.oc_cap_key.cred		= NULL;
 
 		cred.oc_cap.cred_len	= OSD_CAP_LEN;
-		exofs_make_credential(cred.oc_cap.cred, &oid);
+		cred.oc_cap.cred	= oi->one_comp.cred;
+
 		nfserr = pnfs_osd_xdr_encode_layout_cred(xdr, &cred);
-		if (unlikely(nfserr))
+		if (unlikely(nfserr)) {
+			EXOFS_DBGMSG("(0x%lx) nfserr=%u total=%u encoded=%u\n",
+				     inode->i_ino, nfserr, layout.olo_num_comps,
+				     i - 1);
 			goto out;
+		}
 	}
 
 	exp_xdr_encode_opaque_len(start, xdr->p);

@@ -160,9 +163,9 @@ static enum nfsstat4 exofs_layout_get(
 	spin_unlock(&oi->i_layout_lock);
 
 out:
-	kfree(creds);
-	EXOFS_DBGMSG("(0x%lx) nfserr=%u xdr_bytes=%zu\n",
-		     inode->i_ino, nfserr, exp_xdr_qbytes(xdr->p - start));
+	if (unlikely(nfserr))
+		EXOFS_DBGMSG("(0x%lx) nfserr=%u xdr_bytes=%zu\n",
+			  inode->i_ino, nfserr, exp_xdr_qbytes(xdr->p - start));
 	return nfserr;
 }
 

@@ -288,7 +291,6 @@ int exofs_get_device_info(struct super_block *sb, struct exp_xdr_stream *xdr,
 			  const struct nfsd4_pnfs_deviceid *devid)
 {
 	struct exofs_sb_info *sbi = sb->s_fs_info;
-	struct ore_components *ec = &sbi->comps;
 	struct pnfs_osd_deviceaddr devaddr;
 	const struct osd_dev_info *odi;
 	u64 devno = devid->devid;

@@ -297,13 +299,13 @@ int exofs_get_device_info(struct super_block *sb, struct exp_xdr_stream *xdr,
 
 	memset(&devaddr, 0, sizeof(devaddr));
 
-	if (unlikely(devno >= ec->numdevs)) {
+	if (unlikely(devno >= sbi->comps.numdevs)) {
 		EXOFS_DBGMSG("Error: Device((%llx,%llx) does not exist\n",
 			     devid->sbid, devno);
 		return -ENODEV;
 	}
 
-	odi = osduld_device_info(ec->ods[devno]);
+	odi = osduld_device_info(sbi->comps.ods[devno]);
 
 	devaddr.oda_systemid.len = odi->systemid_len;
 	devaddr.oda_systemid.data = (void *)odi->systemid; /* !const cast */

index 25305af..a2c09e7 100644
@@ -287,16 +287,8 @@ EXPORT_SYMBOL(ore_check_io);
  *
  *	O = L % stripe_unit + N * stripe_unit + M * group_depth * stripe_unit
  */
-struct _striping_info {
-	u64 obj_offset;
-	u64 group_length;
-	u64 M; /* for truncate */
-	unsigned dev;
-	unsigned unit_off;
-};
-
-static void _calc_stripe_info(struct ore_layout *layout, u64 file_offset,
-			      struct _striping_info *si)
+void ore_calc_stripe_info(struct ore_layout *layout, u64 file_offset,
+			  struct ore_striping_info *si)
 {
 	u32	stripe_unit = layout->stripe_unit;
 	u32	group_width = layout->group_width;

@@ -329,6 +321,7 @@ static void _calc_stripe_info(struct ore_layout *layout, u64 file_offset,
 	si->group_length = T - H;
 	si->M = M;
 }
+EXPORT_SYMBOL(ore_calc_stripe_info);
 
 static int _add_stripe_unit(struct ore_io_state *ios,  unsigned *cur_pg,
 		unsigned pgbase, struct ore_per_dev_state *per_dev,

@@ -375,7 +368,7 @@ static int _add_stripe_unit(struct ore_io_state *ios,  unsigned *cur_pg,
 }
 
 static int _prepare_one_group(struct ore_io_state *ios, u64 length,
-			      struct _striping_info *si)
+			      struct ore_striping_info *si)
 {
 	unsigned stripe_unit = ios->layout->stripe_unit;
 	unsigned mirrors_p1 = ios->layout->mirrors_p1;

@@ -434,14 +427,14 @@ static int _prepare_for_striping(struct ore_io_state *ios)
 {
 	u64 length = ios->length;
 	u64 offset = ios->offset;
-	struct _striping_info si;
+	struct ore_striping_info si;
 	int ret = 0;
 
 	if (!ios->pages) {
 		if (ios->kern_buff) {
 			struct ore_per_dev_state *per_dev = &ios->per_dev[0];
 
-			_calc_stripe_info(ios->layout, ios->offset, &si);
+			ore_calc_stripe_info(ios->layout, ios->offset, &si);
 			per_dev->offset = si.obj_offset;
 			per_dev->dev = si.dev;
 

@@ -455,7 +448,7 @@ static int _prepare_for_striping(struct ore_io_state *ios)
 	}
 
 	while (length) {
-		_calc_stripe_info(ios->layout, offset, &si);
+		ore_calc_stripe_info(ios->layout, offset, &si);
 
 		if (length < si.group_length)
 			si.group_length = length;

@@ -744,7 +737,7 @@ static int _truncate_mirrors(struct ore_io_state *ios, unsigned cur_comp,
 }
 
 struct _trunc_info {
-	struct _striping_info si;
+	struct ore_striping_info si;
 	u64 prev_group_obj_off;
 	u64 next_group_obj_off;
 

@@ -758,7 +751,7 @@ void _calc_trunk_info(struct ore_layout *layout, u64 file_offset,
 {
 	unsigned stripe_unit = layout->stripe_unit;
 
-	_calc_stripe_info(layout, file_offset, &ti->si);
+	ore_calc_stripe_info(layout, file_offset, &ti->si);
 
 	ti->prev_group_obj_off = ti->si.M * stripe_unit;
 	ti->next_group_obj_off = ti->si.M ? (ti->si.M - 1) * stripe_unit : 0;

index 0ac4931..4779ccc 100644
@@ -96,6 +96,13 @@ struct ore_io_state {
 		unsigned dev;
 	} per_dev[];
 };
+struct ore_striping_info {
+	u64 obj_offset;
+	u64 group_length;
+	u64 M; /* for truncate */
+	unsigned dev;
+	unsigned unit_off;
+};
 
 static inline unsigned ore_io_state_size(unsigned numdevs)
 {

@@ -104,6 +111,9 @@ static inline unsigned ore_io_state_size(unsigned numdevs)
 }
 
 /* ore.c */
+void ore_calc_stripe_info(struct ore_layout *layout, u64 file_offset,
+			  struct ore_striping_info *si);
+
 int ore_get_rw_state(struct ore_layout *layout, struct ore_components *comps,
 		     bool is_reading, u64 offset, u64 length,
 		     struct ore_io_state **ios);

index 10b9adb..5d8333c 100644
@@ -85,6 +85,15 @@ void ore_layout_2_pnfs_layout(struct pnfs_osd_layout *pl,
 	pl->olo_map.odm_raid_algorithm = ol->raid_algorithm;
 }
 
+static void _align_io(struct ore_layout *layout, u64 *offset, u64 *length)
+{
+	u64 stripe_size = layout->group_width * layout->stripe_unit;
+	u64 group_size = stripe_size * layout->group_depth;
+
+	*offset = div64_u64(*offset, group_size) * group_size;
+	*length = group_size;
+}
+
 static enum nfsstat4 exofs_layout_get(
 	struct inode *inode,
 	struct exp_xdr_stream *xdr,

@@ -93,16 +102,24 @@ static enum nfsstat4 exofs_layout_get(
 {
 	struct exofs_i_info *oi = exofs_i(inode);
 	struct exofs_sb_info *sbi = inode->i_sb->s_fs_info;
+	struct ore_striping_info si;
 	struct pnfs_osd_layout layout;
 	__be32 *start;
 	unsigned i;
 	bool in_recall;
 	enum nfsstat4 nfserr;
 
-	res->lg_seg.offset = 0;
-	res->lg_seg.length = NFS4_MAX_UINT64;
+	EXOFS_DBGMSG("(0x%lx) REQUESTED offset=0x%llx len=0x%llx iomod=0x%x\n",
+		     inode->i_ino, res->lg_seg.offset,
+		     res->lg_seg.length, res->lg_seg.iomode);
+
+	_align_io(&sbi->layout, &res->lg_seg.offset, &res->lg_seg.length);
 	res->lg_seg.iomode = IOMODE_RW;
-	res->lg_return_on_close = true; /* TODO: unused but will be soon */
+	res->lg_return_on_close = true;
+
+	EXOFS_DBGMSG("(0x%lx) RETURNED offset=0x%llx len=0x%llx iomod=0x%x\n",
+		     inode->i_ino, res->lg_seg.offset,
+		     res->lg_seg.length, res->lg_seg.iomode);
 
 	/* skip opaque size, will be filled-in later */
 	start = exp_xdr_reserve_qwords(xdr, 1);

@@ -114,15 +131,16 @@ static enum nfsstat4 exofs_layout_get(
 	/* Fill in a pnfs_osd_layout struct */
 	ore_layout_2_pnfs_layout(&layout, &sbi->layout);
 
-	layout.olo_comps_index = 0;
-	layout.olo_num_comps = layout.olo_map.odm_num_comps;
+	ore_calc_stripe_info(&sbi->layout, res->lg_seg.offset, &si);
+	layout.olo_comps_index = si.dev;
+	layout.olo_num_comps = sbi->layout.group_width * sbi->layout.mirrors_p1;
 
 	nfserr = pnfs_osd_xdr_encode_layout_hdr(xdr, &layout);
 	if (unlikely(nfserr))
 		goto out;
 
 	/* Encode layout components */
-	for (i = 0; i <  layout.olo_num_comps; i++) {
+	for (i = si.dev; i < si.dev + layout.olo_num_comps; i++) {
 		struct pnfs_osd_object_cred cred;
 		unsigned sbi_dev = oi->comps.ods - sbi->comps.ods + i;
 

@@ -145,7 +163,7 @@ static enum nfsstat4 exofs_layout_get(
 		if (unlikely(nfserr)) {
 			EXOFS_DBGMSG("(0x%lx) nfserr=%u total=%u encoded=%u\n",
 				     inode->i_ino, nfserr, layout.olo_num_comps,
-				     i - 1);
+				     i - si.dev);
 			goto out;
 		}
 	}