Skip to contentSkip to conversation

64k pagesize/blocksize fixes

PATCH In review Christoph Lameter 3 participants 8 messages Sep 25, 2007 – Sep 26, 2007

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover 64k pagesize/blocksize fixes

Attached the fixes necessary to support 64k pagesize/blocksize. I think these are useful independent of the large blocksize patchset since there are architectures that support 64k page size and that could use these large buffer sizes without the large buffersize patchset.

Are these patches in the right shape to be merged? I rediffed these against 2.6.32-rc8-mm1.

I had to fix some things in the second patch (ext2) that may need some review since the way that commits work changed.

#1 lore raw

PATCH 1/4 Increase limits for 64k page size support for Ext2/3/4

[This patch allows architectures that use 64k blocksizes--like IA64 and PPC64--to use 64k blocks on ext filesystems]

The patches to support blocksizes up to PAGESIZE, max 64KB for ext2/3/4,\
were originally from Takashi Sato.
http://marc.info/?l=linux-ext4&m=115768873518400&w=2

It's quite simple to support large block size in ext2/3/4, mostly just enlarge the block size limit. But it is NOT possible to have 64kB blocksize on ext2/3/4 without some changes to the directory handling code. The reason is that an empty 64kB directory block would have a rec_len == (__u16)2^16 == 0, and this would cause an error to be hit in the filesystem. The proposed solution is to put 2 empty records in such a directory, or to special-case an impossible value like rec_len = 0xffff to handle this.

---
 fs/ext2/super.c         |    2 +-
 fs/ext3/super.c         |    5 ++++-
 fs/ext4/super.c         |    5 +++++
 include/linux/ext2_fs.h |    4 ++--
 include/linux/ext3_fs.h |    4 ++--
 include/linux/ext4_fs.h |    4 ++--
 6 files changed, 16 insertions(+), 8 deletions(-)
fs/ext2/super.c +1 -1
Index: linux-2.6.23-rc8-mm1/fs/ext2/super.c
===================================================================
@@ -856,7 +856,7 @@ static int ext2_fill_super(struct super_
 		brelse(bh);
 
 		if (!sb_set_blocksize(sb, blocksize)) {
-			printk(KERN_ERR "EXT2-fs: blocksize too small for device.\n");
+			printk(KERN_ERR "EXT2-fs: bad blocksize %d.\n", blocksize);
 			goto failed_sbi;
 		}
 
fs/ext3/super.c +4 -1
Index: linux-2.6.23-rc8-mm1/fs/ext3/super.c
===================================================================
@@ -1625,7 +1625,10 @@ static int ext3_fill_super (struct super
 		}
 
 		brelse (bh);
-		sb_set_blocksize(sb, blocksize);
+		if (!sb_set_blocksize(sb, blocksize)) {
+			printk(KERN_ERR "EXT3-fs: bad blocksize %d.\n", blocksize);
+			goto out_fail;
+		}
 		logic_sb_block = (sb_block * EXT3_MIN_BLOCK_SIZE) / blocksize;
 		offset = (sb_block * EXT3_MIN_BLOCK_SIZE) % blocksize;
 		bh = sb_bread(sb, logic_sb_block);
fs/ext4/super.c +5 -0
Index: linux-2.6.23-rc8-mm1/fs/ext4/super.c
===================================================================
@@ -1652,6 +1652,11 @@ static int ext4_fill_super (struct super
 		goto out_fail;
 	}
 
+	if (!sb_set_blocksize(sb, blocksize)) {
+		printk(KERN_ERR "EXT4-fs: bad blocksize %d.\n", blocksize);
+		goto out_fail;
+	}
+
 	/*
 	 * The ext4 superblock will not be buffer aligned for other than 1kB
 	 * block sizes.  We need to calculate the offset from buffer start.
include/linux/ext2_fs.h +2 -2
Index: linux-2.6.23-rc8-mm1/include/linux/ext2_fs.h
===================================================================
@@ -87,8 +87,8 @@ static inline struct ext2_sb_info *EXT2_
  * Macro-instructions used to manage several block sizes
  */
 #define EXT2_MIN_BLOCK_SIZE		1024
-#define	EXT2_MAX_BLOCK_SIZE		4096
-#define EXT2_MIN_BLOCK_LOG_SIZE		  10
+#define EXT2_MAX_BLOCK_SIZE		65536
+#define EXT2_MIN_BLOCK_LOG_SIZE		10
 #ifdef __KERNEL__
 # define EXT2_BLOCK_SIZE(s)		((s)->s_blocksize)
 #else
include/linux/ext3_fs.h +2 -2
Index: linux-2.6.23-rc8-mm1/include/linux/ext3_fs.h
===================================================================
@@ -76,8 +76,8 @@
  * Macro-instructions used to manage several block sizes
  */
 #define EXT3_MIN_BLOCK_SIZE		1024
-#define	EXT3_MAX_BLOCK_SIZE		4096
-#define EXT3_MIN_BLOCK_LOG_SIZE		  10
+#define	EXT3_MAX_BLOCK_SIZE		65536
+#define EXT3_MIN_BLOCK_LOG_SIZE		10
 #ifdef __KERNEL__
 # define EXT3_BLOCK_SIZE(s)		((s)->s_blocksize)
 #else
include/linux/ext4_fs.h +2 -2
Index: linux-2.6.23-rc8-mm1/include/linux/ext4_fs.h
===================================================================
@@ -77,8 +77,8 @@
  * Macro-instructions used to manage several block sizes
  */
 #define EXT4_MIN_BLOCK_SIZE		1024
-#define	EXT4_MAX_BLOCK_SIZE		4096
-#define EXT4_MIN_BLOCK_LOG_SIZE		  10
+#define	EXT4_MAX_BLOCK_SIZE		65536
+#define EXT4_MIN_BLOCK_LOG_SIZE		10
 #ifdef __KERNEL__
 # define EXT4_BLOCK_SIZE(s)		((s)->s_blocksize)
 #else
#2 lore raw

PATCH 2/4 ext2: fix rec_len overflow for 64KB block size

In reply to Christoph Lameter

[2/4] ext2: fix rec_len overflow
- prevent rec_len from overflow with 64KB blocksize

---
 fs/ext2/dir.c           |   46 ++++++++++++++++++++++++++++++++++++----------
 include/linux/ext2_fs.h |   13 +++++++++++++
 2 files changed, 49 insertions(+), 10 deletions(-)
fs/ext2/dir.c +37 -10
Index: linux-2.6.23-rc8-mm1/fs/ext2/dir.c
===================================================================
@@ -105,9 +105,9 @@ static void ext2_check_page(struct page 
 			goto out;
 	}
 	for (offs = 0; offs <= limit - EXT2_DIR_REC_LEN(1); offs += rec_len) {
+		offs = EXT2_DIR_ADJUST_TAIL_OFFS(offs, chunk_size);
 		p = (ext2_dirent *)(kaddr + offs);
 		rec_len = le16_to_cpu(p->rec_len);
-
 		if (rec_len < EXT2_DIR_REC_LEN(1))
 			goto Eshort;
 		if (rec_len & 3)
@@ -119,6 +119,7 @@ static void ext2_check_page(struct page 
 		if (le32_to_cpu(p->inode) > max_inumber)
 			goto Einumber;
 	}
+	offs = EXT2_DIR_ADJUST_TAIL_OFFS(offs, chunk_size);
 	if (offs != limit)
 		goto Eend;
 out:
@@ -294,6 +295,7 @@ ext2_readdir (struct file * filp, void *
 		de = (ext2_dirent *)(kaddr+offset);
 		limit = kaddr + ext2_last_byte(inode, n) - EXT2_DIR_REC_LEN(1);
 		for ( ;(char*)de <= limit; de = ext2_next_entry(de)) {
+			de = EXT2_DIR_ADJUST_TAIL_ADDR(kaddr, de, sb->s_blocksize);
 			if (de->rec_len == 0) {
 				ext2_error(sb, __FUNCTION__,
 					"zero-length directory entry");
@@ -316,8 +318,10 @@ ext2_readdir (struct file * filp, void *
 					return 0;
 				}
 			}
+			filp->f_pos = EXT2_DIR_ADJUST_TAIL_OFFS(filp->f_pos, sb->s_blocksize);
 			filp->f_pos += le16_to_cpu(de->rec_len);
 		}
+		filp->f_pos = EXT2_DIR_ADJUST_TAIL_OFFS(filp->f_pos, sb->s_blocksize);
 		ext2_put_page(page);
 	}
 	return 0;
@@ -354,13 +358,14 @@ struct ext2_dir_entry_2 * ext2_find_entr
 		start = 0;
 	n = start;
 	do {
-		char *kaddr;
+		char *kaddr, *page_start;
 		page = ext2_get_page(dir, n);
 		if (!IS_ERR(page)) {
-			kaddr = page_address(page);
+			kaddr = page_start = page_address(page);
 			de = (ext2_dirent *) kaddr;
 			kaddr += ext2_last_byte(dir, n) - reclen;
 			while ((char *) de <= kaddr) {
+				de = EXT2_DIR_ADJUST_TAIL_ADDR(page_start, de, dir->i_sb->s_blocksize);
 				if (de->rec_len == 0) {
 					ext2_error(dir->i_sb, __FUNCTION__,
 						"zero-length directory entry");
@@ -428,6 +433,7 @@ void ext2_set_link(struct inode *dir, st
 	unsigned len = le16_to_cpu(de->rec_len);
 	int err;
 
+	len = EXT2_DIR_ADJUST_TAIL_OFFS(pos, len);
 	lock_page(page);
 	err = __ext2_write_begin(NULL, page->mapping, pos, len,
 				AOP_FLAG_UNINTERRUPTIBLE, &page, NULL);
@@ -459,6 +465,7 @@ int ext2_add_link (struct dentry *dentry
 	char *kaddr;
 	loff_t pos;
 	int err;
+	char *page_start = NULL;
 
 	/*
 	 * We take care of directory expansion in the same loop.
@@ -473,16 +480,29 @@ int ext2_add_link (struct dentry *dentry
 		if (IS_ERR(page))
 			goto out;
 		lock_page(page);
-		kaddr = page_address(page);
+		kaddr = page_start = page_address(page);
 		dir_end = kaddr + ext2_last_byte(dir, n);
 		de = (ext2_dirent *)kaddr;
-		kaddr += PAGE_CACHE_SIZE - reclen;
+		if (chunk_size < EXT2_DIR_MAX_REC_LEN)
+			kaddr += PAGE_CACHE_SIZE - reclen;
+		else
+			kaddr += PAGE_CACHE_SIZE -
+				(chunk_size - EXT2_DIR_MAX_REC_LEN) - reclen;
+
 		while ((char *)de <= kaddr) {
+			de = EXT2_DIR_ADJUST_TAIL_ADDR(page_start, de, chunk_size);
 			if ((char *)de == dir_end) {
 				/* We hit i_size */
 				name_len = 0;
-				rec_len = chunk_size;
-				de->rec_len = cpu_to_le16(chunk_size);
+				if (chunk_size  < EXT2_DIR_MAX_REC_LEN) {
+					rec_len = chunk_size;
+					de->rec_len = cpu_to_le16(chunk_size);
+				} else {
+					rec_len = EXT2_DIR_MAX_REC_LEN;
+					de->rec_len =
+						cpu_to_le16(EXT2_DIR_MAX_REC_LEN);
+				}
+
 				de->inode = 0;
 				goto got_it;
 			}
@@ -512,6 +532,7 @@ int ext2_add_link (struct dentry *dentry
 got_it:
 	pos = page_offset(page) +
 		(char*)de - (char*)page_address(page);
+	reclen = EXT2_DIR_ADJUST_TAIL_OFFS(pos + rec_len, chunk_size) - pos;
 	err = __ext2_write_begin(NULL, page->mapping, pos, rec_len, 0,
 							&page, NULL);
 	if (err)
@@ -556,6 +577,7 @@ int ext2_delete_entry (struct ext2_dir_e
 	ext2_dirent * de = (ext2_dirent *) (kaddr + from);
 	int err;
 
+	to = EXT2_DIR_ADJUST_TAIL_OFFS(to, inode->i_sb->s_blocksize);
 	while ((char*)de < (char*)dir) {
 		if (de->rec_len == 0) {
 			ext2_error(inode->i_sb, __FUNCTION__,
@@ -617,7 +639,11 @@ int ext2_make_empty(struct inode *inode,
 
 	de = (struct ext2_dir_entry_2 *)(kaddr + EXT2_DIR_REC_LEN(1));
 	de->name_len = 2;
-	de->rec_len = cpu_to_le16(chunk_size - EXT2_DIR_REC_LEN(1));
+	if (chunk_size < EXT2_DIR_MAX_REC_LEN) {
+		de->rec_len = cpu_to_le16(chunk_size - EXT2_DIR_REC_LEN(1));
+	} else {
+		de->rec_len = cpu_to_le16(EXT2_DIR_MAX_REC_LEN - EXT2_DIR_REC_LEN(1));
+	}
 	de->inode = cpu_to_le32(parent->i_ino);
 	memcpy (de->name, "..\0", 4);
 	ext2_set_de_type (de, inode);
@@ -637,18 +663,19 @@ int ext2_empty_dir (struct inode * inode
 	unsigned long i, npages = dir_pages(inode);
 
 	for (i = 0; i < npages; i++) {
-		char *kaddr;
+		char *kaddr, *page_start;
 		ext2_dirent * de;
 		page = ext2_get_page(inode, i);
 
 		if (IS_ERR(page))
 			continue;
 
-		kaddr = page_address(page);
+		kaddr = page_start = page_address(page);
 		de = (ext2_dirent *)kaddr;
 		kaddr += ext2_last_byte(inode, i) - EXT2_DIR_REC_LEN(1);
 
 		while ((char *)de <= kaddr) {
+			de = EXT2_DIR_ADJUST_TAIL_ADDR(page_start, de, inode->i_sb->s_blocksize);
 			if (de->rec_len == 0) {
 				ext2_error(inode->i_sb, __FUNCTION__,
 					"zero-length directory entry");
include/linux/ext2_fs.h +12 -0
Index: linux-2.6.23-rc8-mm1/include/linux/ext2_fs.h
===================================================================
@@ -561,6 +561,18 @@ enum {
 #define EXT2_DIR_ROUND 			(EXT2_DIR_PAD - 1)
 #define EXT2_DIR_REC_LEN(name_len)	(((name_len) + 8 + EXT2_DIR_ROUND) & \
 					 ~EXT2_DIR_ROUND)
+#define EXT2_DIR_MAX_REC_LEN            65532
+
+/*
+ * Align a tail offset(address) to the end of a directory block
+ */
+#define EXT2_DIR_ADJUST_TAIL_OFFS(offs, bsize) \
+	((((offs) & ((bsize) -1)) == EXT2_DIR_MAX_REC_LEN) ? \
+	((offs) + (bsize) - EXT2_DIR_MAX_REC_LEN):(offs))
+
+#define EXT2_DIR_ADJUST_TAIL_ADDR(page, de, bsize) \
+	(((((char*)(de) - (page)) & ((bsize) - 1)) == EXT2_DIR_MAX_REC_LEN) ? \
+	((ext2_dirent*)((char*)(de) + (bsize) - EXT2_DIR_MAX_REC_LEN)):(de))
 
 static inline ext2_fsblk_t
 ext2_group_first_block_no(struct super_block *sb, unsigned long group_no)
#3 lore raw

Re: [patch 2/4] ext2: fix rec_len overflow for 64KB block size

Quoted 6 lines · Christoph Lameter

[2/4] ext2: fix rec_len overflow
- prevent rec_len from overflow with 64KB blocksize

Note that we just got a cleaner implemantation of this code on the ext4 mailing list from Jan Kara yesterday. Please use that one instead, in thread "Avoid rec_len overflow with 64KB block size" instead.

Cheers, Andreas
--
Andreas Dilger
Principal Software Engineer
Cluster File Systems, Inc.

#4 lore raw

Re: [patch 2/4] ext2: fix rec_len overflow for 64KB block size

Quoted 11 lines · Andreas Dilger
Quoted 6 lines · Christoph Lameter

[2/4] ext2: fix rec_len overflow
- prevent rec_len from overflow with 64KB blocksize

Note that we just got a cleaner implemantation of this code on the ext4 mailing list from Jan Kara yesterday. Please use that one instead, in thread "Avoid rec_len overflow with 64KB block size" instead.

Yes that looks much better. He is missing the first patch that changes the constants.

How can we get the 4 patches (the constant change and Jan's changes to ext 2 3 4 merged?

#5 lore raw

PATCH 3/4 ext3: fix rec_len overflow with 64KB block size

In reply to Christoph Lameter

Prevent rec_len from overflow with 64KB blocksize

---
 fs/ext3/dir.c           |   13 ++++---
 fs/ext3/namei.c         |   88 +++++++++++++++++++++++++++++++++++++++---------
 include/linux/ext3_fs.h |    9 ++++
 3 files changed, 91 insertions(+), 19 deletions(-)
fs/ext3/dir.c +9 -4
Index: linux-2.6.23-rc8-mm1/fs/ext3/dir.c
===================================================================
@@ -100,10 +100,11 @@ static int ext3_readdir(struct file * fi
 	unsigned long offset;
 	int i, stored;
 	struct ext3_dir_entry_2 *de;
-	struct super_block *sb;
 	int err;
 	struct inode *inode = filp->f_path.dentry->d_inode;
 	int ret = 0;
+	struct super_block *sb = inode->i_sb;
+	unsigned tail = sb->s_blocksize;
 
 	sb = inode->i_sb;
 
@@ -167,8 +168,11 @@ revalidate:
 		 * readdir(2), then we might be pointing to an invalid
 		 * dirent right now.  Scan from the start of the block
 		 * to make sure. */
-		if (filp->f_version != inode->i_version) {
-			for (i = 0; i < sb->s_blocksize && i < offset; ) {
+		if (tail >  EXT3_DIR_MAX_REC_LEN)
+			tail = EXT3_DIR_MAX_REC_LEN;
+
+               if (filp->f_version != inode->i_version) {
+			for (i = 0; i < tail && i < offset; ) {
 				de = (struct ext3_dir_entry_2 *)
 					(bh->b_data + i);
 				/* It's too expensive to do a full
@@ -189,7 +193,7 @@ revalidate:
 		}
 
 		while (!error && filp->f_pos < inode->i_size
-		       && offset < sb->s_blocksize) {
+		       && offset < tail) {
 			de = (struct ext3_dir_entry_2 *) (bh->b_data + offset);
 			if (!ext3_check_dir_entry ("ext3_readdir", inode, de,
 						   bh, offset)) {
@@ -225,6 +229,7 @@ revalidate:
 			}
 			filp->f_pos += le16_to_cpu(de->rec_len);
 		}
+		filp->f_pos = EXT3_DIR_ADJUST_TAIL_OFFS(filp->f_pos, sb->s_blocksize);
 		offset = 0;
 		brelse (bh);
 	}
fs/ext3/namei.c +73 -15

196 lines not on this page · raw message

include/linux/ext3_fs.h +9 -0
Index: linux-2.6.23-rc8-mm1/include/linux/ext3_fs.h
===================================================================
@@ -660,6 +660,15 @@ struct ext3_dir_entry_2 {
 #define EXT3_DIR_ROUND			(EXT3_DIR_PAD - 1)
 #define EXT3_DIR_REC_LEN(name_len)	(((name_len) + 8 + EXT3_DIR_ROUND) & \
 					 ~EXT3_DIR_ROUND)
+#define EXT3_DIR_MAX_REC_LEN		65532
+
+/*
+ * Align a tail offset to the end of a directory block
+ */
+#define EXT3_DIR_ADJUST_TAIL_OFFS(offs, bsize) \
+	((((offs) & ((bsize) -1)) == EXT3_DIR_MAX_REC_LEN) ? \
+	((offs) + (bsize) - EXT3_DIR_MAX_REC_LEN):(offs))
+
 /*
  * Hash Tree Directory indexing
  * (c) Daniel Phillips, 2001
#6 lore raw

PATCH 4/4 ext4: fix rec_len overflow for 64KB block size

In reply to Christoph Lameter

[4/4] ext4: fix rec_len overflow
- prevent rec_len from overflow with 64KB blocksize

---
 fs/ext4/dir.c           |   11 ++++--
 fs/ext4/namei.c         |   88 +++++++++++++++++++++++++++++++++++++++---------
 include/linux/ext4_fs.h |    9 ++++
 3 files changed, 90 insertions(+), 18 deletions(-)
fs/ext4/dir.c +8 -3
Index: linux-2.6.23-rc8-mm1/fs/ext4/dir.c
===================================================================
@@ -100,10 +100,11 @@ static int ext4_readdir(struct file * fi
 	unsigned long offset;
 	int i, stored;
 	struct ext4_dir_entry_2 *de;
-	struct super_block *sb;
 	int err;
 	struct inode *inode = filp->f_path.dentry->d_inode;
+	struct super_block *sb = inode->i_sb;
 	int ret = 0;
+	unsigned tail = sb->s_blocksize;
 
 	sb = inode->i_sb;
 
@@ -166,8 +167,11 @@ revalidate:
 		 * readdir(2), then we might be pointing to an invalid
 		 * dirent right now.  Scan from the start of the block
 		 * to make sure. */
+		if (tail >  EXT4_DIR_MAX_REC_LEN)
+			tail = EXT4_DIR_MAX_REC_LEN;
+
 		if (filp->f_version != inode->i_version) {
-			for (i = 0; i < sb->s_blocksize && i < offset; ) {
+			for (i = 0; i < tail && i < offset; ) {
 				de = (struct ext4_dir_entry_2 *)
 					(bh->b_data + i);
 				/* It's too expensive to do a full
@@ -188,7 +192,7 @@ revalidate:
 		}
 
 		while (!error && filp->f_pos < inode->i_size
-		       && offset < sb->s_blocksize) {
+		       && offset < tail) {
 			de = (struct ext4_dir_entry_2 *) (bh->b_data + offset);
 			if (!ext4_check_dir_entry ("ext4_readdir", inode, de,
 						   bh, offset)) {
@@ -225,6 +229,7 @@ revalidate:
 			}
 			filp->f_pos += le16_to_cpu(de->rec_len);
 		}
+		filp->f_pos = EXT4_DIR_ADJUST_TAIL_OFFS(filp->f_pos, sb->s_blocksize);
 		offset = 0;
 		brelse (bh);
 	}
fs/ext4/namei.c +73 -15

196 lines not on this page · raw message

include/linux/ext4_fs.h +9 -0
Index: linux-2.6.23-rc8-mm1/include/linux/ext4_fs.h
===================================================================
@@ -779,6 +779,15 @@ struct ext4_dir_entry_2 {
 #define EXT4_DIR_ROUND			(EXT4_DIR_PAD - 1)
 #define EXT4_DIR_REC_LEN(name_len)	(((name_len) + 8 + EXT4_DIR_ROUND) & \
 					 ~EXT4_DIR_ROUND)
+#define	EXT4_DIR_MAX_REC_LEN		65532
+
+/*
+ * Align a tail offset to the end of a directory block
+ */
+#define EXT4_DIR_ADJUST_TAIL_OFFS(offs, bsize) \
+	((((offs) & ((bsize) -1)) == EXT4_DIR_MAX_REC_LEN) ? \
+	((offs) + (bsize) - EXT4_DIR_MAX_REC_LEN):(offs))
+
 /*
  * Hash Tree Directory indexing
  * (c) Daniel Phillips, 2001
#7 lore raw
In reply to Christoph Lameter
Quoted 10 lines · Christoph Lameter

Attached the fixes necessary to support 64k pagesize/blocksize. I think these are useful independent of the large blocksize patchset since there are architectures that support 64k page size and that could use these large buffer sizes without the large buffersize patchset.

Are these patches in the right shape to be merged? I rediffed these against 2.6.32-rc8-mm1.

I had to fix some things in the second patch (ext2) that may need some review since the way that commits work changed.

Thanks, As Andreas mentioned in another email there is a better way to deal with rec_len issue on 64k block size. I will get those patches merged in ext4 patch queue. I think the updated patches could get merged.

Mingming