isofs meta-data beyond 4GB
PATCH 1/1 isofs meta-data beyond 4GB
The ISO 9660 standard allows meta-data to be stored at almost any arbitrary byte on the file system. The current inode scheme uses the byte offset as the inode value making it easy to find the underlying block and block offset.
This scheme is subject to obvious integer overflow problems that prevents it from being able to reach meta-data beyond the 4GB boundary. Looking back through the archives, this problem was anticipated but discounted because mkisofs puts its meta-data near the beginning of the file system.
However, there are at least two common scenarios where meta-data must be located beyond the 4GB boundary. First, growisofs needs to do this when it merges a new session with an old session that ended beyond the 4GB boundary. The following URL has more information:
http://fy.chalmers.se/~appro/linux/DVD+RW/#isofs4gb
Second (and what prompted this patch), I've written two programs that together allow arbitrarily large backups to be made to a set of DVDs without the need for any intermediate files (in the same manner as when working with tapes). In order to use all the space on a DVD, I'm forced to write the meta-data beyond the current 4GB inode boundary because I don't have all it until I reach the end of the file system. The following URL has more information:
To support these types of DVDs, the inode scheme for isofs must be changed. The patch I'm submitting for review provides one such comprehensive inode scheme. It assigns inode numbers sequentially starting with 1 for the root inode. It keeps a mapping for each inode that is indexed both by the inode number and by the block number and block offset. The indexes are implemented using two rbtrees and are protected by a reader-writer spin lock.
Thanks
Paul Serice
========================================================================
fs/isofs/dir.c +12 -2
@@ -115,6 +115,7 @@ static int do_isofs_readdir(struct inodechar *p =NULL ;/* Quiet GCC */ struct iso_directory_record *de;struct isofs_sb_info *sbi =ISOFS_SB (inode->i_sb); +struct isofs_inode_map *inode_map = &sbi->s_inode_map; offset = filp->f_pos & (bufsize -1 ); block = filp->f_pos >> bufbits;
@@ -129,8 +130,17 @@ static int do_isofs_readdir(struct inode } de = (struct iso_directory_record *) (bh->b_data + offset); -if (first_de) - inode_number = (bh->b_blocknr << bufbits) + offset; +if (first_de) { + inode_number = +isofs_inode_map_index_by_phy (inode_map, + bh->b_blocknr, + offset); +if (inode_number ==0 ) { +printk (KERN_ERR "iso9660: Unable to index " +"inode!\n" ); +return 0 ; + } + } de_len = *(unsigned char *) de;
fs/isofs/inode.c +69 -33
241 lines not on this page · raw message
fs/isofs/inode_map.c +411 -0
411 lines not on this page · raw message
fs/isofs/Makefile +1 -1
@@ -4,7 +4,7 @@ obj-$(CONFIG_ISO9660_FS) += isofs.o -isofs-objs-y := namei.o inode.o dir.o util.o rock.o +isofs-objs-y := namei.o inode.o inode_map.o dir.o util.o rock.o isofs-objs-$(CONFIG_JOLIET) += joliet.o isofs-objs-$(CONFIG_ZISOFS) += compress.o isofs-objs := $(isofs-objs-y)
fs/isofs/namei.c +8 -1
@@ -65,6 +65,8 @@ isofs_find_entry(struct inode *dir, struunsigned long bufsize =ISOFS_BUFFER_SIZE (dir);unsigned char bufbits =ISOFS_BUFFER_BITS (dir);unsigned int block, f_pos, offset; +sector_t abs_block; +unsigned long abs_block_offset;struct buffer_head * bh =NULL ;struct isofs_sb_info *sbi =ISOFS_SB (dir->i_sb);
@@ -87,7 +89,8 @@ isofs_find_entry(struct inode *dir, stru } de = (struct iso_directory_record *) (bh->b_data + offset); - inode_number = (bh->b_blocknr << bufbits) + offset; + abs_block = bh->b_blocknr; + abs_block_offset = offset; de_len = *(unsigned char *) de;if (!de_len) {
@@ -151,6 +154,10 @@ isofs_find_entry(struct inode *dir, stru }if (match) {if (bh)brelse (bh); + inode_number = +isofs_inode_map_index_by_phy (&sbi->s_inode_map, + abs_block, + abs_block_offset);return inode_number; } }
fs/isofs/rock.c +26 -10
@@ -164,6 +164,8 @@ int parse_rock_ridge_inode_internal(struint len;unsigned char * chr;int symlink_len =0 ; +unsigned long inode_number =0 ; +struct isofs_inode_map *map = &(ISOFS_SB (inode->i_sb)->s_inode_map);CONTINUE_DECLS ;if (!ISOFS_SB (inode->i_sb)->s_rock)return 0 ;
@@ -306,9 +308,16 @@ int parse_rock_ridge_inode_internal(stru goto out;case SIG ('C' ,'L' ):ISOFS_I (inode)->i_first_extent =isonum_733 (rr->u.CL.location); - reloc =iget (inode->i_sb, - (ISOFS_I (inode)->i_first_extent << -ISOFS_SB (inode->i_sb)->s_log_zone_size)); + inode_number = +isofs_inode_map_index_by_phy (map, +ISOFS_I (inode)->i_first_extent, +0 ); +if (inode_number ==0 ) { +printk (KERN_ERR "Inode number overflowed while " +"parsing rock ridge inode.\n" ); + goto out; + } + reloc =iget (inode->i_sb, inode_number);if (!reloc) goto out; inode->i_mode = reloc->i_mode;
@@ -447,37 +456,43 @@ int parse_rock_ridge_inode(struct iso_distatic int rock_ridge_symlink_readpage (struct file *file,struct page *page) {struct inode *inode = page->mapping->host; +struct isofs_sb_info * sbi =ISOFS_SB (inode->i_sb);char *link =kmap (page);unsigned long bufsize =ISOFS_BUFFER_SIZE (inode); -unsigned char bufbits =ISOFS_BUFFER_BITS (inode);struct buffer_head *bh;char *rpnt = link;unsigned char *pnt;struct iso_directory_record *raw_inode;CONTINUE_DECLS ; -int block; +struct isofs_inode_map *map; +struct isofs_inode_map_node *map_node;int sig;int len;unsigned char *chr;struct rock_ridge *rr; -if (!ISOFS_SB (inode->i_sb)->s_rock) +if (!sbi->s_rock)panic ("Cannot have symlink with high sierra variant of iso filesystem\n" ); - block = inode->i_ino >> bufbits; + map = &sbi->s_inode_map; + map_node =isofs_inode_map_find_by_ino (map, inode->i_ino); +if (map_node ==NULL ) { + goto out_eio; + } +lock_kernel (); - bh =sb_bread (inode->i_sb, block); + bh =sb_bread (inode->i_sb, map_node->block);if (!bh) goto out_noread; - pnt = (unsigned char *) bh->b_data + (inode->i_ino & (bufsize -1 )); + pnt = (unsigned char *) bh->b_data + map_node->block_offset; raw_inode = (struct iso_directory_record *) pnt;/* * If we go past the end of the buffer, there is some sort of error. */ -if ((inode->i_ino & (bufsize -1 )) + *pnt > bufsize) +if (map_node->block_offset + *pnt > bufsize) goto out_bad_span;/* Now test for possible Rock Ridge extensions which will override
@@ -543,6 +558,7 @@ static int rock_ridge_symlink_readpage(sSetPageError (page);kunmap (page);unlock_page (page); + out_eio:return -EIO ; }
include/linux/iso_fs.h +38 -0
@@ -2,6 +2,7 @@#ifndef _ISOFS_FS_H#define _ISOFS_FS_H +#include <linux/rbtree.h> #include <linux/types.h> /* * The isofs filesystem constants/structures
@@ -155,6 +156,18 @@ struct iso_directory_record {char name [0 ]; } __attribute__((packed)); + +/* Inode Map Node Cache -- Explicitly map 32-bit inode numbers to the +* arbitrary block and offset of an ISO 9660 Directory Record. */ +struct isofs_inode_map_node { +unsigned long inode_number; +sector_t block; +unsigned long block_offset; +struct rb_node node_for_phy; +struct rb_node node_for_ino; +}; + +#define ISOFS_BLOCK_BITS 11#define ISOFS_BLOCK_SIZE 2048
@@ -231,6 +244,31 @@ extern struct dentry *isofs_lookup(strucextern struct buffer_head *isofs_bread (struct inode *,sector_t );extern int isofs_get_blocks (struct inode *,sector_t ,struct buffer_head **,unsigned long ); +extern int init_inode_map_node_cache (void ); +extern void destroy_inode_map_node_cache (void ); + +extern void isofs_inode_map_init (struct isofs_inode_map *map); + +extern void isofs_inode_map_set_root_inode (struct isofs_inode_map *map, +sector_t block, +unsigned long block_offset); + +extern struct isofs_inode_map_node * +isofs_inode_map_find_by_ino (struct isofs_inode_map *map, +unsigned long inode_number); + +extern struct isofs_inode_map_node * +isofs_inode_map_find_by_phy (struct isofs_inode_map *map, +sector_t block, +unsigned long block_offset); + +extern unsigned long +isofs_inode_map_index_by_phy (struct isofs_inode_map *map, +sector_t block, +unsigned long block_offset); + +extern void isofs_inode_map_destroy (struct isofs_inode_map *map); +extern struct inode_operations isofs_dir_inode_operations;extern struct file_operations isofs_dir_operations;extern struct address_space_operations isofs_symlink_aops;
include/linux/iso_fs_sb.h +11 -0
@@ -1,6 +1,15 @@#ifndef _ISOFS_FS_SB#define _ISOFS_FS_SB + +struct isofs_inode_map { +rwlock_t lock; +unsigned long next_inode_number; +struct rb_root index_by_phy; +struct rb_root index_by_ino; +}; + +/* * iso9660 super-block data in memory */
@@ -29,6 +38,8 @@ struct isofs_sb_info {gid_t s_gid;uid_t s_uid;struct nls_table *s_nls_iocharset;/* Native language support table */ + + struct isofs_inode_map s_inode_map; };#endif
On Thu, May 20, 2004 at 09:26:12AM -0500, Paul Serice wrote:
<skip/>
Quoted 7 lines
To support these types of DVDs, the inode scheme for isofs must be changed. The patch I'm submitting for review provides one such comprehensive inode scheme. It assigns inode numbers sequentially starting with 1 for the root inode. It keeps a mapping for each inode that is indexed both by the inode number and by the block number and block offset. The indexes are implemented using two rbtrees and are protected by a reader-writer spin lock.
Why use a spinlock here? The filesystem code is always called in a process context, therefore a semaphore is much more appropriate than a spinlock. BTW, your code allocates memory with SLAB_KERNEL while holding a spinlock, which is wrong; the check for allocation failure is also missing.
Also there are other, more serious problems with your approach:
- The inode map can consume a lot of memory, which is never released until unmount. Someone could prepare a disk full of zero-length files and run "find" on it to crash the machine.
- Inode numbers for the same disk become not stable across mounts (can be annoying if automounters are used).
However, the 4GB limit problem really needs to be solved. Here is another idea (sorry, no patch yet): instead of the byte offset of the directory entry, use its sector number and index in the sector. isofs_read_inode() then would need to read the sector and skip the specified number of directory entries to find the needed one.
The minimum possible size of an ISO9660 directory entry is 34 bytes (33 bytes for struct iso_directory_record, 1 byte for file name). Therefore there cannot be more than 60 directory entries in a single CD-ROM sector (2048 bytes), so the number of a directory entry in the sector can fit into 6 bits. With 32-bit inode numbers this leaves 26 bits for the sector number, which is enough for up to 128GB.
Quoted 5 lines
Why use a spinlock here? The filesystem code is always called in a process context, therefore a semaphore is much more appropriate than a spinlock. BTW, your code allocates memory with
SLAB_KERNELwhile holding a spinlock, which is wrong; the check for allocation failure is also missing.
Thanks! I'm attaching a full patch with your recommended changes.
Quoted 5 lines
Also there are other, more serious problems with your approach:
- The inode map can consume a lot of memory, which is never released until unmount. Someone could prepare a disk full of zero-length files and run "find" on it to crash the machine.
I did consider this. I'm not sure my reasoning is correct, but because mount is a privileged operation I thought I might have some leeway.
Quoted 2 lines
- Inode numbers for the same disk become not stable across mounts (can be annoying if automounters are used).
I thought about this for a while too, but unlike you, I wasn't smart enough to think of anything that would break.
I'm convinced to try for a better solution.
Quoted 6 lines
However, the 4GB limit problem really needs to be solved. Here is another idea (sorry, no patch yet): instead of the byte offset of the directory entry, use its sector number and index in the sector.
isofs_read_inode()then would need to read the sector and skip the specified number of directory entries to find the needed one.
Quoted 6 lines
The minimum possible size of an ISO9660 directory entry is 34 bytes (33 bytes for struct
iso_directory_record, 1 byte for file name). Therefore there cannot be more than 60 directory entries in a single CD-ROM sector (2048 bytes), so the number of a directory entry in the sector can fit into 6 bits. With 32-bit inode numbers this leaves 26 bits for the sector number, which is enough for up to 128GB.
That's clever. I think I can get this done quickly (if you haven't already started).
Paul Serice
fs/isofs/dir.c +12 -2
@@ -115,6 +115,7 @@ static int do_isofs_readdir(struct inodechar *p =NULL ;/* Quiet GCC */ struct iso_directory_record *de;struct isofs_sb_info *sbi =ISOFS_SB (inode->i_sb); +struct isofs_inode_map *inode_map = &sbi->s_inode_map; offset = filp->f_pos & (bufsize -1 ); block = filp->f_pos >> bufbits;
@@ -129,8 +130,17 @@ static int do_isofs_readdir(struct inode } de = (struct iso_directory_record *) (bh->b_data + offset); -if (first_de) - inode_number = (bh->b_blocknr << bufbits) + offset; +if (first_de) { + inode_number = +isofs_inode_map_index_by_phy (inode_map, + bh->b_blocknr, + offset); +if (inode_number ==0 ) { +printk (KERN_ERR "iso9660: Unable to index " +"inode!\n" ); +return 0 ; + } + } de_len = *(unsigned char *) de;
fs/isofs/inode.c +69 -33
241 lines not on this page · raw message
fs/isofs/inode_map.c +413 -0
413 lines not on this page · raw message
fs/isofs/Makefile +1 -1
@@ -4,7 +4,7 @@ obj-$(CONFIG_ISO9660_FS) += isofs.o -isofs-objs-y := namei.o inode.o dir.o util.o rock.o +isofs-objs-y := namei.o inode.o inode_map.o dir.o util.o rock.o isofs-objs-$(CONFIG_JOLIET) += joliet.o isofs-objs-$(CONFIG_ZISOFS) += compress.o isofs-objs := $(isofs-objs-y)
fs/isofs/namei.c +8 -1
@@ -65,6 +65,8 @@ isofs_find_entry(struct inode *dir, struunsigned long bufsize =ISOFS_BUFFER_SIZE (dir);unsigned char bufbits =ISOFS_BUFFER_BITS (dir);unsigned int block, f_pos, offset; +sector_t abs_block; +unsigned long abs_block_offset;struct buffer_head * bh =NULL ;struct isofs_sb_info *sbi =ISOFS_SB (dir->i_sb);
@@ -87,7 +89,8 @@ isofs_find_entry(struct inode *dir, stru } de = (struct iso_directory_record *) (bh->b_data + offset); - inode_number = (bh->b_blocknr << bufbits) + offset; + abs_block = bh->b_blocknr; + abs_block_offset = offset; de_len = *(unsigned char *) de;if (!de_len) {
@@ -151,6 +154,10 @@ isofs_find_entry(struct inode *dir, stru }if (match) {if (bh)brelse (bh); + inode_number = +isofs_inode_map_index_by_phy (&sbi->s_inode_map, + abs_block, + abs_block_offset);return inode_number; } }
fs/isofs/rock.c +26 -10
@@ -164,6 +164,8 @@ int parse_rock_ridge_inode_internal(struint len;unsigned char * chr;int symlink_len =0 ; +unsigned long inode_number =0 ; +struct isofs_inode_map *map = &(ISOFS_SB (inode->i_sb)->s_inode_map);CONTINUE_DECLS ;if (!ISOFS_SB (inode->i_sb)->s_rock)return 0 ;
@@ -306,9 +308,16 @@ int parse_rock_ridge_inode_internal(stru goto out;case SIG ('C' ,'L' ):ISOFS_I (inode)->i_first_extent =isonum_733 (rr->u.CL.location); - reloc =iget (inode->i_sb, - (ISOFS_I (inode)->i_first_extent << -ISOFS_SB (inode->i_sb)->s_log_zone_size)); + inode_number = +isofs_inode_map_index_by_phy (map, +ISOFS_I (inode)->i_first_extent, +0 ); +if (inode_number ==0 ) { +printk (KERN_ERR "Inode number overflowed while " +"parsing rock ridge inode.\n" ); + goto out; + } + reloc =iget (inode->i_sb, inode_number);if (!reloc) goto out; inode->i_mode = reloc->i_mode;
@@ -447,37 +456,43 @@ int parse_rock_ridge_inode(struct iso_distatic int rock_ridge_symlink_readpage (struct file *file,struct page *page) {struct inode *inode = page->mapping->host; +struct isofs_sb_info * sbi =ISOFS_SB (inode->i_sb);char *link =kmap (page);unsigned long bufsize =ISOFS_BUFFER_SIZE (inode); -unsigned char bufbits =ISOFS_BUFFER_BITS (inode);struct buffer_head *bh;char *rpnt = link;unsigned char *pnt;struct iso_directory_record *raw_inode;CONTINUE_DECLS ; -int block; +struct isofs_inode_map *map; +struct isofs_inode_map_node *map_node;int sig;int len;unsigned char *chr;struct rock_ridge *rr; -if (!ISOFS_SB (inode->i_sb)->s_rock) +if (!sbi->s_rock)panic ("Cannot have symlink with high sierra variant of iso filesystem\n" ); - block = inode->i_ino >> bufbits; + map = &sbi->s_inode_map; + map_node =isofs_inode_map_find_by_ino (map, inode->i_ino); +if (map_node ==NULL ) { + goto out_eio; + } +lock_kernel (); - bh =sb_bread (inode->i_sb, block); + bh =sb_bread (inode->i_sb, map_node->block);if (!bh) goto out_noread; - pnt = (unsigned char *) bh->b_data + (inode->i_ino & (bufsize -1 )); + pnt = (unsigned char *) bh->b_data + map_node->block_offset; raw_inode = (struct iso_directory_record *) pnt;/* * If we go past the end of the buffer, there is some sort of error. */ -if ((inode->i_ino & (bufsize -1 )) + *pnt > bufsize) +if (map_node->block_offset + *pnt > bufsize) goto out_bad_span;/* Now test for possible Rock Ridge extensions which will override
@@ -543,6 +558,7 @@ static int rock_ridge_symlink_readpage(sSetPageError (page);kunmap (page);unlock_page (page); + out_eio:return -EIO ; }
include/linux/iso_fs.h +38 -0
@@ -2,6 +2,7 @@#ifndef _ISOFS_FS_H#define _ISOFS_FS_H +#include <linux/rbtree.h> #include <linux/types.h> /* * The isofs filesystem constants/structures
@@ -155,6 +156,18 @@ struct iso_directory_record {char name [0 ]; } __attribute__((packed)); + +/* Inode Map Node Cache -- Explicitly map 32-bit inode numbers to the +* arbitrary block and offset of an ISO 9660 Directory Record. */ +struct isofs_inode_map_node { +unsigned long inode_number; +sector_t block; +unsigned long block_offset; +struct rb_node node_for_phy; +struct rb_node node_for_ino; +}; + +#define ISOFS_BLOCK_BITS 11#define ISOFS_BLOCK_SIZE 2048
@@ -231,6 +244,31 @@ extern struct dentry *isofs_lookup(strucextern struct buffer_head *isofs_bread (struct inode *,sector_t );extern int isofs_get_blocks (struct inode *,sector_t ,struct buffer_head **,unsigned long ); +extern int init_inode_map_node_cache (void ); +extern void destroy_inode_map_node_cache (void ); + +extern void isofs_inode_map_init (struct isofs_inode_map *map); + +extern void isofs_inode_map_set_root_inode (struct isofs_inode_map *map, +sector_t block, +unsigned long block_offset); + +extern struct isofs_inode_map_node * +isofs_inode_map_find_by_ino (struct isofs_inode_map *map, +unsigned long inode_number); + +extern struct isofs_inode_map_node * +isofs_inode_map_find_by_phy (struct isofs_inode_map *map, +sector_t block, +unsigned long block_offset); + +extern unsigned long +isofs_inode_map_index_by_phy (struct isofs_inode_map *map, +sector_t block, +unsigned long block_offset); + +extern void isofs_inode_map_destroy (struct isofs_inode_map *map); +extern struct inode_operations isofs_dir_inode_operations;extern struct file_operations isofs_dir_operations;extern struct address_space_operations isofs_symlink_aops;
include/linux/iso_fs_sb.h +11 -0
@@ -1,6 +1,15 @@#ifndef _ISOFS_FS_SB#define _ISOFS_FS_SB + +struct isofs_inode_map { +struct rw_semaphore lock; +unsigned long next_inode_number; +struct rb_root index_by_phy; +struct rb_root index_by_ino; +}; + +/* * iso9660 super-block data in memory */
@@ -29,6 +38,8 @@ struct isofs_sb_info {gid_t s_gid;uid_t s_uid;struct nls_table *s_nls_iocharset;/* Native language support table */ + + struct isofs_inode_map s_inode_map; };#endif