overlayfs patches for ovl_copy_up & ovl_rename
Subsystems (2)
Hello,
Here are some comments obtained from past days testing overlayfs in Live systems,
- to improve copy_up performance I have redesigned the procedure, it traverses the tree looking for the topmost lower dentry, anotating the concerning dentries and locking them; after, for each dentry from top to bottom, it copies the inodes and releases the dentry.
- a repeated line is found in overlayfs.h
- a few days ago we talked about an error when repetitive sed instructions were executed, also it happens in other processes, like mantaining list of installed files for actual packages,
that is solved in ovl_rename issuing a dget for the old-dentry and parent dentries. (This code is inspired on rename.c from unionfs)
Now Overlayfs is working properly on my desktop Live system, it is fast and reliable,
Thanks,
Jordi Pujol
Live never ending Tale
GNU/Linux Live forever!
http://livenet.selfip.com
fs/overlayfs/copy_up.c +51 -50
@@ -315,69 +315,70 @@ out_free_link:return err; } -int ovl_copy_up (struct dentry *dentry) +/* Optimize by not copying up the file first and truncating later */ +int ovl_copy_up_truncate (struct dentry *dentry,loff_t size) {int err; +struct dentry *next; +struct dentry *parent; +struct path lowerpath; +struct kstat stat; +enum ovl_path_type type; +struct dentry **adentry; +struct dentry **adentryr; +int sc, st, i; err =0 ; -while (!err) { -struct dentry *next; -struct dentry *parent; -struct path lowerpath; -struct kstat stat; -enum ovl_path_type type =ovl_path_type (dentry); - -if (type !=OVL_PATH_LOWER ) -break ; - next =dget (dentry); -/* find the topmost dentry not yet copied up */ -for (;;) { - parent =dget_parent (next); - - type =ovl_path_type (parent); -if (type !=OVL_PATH_LOWER ) +/* look for dentries that are pending to copy */ + st =5 ; + adentry =kmalloc (st *sizeof (struct dentry *),GFP_KERNEL ); +if (!adentry) +return -ENOMEM ; + + next = dentry; +for (sc =0 ; ; sc++) { +if (sc >= st) { + st +=5 ; + adentryr =krealloc (adentry, st *sizeof (struct dentry *),GFP_KERNEL ); +if (!adentryr) { + err = -ENOMEM ;break ; - -dput (next); - next = parent; + } + adentry = adentryr; } -ovl_path_lower (next, &lowerpath); - err =vfs_getattr (lowerpath.mnt, lowerpath.dentry, &stat); -if (!err) - err =ovl_copy_up_one (parent, next, &lowerpath, &stat); + adentry[sc] =dget (next); + type =ovl_path_type (adentry[sc]); +if (type !=OVL_PATH_LOWER ) { +dput (adentry[sc]); +break ; + } + next = adentry[sc]->d_parent; + } -dput (parent); -dput (next); +/* copy the dentries */ +for (i = sc -1 ; i >=0 ; i--) { +if (!err) { + parent =dget_parent (adentry[i]); +ovl_path_lower (adentry[i], &lowerpath); + err =vfs_getattr (lowerpath.mnt, lowerpath.dentry, &stat); +if (!err) { +if (i ==0 && size >=0 && size < stat.size) + stat.size = size; + err =ovl_copy_up_one (parent, adentry[i], &lowerpath, &stat); + } +dput (parent); + } +dput (adentry[i]); } +kfree (adentry); +return err; } -/* Optimize by not copying up the file first and truncating later */ -int ovl_copy_up_truncate (struct dentry *dentry,loff_t size) +int ovl_copy_up (struct dentry *dentry) { -int err; -struct kstat stat; -struct path lowerpath; -struct dentry *parent =dget_parent (dentry); - - err =ovl_copy_up (parent); -if (err) - goto out_dput_parent; - -ovl_path_lower (dentry, &lowerpath); - err =vfs_getattr (lowerpath.mnt, lowerpath.dentry, &stat); -if (err) - goto out_dput_parent; - -if (size < stat.size) - stat.size = size; - - err =ovl_copy_up_one (parent, dentry, &lowerpath, &stat); - -out_dput_parent: -dput (parent); -return err; +return ovl_copy_up_truncate (dentry,-1LL ); }
fs/overlayfs/dir.c +14 -3
@@ -512,12 +512,17 @@ static int ovl_rename(struct inode *oldd old_upperdir =ovl_dentry_upper (old->d_parent); new_upperdir =ovl_dentry_upper (new->d_parent); + (void )dget (old_upperdir); + (void )dget (new_upperdir); + trap =lock_rename (new_upperdir, old_upperdir); olddentry =ovl_dentry_upper (old); - newdentry =ovl_dentry_upper (new); -if (newdentry) { -dget (newdentry); + (void )dget (olddentry); + +if (!ovl_is_whiteout (new) && + (newdentry =ovl_dentry_upper (new))) { + (void )dget (newdentry); }else { new_create = true; newdentry =ovl_lookup_create (new_upperdir, new);
@@ -570,7 +575,13 @@ static int ovl_rename(struct inode *oldd out_dput:dput (newdentry); out_unlock: +dput (olddentry); +unlock_rename (new_upperdir, old_upperdir); + +dput (old_upperdir); +dput (new_upperdir); +return err; }
fs/overlayfs/overlayfs.h +0 -1
@@ -33,7 +33,6 @@void ovl_dentry_set_opaque (struct dentry *dentry,bool opaque);bool ovl_is_whiteout (struct dentry *dentry);void ovl_dentry_update (struct dentry *dentry,struct dentry *upperdentry); -void ovl_dentry_set_opaque (struct dentry *dentry,bool opaque);int ovl_do_lookup (struct dentry *dentry); struct dentry *ovl_upper_create(struct dentry *upperdir, struct dentry *dentry,
Miklos Szeredi
maintainerHi Jordi,
Quoted 4 lines · Jordi Pujol
- to improve
copy_upperformance I have redesigned the procedure, it traverses the tree looking for the topmost lower dentry, anotating the concerning dentries and locking them; after, for each dentry from top to bottom, it copies the inodes and releases the dentry.
Looks good, but I'm only willing to look at performance issues after the bugs have been fixed.
Quoted 1 line
- a repeated line is found in
overlayfs.h
Yeah, patch applied. Thanks.
Quoted 10 lines
- a few days ago we talked about an error when repetitive sed instructions were executed, also it happens in other processes, like mantaining list of installed files for actual packages,
that is solved inovl_renameissuing a dget for the old-dentry and parent dentries. (This code is inspired onrename.cfrom unionfs)
fs/overlayfs/dir.c static int ovl_rename(struct inode *oldd · line 517
Quoted 6 lines
fs/overlayfs/dir.c +3
-0@@ -512,12 +512,17 @@ static int ovl_rename(struct inode *oldd old_upperdir =ovl_dentry_upper (old->d_parent); new_upperdir =ovl_dentry_upper (new->d_parent); + (void )dget (old_upperdir); + (void )dget (new_upperdir); +
This should not be necessary. vfs_rename() locks old and new parents which means taking an extra ref is totally superfluous. If this does make some difference then there's something very sinister going on an we need to investigate further.
fs/overlayfs/dir.c static int ovl_rename(struct inode *oldd · line 521
Quoted 7 lines
fs/overlayfs/dir.c +1
-3trap =lock_rename (new_upperdir, old_upperdir); olddentry =ovl_dentry_upper (old); - newdentry =ovl_dentry_upper (new); -if (newdentry) { -dget (newdentry); + (void )dget (olddentry);
Again, while we have a ref on old, we should have a ref on olddentry as well, so this shouldn't make a difference.
fs/overlayfs/dir.c static int ovl_rename(struct inode *oldd · line 525
Quoted 4 lines
fs/overlayfs/dir.c +4
-0+ +if (!ovl_is_whiteout (new) && + (newdentry =ovl_dentry_upper (new))) { + (void )dget (newdentry);
This shouldn't make a difference either, ovl_dentry_upper() will return NULL for whiteouts.
Can you try introducing the above changes one by one to see which one, if any of them, makes a difference to your test case?
Thanks,
Miklos
Hello,
A Dijous 05 Maig 2011 11:06:08, Miklos Szeredi va escriure:
Quoted 2 lines
Looks good, but I'm only willing to look at performance issues after the bugs have been fixed.
This also locks the whole path before doing the copy, that could be better also, give it a try,
Pass me the list of pending bugs and I will help to debug, if I can
Quoted 17 lines
fs/overlayfs/dir.c static int ovl_rename(struct inode *oldd · line 517
Quoted 7 lines
fs/overlayfs/dir.c +3
-0old_upperdir =ovl_dentry_upper (old->d_parent); new_upperdir =ovl_dentry_upper (new->d_parent); + (void )dget (old_upperdir); + (void )dget (new_upperdir); +This should not be necessary.
vfs_rename()locks old and new parents which means taking an extra ref is totally superfluous. If this does make some difference then there's something very sinister going on an we need to investigate further.Quoted 4 lines
fs/overlayfs/dir.c +0
-0trap =lock_rename (new_upperdir, old_upperdir); olddentry =ovl_dentry_upper (old);
Quoted 9 lines
fs/overlayfs/dir.c static int ovl_rename(struct inode *oldd · line 521
Quoted 4 lines
fs/overlayfs/dir.c +1
-3- newdentry =ovl_dentry_upper (new); -if (newdentry) { -dget (newdentry); + (void )dget (olddentry);Again, while we have a ref on old, we should have a ref on olddentry as well, so this shouldn't make a difference.
Quoted 1 line
+
Maybe the key action is to take this reference as early as possible; consider that this really works, and it looks the same that is done by unionfs.
Quoted 6 lines
fs/overlayfs/dir.c static int ovl_rename(struct inode *oldd · line 525
Quoted 3 lines
fs/overlayfs/dir.c +3
-0+if (!ovl_is_whiteout (new) && + (newdentry =ovl_dentry_upper (new))) { + (void )dget (newdentry);This shouldn't make a difference either,
ovl_dentry_upper()will return NULL for whiteouts.
Agree, this has not any impact on the final behaviour,
Quoted 3 lines
Can you try introducing the above changes one by one to see which one, if any of them, makes a difference to your test case?
In my tests, locking the parent directories makes the right thing, but maybe there are other cases that should be considered...
Thanks,
Jordi Pujol
Live never ending Tale
GNU/Linux Live forever!
http://livenet.selfip.com
Miklos Szeredi
maintainerQuoted 9 lines · Jordi Pujol
Hello,
A Dijous 05 Maig 2011 11:06:08, Miklos Szeredi va escriure:
Quoted 2 lines
Looks good, but I'm only willing to look at performance issues after the bugs have been fixed.
This also locks the whole path before doing the copy, that could be better also, give it a try,
Pass me the list of pending bugs and I will help to debug, if I can
Currently the only pending bug is the one you reported.
Quoted 25 lines
Quoted 25 lines
Review
Quoted 7 lines
(quoted diff) +3
-0old_upperdir = ovl_dentry_upper(old->d_parent); new_upperdir = ovl_dentry_upper(new->d_parent); + (void) dget(old_upperdir); + (void) dget(new_upperdir); +This should not be necessary.
vfs_rename()locks old and new parents which means taking an extra ref is totally superfluous. If this does make some difference then there's something very sinister going on an we need to investigate further.Review
Quoted 8 lines
(quoted diff) +1
-3trap = lock_rename(new_upperdir, old_upperdir); olddentry = ovl_dentry_upper(old); - newdentry = ovl_dentry_upper(new); - if (newdentry) { - dget(newdentry); + (void) dget(olddentry);Again, while we have a ref on old, we should have a ref on olddentry as well, so this shouldn't make a difference.
Quoted 5 lines
Quoted 1 line
Quoted 1 line
+
Maybe the key action is to take this reference as early as possible; consider that this really works, and it looks the same that is done by unionfs.
The fact that the same is done by unionfs doesn't make it right and it also doesn't explain why unionfs does this.
Quoted 8 lines
Quoted 7 lines
Review
Quoted 4 lines
(quoted diff) +3
-0+ if (!ovl_is_whiteout(new) && + (newdentry = ovl_dentry_upper(new))) { + (void) dget(newdentry);This shouldn't make a difference either,
ovl_dentry_upper()will return NULL for whiteouts.
Quoted 8 lines
Agree, this has not any impact on the final behaviour,
Quoted 3 lines
Can you try introducing the above changes one by one to see which one, if any of them, makes a difference to your test case?
In my tests, locking the parent directories makes the right thing, but maybe there are other cases that should be considered...
Can you please try the following patch then? Does any of the WARN_ONs trigger when you observe the bad behavior?
Thanks,
Miklos
--- fs/overlayfs/dir.c | 10 ++++++++++ 1 file changed, 10 insertions(+)
fs/overlayfs/dir.c +10 -0
@@ -512,6 +512,9 @@ static int ovl_rename(struct inode *oldd old_upperdir =ovl_dentry_upper (old->d_parent); new_upperdir =ovl_dentry_upper (new->d_parent); + (void )dget (old_upperdir); + (void )dget (new_upperdir); + trap =lock_rename (new_upperdir, old_upperdir); olddentry =ovl_dentry_upper (old);
@@ -571,6 +574,13 @@ static int ovl_rename(struct inode *oldddput (newdentry); out_unlock:unlock_rename (new_upperdir, old_upperdir); + +WARN_ON (old_upperdir->d_count ==1 ); +WARN_ON (new_upperdir->d_count ==1 ); + +dput (old_upperdir); +dput (new_upperdir); +return err; }
Miklos Szeredi
maintainerQuoted 7 lines · Miklos Szeredi
Quoted 3 lines · Jordi Pujol
In my tests, locking the parent directories makes the right thing, but maybe there are other cases that should be considered...
Can you please try the following patch then? Does any of the
WARN_ONstrigger when you observe the bad behavior?
Jordi, what's up with this?
Could you please test the .v8 codebase with this single patch applied?
Thanks,
Miklos
Quoted 32 lines
--- fs/overlayfs/dir.c | 10 ++++++++++ 1 file changed, 10 insertions(+)Index:
linux-2.6/fs/overlayfs/dir.c
===================================================================fs/overlayfs/dir.c +10
-0@@ -512,6 +512,9 @@ static int ovl_rename(struct inode *oldd old_upperdir =ovl_dentry_upper (old->d_parent); new_upperdir =ovl_dentry_upper (new->d_parent); + (void )dget (old_upperdir); + (void )dget (new_upperdir); + trap =lock_rename (new_upperdir, old_upperdir); olddentry =ovl_dentry_upper (old);@@ -571,6 +574,13 @@ static int ovl_rename(struct inode *oldddput (newdentry); out_unlock:unlock_rename (new_upperdir, old_upperdir); + +WARN_ON (old_upperdir->d_count ==1 ); +WARN_ON (new_upperdir->d_count ==1 ); + +dput (old_upperdir); +dput (new_upperdir); +return err; }
A Dimarts 17 Maig 2011 10:43:39, vàreu escriure:
Quoted 4 lines
Miklos Szeredi <[email protected]> writes:
Jordi, what's up with this?Could you please test the .v8 codebase with this single patch applied?
Hello,
I did so, but the system has never logged any dump from overlayfs, this condition does not occur.
But I am so confused about the meaning of this test,
I have seen that other union filesystems will lock dentries for directory operations and super_blocks for read/write operations, and overlayfs does not.
Thanks,
Jordi Pujol
Live never ending Tale
GNU/Linux Live forever!
http://livenet.selfip.com
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Miklos Szeredi
maintainerQuoted 10 lines · Jordi Pujol
A Dimarts 17 Maig 2011 10:43:39, vàreu escriure:
Quoted 4 lines
Miklos Szeredi <[email protected]> writes:
Jordi, what's up with this?Could you please test the .v8 codebase with this single patch applied?
Hello,
I did so, but the system has never logged any dump from overlayfs, this condition does not occur.
Which is expected.
Quoted 3 lines
But I am so confused about the meaning of this test, I have seen that other union filesystems will lock dentries for directory operations and
super_blocksfor read/write operations, and overlayfs does not.
I'm not sure what other filesystems do, but in overlayfs the upper/lower dentry references do not go away until the overlayfs dentry is freed.
Jordi, what about the original bug you reported:
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
- a few days ago we talked about an error when repetitive sed instructions were executed, also it happens in other processes, like mantaining list of installed files for actual packages,
that is solved inovl_renameissuing a dget for the old-dentry and parent dentries. (This code is inspired onrename.cfrom unionfs)
Is this still happening? Does the debug patch make any difference?
Thanks,
Miklos
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
A Dimarts 17 Maig 2011 20:07:13, vàreu escriure:
Quoted 8 lines
Jordi, what about the original bug you reported:
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
- a few days ago we talked about an error when repetitive sed instructions were executed, also it happens in other processes, like mantaining list of installed files for actual packages, that is solved in
ovl_renameissuing a dget for the old-dentry and parent dentries. (This code is inspired onrename.cfrom unionfs)Is this still happening? Does the debug patch make any difference?
my patch adds some dget/dput operations that minimize the problem, it occurs less times, but it happens.
I suppose that those extra operations add an execution delay before processing the main routine;
a deduction is that locking dentries is required to synchronize a rename to not be executed until another rename operation has finished.
As an example of code that looks like that of unionfs, (this is not tested)
struct ovl_entry {
struct dentry *__upperdentry;
struct dentry *lowerdentry;
+ struct mutex d_mutex;
union {
struct {
u64 version;
bool opaque;
};
struct rcu_head rcu;
};
};static struct ovl_entry *ovl_alloc_entry(void)
{
struct ovl_entry *oe;
oe = kzalloc(sizeof(struct ovl_entry), GFP_KERNEL);
mutex_init(&oe->d_mutex);
return oe;
}void ovl_lock_dentry(struct dentry *dentry)
{
struct ovl_entry *oe = dentry->d_fsdata;
mutex_lock(&oe->d_mutex);
}void ovl_unlock_dentry(struct dentry *dentry)
{
struct ovl_entry *oe = dentry->d_fsdata;
mutex_unlock(&oe->d_mutex);
}static int ovl_rename(struct inode *olddir, struct dentry *old,
struct inode *newdir, struct dentry *new)
{
int err;...........
+ ovl_lock_dentry(old);
err = ovl_copy_up(old); ........... + ovl_unlock_dentry(old); return err; }
consider this idea,
Thanks,
Jordi Pujol
Live never ending Tale
GNU/Linux Live forever!
http://livenet.selfip.com
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Miklos Szeredi
maintainerQuoted 8 lines · Jordi Pujol
A Dimarts 17 Maig 2011 20:07:13, vàreu escriure:
Quoted 7 lines
Jordi, what about the original bug you reported:
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
- a few days ago we talked about an error when repetitive sed instructions were executed, also it happens in other processes, like mantaining list of installed files for actual packages, that is solved in
ovl_renameissuing a dget for the old-dentry and parent dentries. (This code is inspired onrename.cfrom unionfs)
Quoted 5 lines
Quoted 1 line
Is this still happening? Does the debug patch make any difference?
my patch adds some dget/dput operations that minimize the problem, it occurs less times, but it happens. I suppose that those extra operations add an execution delay before processing the main routine;
Probably timing related, yes.
Quoted 2 lines
a deduction is that locking dentries is required to synchronize a rename to not be executed until another rename operation has finished.
Already done by the VFS. The locks used actually provide stronger guarantees than your code. See
Documentation/filesystems/directory-locking
for all the gory details.
So, is there a minimal test case that I can try to replicate the bad behavior?
Thanks,
Miklos
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
A Dimecres 18 Maig 2011 10:42:53, Miklos Szeredi va escriure:
Quoted 3 lines
Documentation/filesystems/directory-lockingfor all the gory details.
ok, i'll look this,
Quoted 2 lines
So, is there a minimal test case that I can try to replicate the bad behavior?
it's an occasional error, occurs when the system is occupied processing diverse tasks at the same time,
for example when startpar is running in parallel the initialization of system daemons, this is detected because some changes done by the same daemon program are lost, (the case of consecutive sed instructions)
or when doing a big upgrade of the system packages, some files are lost.
I believe than this could be reproduced in an stress test, running a process that creates a lot of files around, deletes them and moves from tmpfs to overlays; and the system must be very busy, because this one and the others processeses are working hard.
Thanks,
Jordi Pujol
Live never ending Tale
GNU/Linux Live forever!
http://livenet.selfip.com
Miklos Szeredi
maintainerQuoted 7 lines · Jordi Pujol
A Dimecres 18 Maig 2011 10:42:53, Miklos Szeredi va escriure:
Quoted 3 lines
Documentation/filesystems/directory-lockingfor all the gory details.
ok, i'll look this,
Quoted 8 lines
Quoted 2 lines
So, is there a minimal test case that I can try to replicate the bad behavior?
it's an occasional error, occurs when the system is occupied processing diverse tasks at the same time, for example when startpar is running in parallel the initialization of system daemons, this is detected because some changes done by the same daemon program are lost, (the case of consecutive sed instructions)
This file, which modifications are lost, is this on the upper filesystem, the lower filesystem or both, before the modification? And after the modification?
Quoted 2 lines
or when doing a big
upgrade of the system packages, some files are lost.
How are they lost? They don't exist when they should? What's the contents of the upper and lower filesystems?
Thanks,
Miklos
A Dijous 19 Maig 2011 10:49:31, Miklos Szeredi va escriure:
Quoted 9 lines
This file, which modifications are lost, is this on the upper filesystem, the lower filesystem or both, before the modification? And after the modification?
Quoted 2 lines
or when doing a big
upgrade of the system packages, some files are lost.How are they lost? They don't exist when they should? What's the contents of the upper and lower filesystems?
in the sed case, the file exists in the lower filesystem and is copied to tmpfs, removed and moved from tmpfs to the original file; several times the same operations are done.
in the package upgrade, the file does not exists, it is created on tmpfs and moved to the final overlayfs directory; after we can check that it does not exist there.
As I thinked previously, the common operation that appears in both processes and maybe the cause of the problem is move from another filesystem to overlayfs.
Thanks,
Jordi Pujol
Live never ending Tale
GNU/Linux Live forever!
http://livenet.selfip.com
Miklos Szeredi
maintainerQuoted 8 lines · Jordi Pujol
A Dijous 19 Maig 2011 10:49:31, Miklos Szeredi va escriure:
Quoted 7 lines
This file, which modifications are lost, is this on the upper filesystem, the lower filesystem or both, before the modification? And after the modification?
Quoted 2 lines
or when doing a big
upgrade of the system packages, some files are lost.
Quoted 6 lines
Quoted 2 lines
How are they lost? They don't exist when they should? What's the contents of the upper and lower filesystems?
in the sed case, the file exists in the lower filesystem and is copied to tmpfs, removed and moved from tmpfs to the original file; several times the same operations are done.
Thanks for the info.
So what you see as the end result is the same as the original lower file? Or there's a file on the upper filesystem which contains old data?
Quoted 3 lines
in the package upgrade, the file
does not exists, it is created on tmpfs and moved to the final overlayfs directory; after we can check that it does not exist there.
Does it exist on the upper filesystem?
Quoted 3 lines
As I thinked previously, the common operation that appears in both processes and maybe the cause of the problem is move from another filesystem to overlayfs.
"mv" between two filesystems is actually impelemented as copy+remove.
Thanks,
Miklos
A Dijous 19 Maig 2011 11:52:46, vàreu escriure:
Quoted 1 line
Thanks for the info.
welcome,
Quoted 2 lines
So what you see as the end result is the same as the original lower file? Or there's a file on the upper filesystem which contains old data?
it does consecutive changes in upperdir, one of the intermediate changes is not done,
like if a process is using a file while a previous process has not yet finished to replace it.
Quoted 7 lines
Quoted 5 lines
in the package upgrade, the file
does not exists, it is created on tmpfs and moved to the final overlayfs directory; after we can check that it does not exist there.
Does it exist on the upper filesystem?
no, neither.
Quoted 1 line
"mv" between two filesystems is actually impelemented as copy+remove.
therefore is that my patch is not modifying the right procedure,
Thanks,
Jordi Pujol
Live never ending Tale
GNU/Linux Live forever!
http://livenet.selfip.com
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
A Dijous 19 Maig 2011 11:24:19, Jordi Pujol va escriure:
Quoted 2 lines
processes and maybe the cause of the problem is move from another filesystem to overlayfs.
observing a bit more about this, this is not a synchronization problem because in this case the file has not been copied to overlayfs,
maybe there is some condition that does not permit the copy of that file.
Thanks,
Jordi Pujol
Live never ending Tale
GNU/Linux Live forever!
http://livenet.selfip.com
Miklos Szeredi
maintainerQuoted 7 lines · Jordi Pujol
A Dijous 19 Maig 2011 11:24:19, Jordi Pujol va escriure:
Quoted 2 lines
processes and maybe the cause of the problem is move from another filesystem to overlayfs.
observing a bit more about this, this is not a synchronization problem because in this case the file has not been copied to overlayfs, maybe there is some condition that does not permit the copy of that file.
Could be, but in that case the script should be failing with an error indicating what went wrong.
The fix for the bug discovered by Erez just went into the .v9 tree. I don't think it has any relevance to the problem you are observing, but just to be sure, could you please test this update if it makes any difference?
Thanks,
Miklos
A Dilluns 23 Maig 2011 11:21:20, Miklos Szeredi va escriure:
Quoted 4 lines
The fix for the bug discovered by Erez just went into the .v9 tree. I don't think it has any relevance to the problem you are observing, but just to be sure, could you please test this update if it makes any difference?
I have tested it, there is a big probability that now the sed case works, running it ten times, had failed zero.
next days I will test big updates of packages.
Thanks,
Jordi Pujol
Live never ending Tale
GNU/Linux Live forever!
http://livenet.selfip.com
Hello,
that is a list of some errors found when doiing a remaster that uses overlayfs as a filesystem modification layer,
After doing a big package upgrade we note that the rootdir has some errors, this is only a little percent of the total files contained in the filesystem, 44 files of 86173
overlayfs mounted on /tmp/live-net-remaster-pro1Wm/chroot type overlayfs options rw,relatime,lowerdir=/tmp/live-net-remaster- pro1Wm/image/00filesystem.squashfs,upperdir=/home/tmp.7P8ARuGANF
CASE 1: complete directory /usr/share/doc/xorg disappears, is in lowerdir, has not been modified, but rootdir does not show it,
# lowerdir # find /tmp/live-net-remaster-pro1Wm/image/00filesystem.squashfs/usr/share/doc/ -maxdepth 1 -name 'xorg*' /tmp/live-net-remaster-pro1Wm/image/00filesystem.squashfs/usr/share/doc/xorg
# upperdir # find /home/tmp.7P8ARuGANF/usr/share/doc/ -maxdepth 1 -name 'x*' /home/tmp.7P8ARuGANF/usr/share/doc/xserver-xorg-video-ati /home/tmp.7P8ARuGANF/usr/share/doc/xml-core /home/tmp.7P8ARuGANF/usr/share/doc/xserver-xorg-video-radeon (xorg is not listed)
# rootdir # find /tmp/live-net-remaster-pro1Wm/chroot/usr/share/doc/ -maxdepth 1 -name 'xorg*' (not listed)
directory xorg contains a lot of files, not shown in rootdir.
5 root root 4096 /usr/share/doc/xorg
-rw-r--r-- 1 root root 7381 /usr/share/doc/xorg/asciidoc-xhtml11.css -rw-r--r-- 1 root root 4180 /usr/share/doc/xorg/asciidoc-xhtml11.js drwxr-xr-x 2 root root 4096 /usr/share/doc/xorg/faq -rw-r--r-- 1 root root 5163 /usr/share/doc/xorg/faq/general.html -rw-r--r-- 1 root root 2732 /usr/share/doc/xorg/faq/general.txt drwxr-xr-x 2 root root 4096 /usr/share/doc/xorg/howto -rw-r--r-- 1 root root 11521 /usr/share/doc/xorg/howto/build-mesa.html -rw-r--r-- 1 root root 2717 /usr/share/doc/xorg/howto/build-mesa.txt.gz -rw-r--r-- 1 root root 10758 /usr/share/doc/xorg/howto/configure-input.html -rw-r--r-- 1 root root 2891 /usr/share/doc/xorg/howto/configure-input.txt.gz -rw-r--r-- 1 root root 2994 /usr/share/doc/xorg/howto/report-bugs.html -rw-r--r-- 1 root root 1278 /usr/share/doc/xorg/howto/report-bugs.txt -rw-r--r-- 1 root root 6703 /usr/share/doc/xorg/howto/triage-bugs.html -rw-r--r-- 1 root root 3529 /usr/share/doc/xorg/howto/triage-bugs.txt -rw-r--r-- 1 root root 8145 /usr/share/doc/xorg/howto/use-gdb.html -rw-r--r-- 1 root root 1940 /usr/share/doc/xorg/howto/use-gdb.txt.gz -rw-r--r-- 1 root root 10678 /usr/share/doc/xorg/howto/use-xrandr.html -rw-r--r-- 1 root root 2615 /usr/share/doc/xorg/howto/use-xrandr.txt.gz -rw-r--r-- 1 root root 3349 /usr/share/doc/xorg/index.html -rw-r--r-- 1 root root 1341 /usr/share/doc/xorg/index.txt drwxr-xr-x 2 root root 4096 /usr/share/doc/xorg/reference
-rw-r--r-- 1 root root 12636 /usr/share/doc/xorg/reference/dependencies.html
-rw-r--r-- 1 root root 3415/usr/share/doc/xorg/reference/dependencies.txt.gz
-rw-r--r-- 1 root root 4690 /usr/share/doc/xorg/reference/squeeze-
backports.html
-rw-r--r-- 1 root root 2565 /usr/share/doc/xorg/reference/squeeze-
backports.txt
-rw-r--r-- 1 root root 3529 /usr/share/doc/xorg/reference/upstream-
contacts.html
-rw-r--r-- 1 root root 1392 /usr/share/doc/xorg/reference/upstream-
contacts.txt
-rw-r--r-- 1 root root 1953 /usr/share/doc/xorg/upstream-features.html
-rw-r--r-- 1 root root 556 /usr/share/doc/xorg/upstream-features.txt
-rw-r--r-- 1 root root 179 /usr/share/doc/xorg/xsf.css
-rw-r--r-- 1 root root 3914 /usr/share/doc/xorg/xsf.png
-rw-r--r-- 1 root root 11221 /usr/share/doc/xorg/xsf.svg
CASE 2: link /usr/share/man/man1/ucat.1.gz -> unp.1.gz also disappears, same obs. as before,
# lowerdir # find /tmp/live-net-remaster- pro1Wm/image/00filesystem.squashfs/usr/share/man/man1/ -maxdepth 1 -name 'ucat*' /tmp/live-net-remaster- pro1Wm/image/00filesystem.squashfs/usr/share/man/man1/ucat.1.gz
# upperdir # find /home/tmp.7P8ARuGANF/usr/share/man/man1 -maxdepth 1 -name 'ucat*' (not listed)
# rootdir # find /tmp/live-net-remaster-pro1Wm/chroot/usr/share/man/man1/ -maxdepth 1 - name 'ucat1*' (not listed)
CASE 3: file /var/lib/dpkg/info/krdc.postrm
has been modified, but is not in rootdir
# lowerdir # find /tmp/live-net-remaster- pro1Wm/image/00filesystem.squashfs/var/lib/dpkg/info/ -maxdepth 1 -name 'krdc.postrm*' /tmp/live-net-remaster- pro1Wm/image/00filesystem.squashfs/var/lib/dpkg/info/krdc.postrm ls -la /tmp/live-net-remaster- pro1Wm/image/00filesystem.squashfs/var/lib/dpkg/info/krdc.postrm -rwxr-xr-x 1 root root 132 2010-11-28 16:51 /tmp/live-net-remaster- pro1Wm/image/00filesystem.squashfs/var/lib/dpkg/info/krdc.postrm
# upperdir # find /home/tmp.7P8ARuGANF/var/lib/dpkg/info/ -maxdepth 1 -name 'krdc.postrm*' /home/tmp.7P8ARuGANF/var/lib/dpkg/info/krdc.postrm # ls -la /home/tmp.7P8ARuGANF/var/lib/dpkg/info/krdc.postrm -rwxr-xr-x 1 root root 132 2011-05-26 04:35 /home/tmp.7P8ARuGANF/var/lib/dpkg/info/krdc.postrm
# rootdir # find /tmp/live-net-remaster-pro1Wm/chroot/var/lib/dpkg/info/ -maxdepth 1 - name 'krdc.postrm*' (not listed)
CASE 4: file /var/lib/dpkg/info/libgraphicsmagick3.shlibs not modified, is not in rootdir
# lowerdir # find /tmp/live-net-remaster- pro1Wm/image/00filesystem.squashfs/var/lib/dpkg/info/ -maxdepth 1 -name 'libgraphicsmagick3.shlibs*' /tmp/live-net-remaster- pro1Wm/image/00filesystem.squashfs/var/lib/dpkg/info/libgraphicsmagick3.shlibs # ls -la /tmp/live-net-remaster- pro1Wm/image/00filesystem.squashfs/var/lib/dpkg/info/libgraphicsmagick3.shlibs -rw-r--r-- 1 root root 82 2011-05-03 20:30 /tmp/live-net-remaster- pro1Wm/image/00filesystem.squashfs/var/lib/dpkg/info/libgraphicsmagick3.shlibs
# upperdir # find /home/tmp.7P8ARuGANF/var/lib/dpkg/info/ -maxdepth 1 -name 'libgraphicsmagick3.shlibs*' (not listed)
# rootdir # find /tmp/live-net-remaster-pro1Wm/chroot/var/lib/dpkg/info/ -maxdepth 1 - name 'libgraphicsmagick3.shlibs*' (not listed)
some additional files about this test can be found in:
http://livenet.selfip.com/ftp/debian/overlayfs/ovl-test.tar.bz2
Thanks,
Jordi Pujol
Live never ending Tale
GNU/Linux Live forever!
http://livenet.selfip.com
Miklos Szeredi
maintainerHello Jordi,
Quoted 2 lines · Jordi Pujol
that is a list of some errors found when doiing a remaster that uses overlayfs as a filesystem modification layer,
Thanks for testing.
Quoted 10 lines
After doing a big package upgrade we note that the rootdir has some errors, this is only a little percent of the total files contained in the filesystem, 44 files of 86173
overlayfs mounted on
/tmp/live-net-remaster-pro1Wm/chroottype overlayfs options rw,relatime,lowerdir=/tmp/live-net-remaster- pro1Wm/image/00filesystem.squashfs,upperdir=/home/tmp.7P8ARuGANFCASE 1: complete directory
/usr/share/doc/xorgdisappears, is in lowerdir, has not been modified, but rootdir does not show it,
I see the common theme in all these cases: a file should be there in rootdir but isn't.
I also see that you are using 'find'. Does the file show if you are using 'ls'?
ls /tmp/live-net-remaster-pro1Wm/chroot/usr/share/doc/ | grep xorg ls -d /tmp/live-net-remaster-pro1Wm/chroot/usr/share/doc/xorg
Could you please do a strace of 'find'
strace -v -o /tmp/strace find /tmp/live-net-remaster-pro1Wm/chroot/usr/share/doc/ -maxdepth 1 -name 'xorg*'
And send the resulting trace in /tmp/strace?
Thanks,
Miklos
A Dijous 19 Maig 2011 09:47:16, Jordi Pujol va escriure:
the problem maybe moving files from another filesystem to overlayfs, now I'm observing that the process works fine if it only creates files on overlayfs
A kernel compilation on overlayfs that uses an ext4 upperdir, is finished successfully; and the execution time is similar that the time for the compilation executed directly over the ext4 fs
Thanks,
Jordi Pujol
Live never ending Tale
GNU/Linux Live forever!
http://livenet.selfip.com