xfs_mkfs: wipe old signatures from the device
PATCH 1/1 xfs_mkfs: wipe old signatures from the device
We should wipe off all the signatures from the device prior the file system creation. It is because because some file systems (btrfs) may still have their signatures on the device which can be confusing for userspace possibly resulting in the unmountable file system.
This patch adds a function which uses libblkid library to wipe all the signatures from the device.
If user disables libblkid library or does not have it installed this new feature will not be used. This case can be implemented separately.
--- mkfs/xfs_mkfs.c | 134 +++++++++++++++++++++++++++++++++++++++--------------- 1 files changed, 97 insertions(+), 37 deletions(-)
mkfs/xfs_mkfs.c +97 -37
@@ -286,18 +286,73 @@ calc_stripe_factors( }#ifdef ENABLE_BLKID +static int +wipe_signatures ( +char *device, +int dryrun) +{ +blkid_probe pr =NULL ; +int ret =0 ; +int fd; + + pr =blkid_new_probe_from_filename (device); +if (!pr) + goto out; + + fd =open (device,O_RDWR |O_CLOEXEC ); +if (fd <0 ) { + ret =-1 ; + goto out; + } + ret =blkid_probe_set_device (pr, fd,0 ,0 ); +if (ret) + goto out; + +/* No need to check return values we know that 'pr' is initialized */ +blkid_probe_enable_partitions (pr,1 ); +blkid_probe_enable_superblocks (pr,1 ); +blkid_probe_set_partitions_flags (pr,BLKID_PARTS_MAGIC ); +blkid_probe_set_superblocks_flags (pr, +BLKID_SUBLKS_MAGIC |BLKID_SUBLKS_TYPE ); + +while (1 ) { + ret =blkid_do_probe (pr); +if (ret !=0 ) +break ; + +/* Wipe off the signature */ + ret =blkid_do_wipe (pr, dryrun); +if (ret) +break ; + } +close (fd); + +out: +if (pr) +blkid_free_probe (pr); +if (ret <0 ) +fprintf (stderr, +_ ("%s: Cannot wipe off existing signatures\n" ), progname); +return (ret <0 ? ret :0 ); +} +/* * Check for existing filesystem or partition table on device. * Returns: * 1 for existing fs or partition * 0 for nothing found * -1 for internal error +* If wipe is set, we'll attempt to wipe of all the signatures from +* the device. In this case we never return 1 (existing fs or partition). */ static int check_overwrite ( -char *device) +char *device, +int wipe, +int dryrun) {const char *type; +const char *name;blkid_probe pr =NULL ;int ret;int fd;
@@ -325,40 +380,43 @@ check_overwrite(if (!pr) goto out; - ret =blkid_probe_enable_partitions (pr,1 ); -if (ret <0 ) - goto out; +/* No need to check return values we know that 'pr' is initialized */ +blkid_probe_enable_partitions (pr,1 ); +blkid_probe_enable_superblocks (pr,1 ); +blkid_probe_set_partitions_flags (pr,BLKID_PARTS_MAGIC ); +blkid_probe_set_superblocks_flags (pr, +BLKID_SUBLKS_MAGIC |BLKID_SUBLKS_TYPE ); - ret =blkid_do_fullprobe (pr); -if (ret <0 ) - goto out; + ret =0 ; +while (1 ) { +int retval =0 ; -/* -* Blkid returns 1 for nothing found and 0 when it finds a signature, -* but we want the exact opposite, so reverse the return value here. -* -* In addition print some useful diagnostics about what actually is -* on the device. -*/ -if (ret) { - ret =0 ; - goto out; - } + retval =blkid_do_probe (pr); +if (retval !=0 ) { + ret = retval; +break ; + } -if (!blkid_probe_lookup_value (pr,"TYPE" , &type,NULL )) { -fprintf (stderr, -_ ("%s: %s appears to contain an existing " -"filesystem (%s).\n" ), progname, device, type); - }else if (!blkid_probe_lookup_value (pr,"PTTYPE" , &type,NULL )) { -fprintf (stderr, -_ ("%s: %s appears to contain a partition " -"table (%s).\n" ), progname, device, type); - }else { -fprintf (stderr, -_ ("%s: %s appears to contain something weird " -"according to blkid\n" ), progname, device); +if (blkid_probe_lookup_value (pr,"TYPE" , &type,NULL ) ==0 ) + name ="filesystem" ; + +else if (blkid_probe_lookup_value (pr,"PTTYPE" , + &type,NULL ) ==0 ) + name ="partition table" ; +else +continue ; + +if (wipe) +fprintf (stderr,_ ("%s: Wiping off %s signature.\n" ), + progname, type); +else { +fprintf (stderr, +_ ("%s: %s appears to contain an existing " +"%s (%s).\n" ), progname, device, name, type); + ret =1 ; +continue ; + } } - ret =1 ; out:if (pr)
@@ -367,6 +425,8 @@ out:fprintf (stderr,_ ("%s: probe of %s failed, cannot detect " "existing filesystem.\n" ), progname, device); +if (ret ==1 && wipe) + ret =wipe_signatures (device, dryrun);return ret; }
@@ -464,7 +524,8 @@ static void get_topology(#else /* ENABLE_BLKID */ static int check_overwrite ( -char *device) +char *device, +int wipe) {char *type;
@@ -1939,16 +2000,15 @@ _("block size %d cannot be smaller than logical sector size %d\n"), xi.rtsize &= sector_mask; xi.logBBsize &= (__uint64_t)-1 << (MAX (lsectorlog,10 ) -BBSHIFT ); -if (!force_overwrite) { -if (check_overwrite (dfile) || -check_overwrite (logfile) || -check_overwrite (xi.rtname)) { +if (check_overwrite (dfile, force_overwrite, Nflag) || +check_overwrite (logfile, force_overwrite, Nflag) || +check_overwrite (xi.rtname, force_overwrite, Nflag)) +if (force_overwrite) {fprintf (stderr,_ ("%s: Use the -f option to force overwrite.\n" ), progname);exit (1 ); } - }if (discard) {discard_blocks (xi.ddev, xi.dsize);
Signature · 6 lines
1.7.7.6 _______________________________________________ xfs mailing list [email protected] http://oss.sgi.com/mailman/listinfo/xfs
Quoted 8 lines · Lukas Czerner
mkfs/xfs_mkfs.c +8
-0+wipe_signatures ( + char *device, + int dryrun) +{ +blkid_probe pr =NULL ; +int ret =0 ; +int fd; +
mkfs/xfs_mkfs.c calc_stripe_factors( · line 307
Quoted 10 lines
mkfs/xfs_mkfs.c +10
-0+ pr =blkid_new_probe_from_filename (device); +if (!pr) + goto out; + + fd =open (device,O_RDWR |O_CLOEXEC ); +if (fd <0 ) { + ret =-1 ; + goto out; + } + ret =blkid_probe_set_device (pr, fd,0 ,0 );
This does not make sense. The blkid_new_probe_from_filename() also opens (read-only) the device ;-)
You need:
pr = blkid_new_probe();
if (!pr)
goto out; fd = open(device, O_RDWR|O_CLOEXEC);
if (fd < 0) {
ret = -1;
goto out; }
ret = blkid_probe_set_device(pr, fd, 0, 0);
Maybe you also need something like
AC_CHECK_LIB(blkid, blkid_do_wipe, [have_blkidwipe=yes], [have_blkidwipe=no])
or so, because old libblkid versions does not contain wipe stuff. See also m4/package_blkid.m4 in xfsprogs where is already check for blkid topology stuff.
Karel
Signature · 7 lines
Karel Zak <[email protected]> http://karelzak.blogspot.com _______________________________________________ xfs mailing list [email protected] http://oss.sgi.com/mailman/listinfo/xfs
Quoted 20 lines · Karel Zak
Date: Tue, 12 Feb 2013 12:31:52 +0100
From: Karel Zak <[email protected]>
To: Lukas Czerner <[email protected]>
Cc: [email protected], [email protected]
Subject: Re: [PATCH]xfs_mkfs: wipe old signatures from the deviceQuoted 13 lines · Lukas Czerner
mkfs/xfs_mkfs.c +13
-0+wipe_signatures ( + char *device, + int dryrun) +{ +blkid_probe pr =NULL ; +int ret =0 ; +int fd; + + pr =blkid_new_probe_from_filename (device); +if (!pr) + goto out; + + fd =open (device,O_RDWR |O_CLOEXEC );
Quoted 8 lines
mkfs/xfs_mkfs.c calc_stripe_factors( · line 307
Quoted 5 lines
mkfs/xfs_mkfs.c +5
-0+if (fd <0 ) { + ret =-1 ; + goto out; + } + ret =blkid_probe_set_device (pr, fd,0 ,0 );This does not make sense. The
blkid_new_probe_from_filename()also opens (read-only) the device ;-)
Good to know thanks!
-Lukas
Quoted 17 lines
You need:
pr = blkid_new_probe(); if (!pr) goto out;fd = open(device, O_RDWR|O_CLOEXEC); if (fd < 0) { ret = -1; goto out; } ret = blkid_probe_set_device(pr, fd, 0, 0);Maybe you also need something like
Quoted 10 lines
AC_CHECK_LIB(blkid, blkid_do_wipe, [have_blkidwipe=yes], [have_blkidwipe=no])or so, because old libblkid versions does not contain wipe stuff. See also m4/package_blkid.m4 in xfsprogs where is already check for blkid topology stuff.
Karel
_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
Quoted 10 lines · Lukas Czerner
We should wipe off all the signatures from the device prior the file system creation. It is because because some file systems (btrfs) may still have their signatures on the device which can be confusing for userspace possibly resulting in the unmountable file system.
This patch adds a function which uses libblkid library to wipe all the signatures from the device.
If user disables libblkid library or does not have it installed this new feature will not be used. This case can be implemented separately.
mkfs.xfs already zeros the first part of the block device regardless of whether it is using libblkid or not:
....
/* amount (in bytes) we zero at the beginning and end of the device to
* remove traces of other filesystems, raid superblocks, etc. */
#define WHACK_SIZE (128 * 1024)
....
/*
* Zero out the beginning of the device, to obliterate any old
* filesystem signatures out there. This should take care of
* swap (somewhere around the page size), jfs (32k),
* ext[2,3] and reiserfs (64k) - and hopefully all else.
*/
mp->m_ddev_targ.dev = xi.ddev;
mp->m_ddev_targ.bt_mount = mp;
buf = libxfs_getbuf(mp->m_ddev_targ, 0, BTOBB(WHACK_SIZE));
memset(XFS_BUF_PTR(buf), 0, WHACK_SIZE);
libxfs_writebuf(buf, LIBXFS_EXIT_ON_FAILURE);
libxfs_purgebuf(buf);Isn't 128k of zeroing enough to kill existing filesystem signatures? If not how much is, and why can't we just change WHACK_SIZE to reflect the size that will kill those signatures that are further offset into the device?
Cheers,
Dave.
Signature · 7 lines
Dave Chinner [email protected] _______________________________________________ xfs mailing list [email protected] http://oss.sgi.com/mailman/listinfo/xfs
Quoted 1 line · Dave Chinner
Isn't 128k of zeroing enough to kill existing filesystem signatures?
Really no.
Quoted 3 lines
If not how much is, and why can't we just change
WHACK_SIZEto reflect the size that will kill those signatures that are further offset into the device?
btrfs: first superblock at 64KB, second at 64MB, third at 256GB.
zfs has at least 4 blocks at the begin and end of the device.
GPT has the backup table at the end of the device.
RAIDs have signatures also at the end of the device, etc.
Karel
Signature · 7 lines
Karel Zak <[email protected]> http://karelzak.blogspot.com _______________________________________________ xfs mailing list [email protected] http://oss.sgi.com/mailman/listinfo/xfs
Quoted 9 lines · Karel Zak
Date: Wed, 13 Feb 2013 09:01:54 +0100
From: Karel Zak <[email protected]>
To: Dave Chinner <[email protected]>
Cc: Lukas Czerner <[email protected]>, [email protected], [email protected]
Subject: Re: [PATCH]xfs_mkfs: wipe old signatures from the deviceQuoted 1 line · Dave Chinner
Isn't 128k of zeroing enough to kill existing filesystem signatures?
Quoted 7 lines
Really no.
Quoted 3 lines
If not how much is, and why can't we just change
WHACK_SIZEto reflect the size that will kill those signatures that are further offset into the device?btrfs: first superblock at 64KB, second at 64MB, third at 256GB.
We've been discussing this with Dave and others and it seems really odd that blkid would detect file system signature from backup superblock. It seems that it's doing this only for btrfs, ignoring backup superblocks for any other file system.
I know that btrfs user space is broken in a way that it would unconditionally use backup superblock, but it has to be fixed!
Also I think that the bug in btrfs is not a reason for blkid to treat btrfs backup superblocks as primary (which is what you're essentially doing). We should fix that first and then we can discuss whether fs utilities should be using blkid to wipe all the signatures from the device I think.
It seem to me that this is real problem and also nasty regression because:
mkfs.btrfs /dev/sda
mkfs.xfs -f /dev/sda
and the device is unmountable, even though it contains valid xfs file system.
However
mkfs.btrfs /dev/sda
mkfs.ext4 -F /dev/sda
works well, however I am not sure why that is. Is that some kind of mount(8) magic ?
So I think that liblkid _and_ btrfs tools has to be fixed not to treat backup superblocks as primary!
Thanks!
-Lukas
Quoted 11 lines
zfs has at least 4 blocks at the begin and end of the device.
GPT has the backup table at the end of the device.
RAIDs have signatures also at the end of the device, etc.
Karel
_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
Quoted 7 lines · Lukáš Czerner
However
mkfs.btrfs
/dev/sda
mkfs.ext4 -F/dev/sdaworks well, however I am not sure why that is. Is that some kind of mount(8) magic ?
This is bug in libmount. Fixed in upstream tree. The libmount in some cases ignores the ambivalent probing result from libblkid and tries stuff from /etc/filesystems (where is for example ext4).
Quoted 2 lines
So I think that liblkid _and_ btrfs tools has to be fixed not to treat backup superblocks as primary!
The problem with additional btrfs superblocks has been already reported to btrfs guys:
http://www.mail-archive.com/[email protected]/msg20957.html
and I don't see a reply "yep, this is btrfs-progs bug" :-)
Karel
Signature · 2 lines
Karel Zak <[email protected]> http://karelzak.blogspot.com
Quoted 16 lines · Karel Zak
Quoted 7 lines · Lukáš Czerner
However
mkfs.btrfs
/dev/sda
mkfs.ext4 -F/dev/sdaworks well, however I am not sure why that is. Is that some kind of mount(8) magic ?
This is bug in libmount. Fixed in upstream tree. The libmount in some cases ignores the ambivalent probing result from libblkid and tries stuff from
/etc/filesystems(where is for example ext4).Quoted 2 lines
So I think that liblkid _and_ btrfs tools has to be fixed not to treat backup superblocks as primary!
Quoted 6 lines
The problem with additional btrfs superblocks has been already reported to btrfs guys:
http://www.mail-archive.com/[email protected]/msg20957.htmland I don't see a reply "yep, this is btrfs-progs bug" :-)
Oh, it's most definitely a btrfs-progs bug. Using stale metadata data on the block device in preference to current, primary metadata identifying the device to belong to a different filesystem is, well, rather unfriendly.
In the above case, mkfs.ext4 overwrites the btrfs magic block field so that it is not zero (as it is after wipefs runs), and hence the btrfs tools see it as a broken superblock and continue to the next location for recovery rather than see the device as containing a valid but foreign filesystem.
None of the filesystem tools from ext4, btrfs and XFS handle this sort of thing consistently, even though it is their responsibility to do so. XFS is probably the best of them, in that *some* of the tools will refuse to run unless you provide a force flag when the device appears to contain primary identifiers for a different filesystem. e.g. mkfs.xfs won't overwrite existing filesystems without a force flag, but mkfs.ext4 and mkfs.btrfs will:
$ sudo mkfs.xfs /dev/vdb
mkfs.xfs: /dev/vdb appears to contain an existing filesystem (btrfs).
mkfs.xfs: Use the -f option to force overwrite.
$ sudo mkfs.xfs -f /dev/vdb
meta-data=/dev/vdb isize=256 agcount=4, agsize=720896 blks
= sectsz=512 attr=2, projid32bit=0
= crc=0
data = bsize=4096 blocks=2883584, imaxpct=25
= sunit=0 swidth=0 blks
naming =version 2 bsize=4096 ascii-ci=0
log =internal log bsize=4096 blocks=2560, version=2
= sectsz=512 sunit=0 blks, lazy-count=1
realtime =none extsz=4096 blocks=0, rtextents=0
$ sudo mkfs.xfs /dev/vdb
mkfs.xfs: /dev/vdb appears to contain an existing filesystem (xfs).
mkfs.xfs: Use the -f option to force overwrite.
$ sudo mkfs.btrfs /dev/vdbWARNING! - Btrfs Btrfs v0.19 IS EXPERIMENTAL WARNING! - see http://btrfs.wiki.kernel.org before using
fs created label (null) on /dev/vdb
nodesize 4096 leafsize 4096 sectorsize 4096 size 11.00GB
Btrfs Btrfs v0.19
$ sudo mkfs.ext4 /dev/vdb
mke2fs 1.42.5 (29-Jul-2012)
Filesystem label=
OS type: Linux
Block size=4096 (log=2)
Fragment size=4096 (log=2)
Stride=0 blocks, Stripe width=0 blocks
720896 inodes, 2883584 blocks
144179 blocks (5.00%) reserved for the super user
First data block=0
Maximum filesystem blocks=2952790016
88 block groups
32768 blocks per group, 32768 fragments per group
8192 inodes per group
Superblock backups stored on blocks:
32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208Allocating group tables: done
Writing inode tables: done
Creating journal (32768 blocks): done
Writing superblocks and filesystem accounting information: done
$ sudo mkfs.xfs /dev/vdb mkfs.xfs: /dev/vdb appears to contain an existing filesystem (ext4). mkfs.xfs: Use the -f option to force overwrite. $
IOWs, it is the responsibility of the filesystem tools to correctly identify the filesystem being operated on, and not to modify/trash/recover anything unless specifically directed by the user. When a filesystem tool gets it wrong, then that specific tool needs to be fixed. i.e. it is the responsibility of filesystem tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.
And that makes it a btrfs-progs bug that needs to be fixed.
Cheers,
Dave.
Signature · 7 lines
Dave Chinner [email protected] _______________________________________________ xfs mailing list [email protected] http://oss.sgi.com/mailman/listinfo/xfs
Quoted 4 lines · Dave Chinner
it is the responsibility of filesystem
tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.
I appreciate this, and in particular that mkfs.xfs doesn't nerf a file system without the use of -f; even an existing XFS file system. Considering most data loss is user induced, I'd appreciate it if other file systems's tools weren't so easily made belligerent by (hopefully temporarily) confused apes wearing pants.
Chris Murphy
Quoted 10 lines · Chris Murphy
Date: Thu, 14 Feb 2013 00:29:59 -0700
From: Chris Murphy <[email protected]>
To: Dave Chinner <[email protected]>
Cc: Karel Zak <[email protected]>, Lukáš Czerner <[email protected]>,
[email protected], [email protected], Zach Brown <[email protected]>, [email protected]
Subject: Re: [PATCH]xfs_mkfs: wipe old signatures from the deviceOn Feb 13, 2013, at 3:17 PM, Dave Chinner <[email protected]> wrote:
Quoted 8 lines
Quoted 4 lines
it is the responsibility of filesystem
tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.I appreciate this, and in particular that mkfs.xfs doesn't nerf a file system without the use of -f; even an existing XFS file system. Considering most data loss is user induced, I'd appreciate it if other file systems's tools weren't so easily made belligerent by (hopefully temporarily) confused apes wearing pants.
Chris Murphy
I would not be so optimistic about it. The reason being that there are almost _always_ old file system signatures on the device. So I think that it got to the point where users will usually use mkfs.xfs -f all the time. And even if they did not and they would use a wrong device they would probably get the same warning even for the device they wanted to use in the first place.
So even thoug it might help in some cases I do not think that we should go and change all file systems to do that as well, it would not be very useful anyway.
-Lukas
_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
Quoted 17 lines · Lukáš Czerner
Quoted 15 lines · Chris Murphy
Date: Thu, 14 Feb 2013 00:29:59 -0700
From: Chris Murphy <[email protected]>
To: Dave Chinner <[email protected]>
Cc: Karel Zak <[email protected]>, Lukáš Czerner <[email protected]>,
[email protected], [email protected], Zach Brown <[email protected]>, [email protected]
Subject: Re: [PATCH]xfs_mkfs: wipe old signatures from the deviceQuoted 4 lines · Dave Chinner
it is the responsibility of filesystem
tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.
Quoted 6 lines
Quoted 3 lines
I appreciate this, and in particular that mkfs.xfs doesn't nerf a file system without the use of -f; even an existing XFS file system. Considering most data loss is user induced, I'd appreciate it if other file systems's tools weren't so easily made belligerent by (hopefully temporarily) confused apes wearing pants.
Chris Murphy
I would not be so optimistic about it. The reason being that there are almost _always_ old file system signatures on the device.
That assumption is way off the mark. What you do as a filesystem developer (remake filesystems on the same block device hundreds of times a day) is not at all typical, so you cannot extrapolate from your usage habits to what typically happens in production environments.
Admins don't tend to use "force" options by default (especially for destructive comands like mkfs) as 1) they are rarely necessary in the real world and 2) the consequences of errors are severe. The most common filesystem creation pattern in production systems (be it desktop, workstation or server) is that storage, devices and filesystems are set up once and then used for the entire lifetime ofthe system without ever having mkfs run on them again. i.e on pristine, empty hardware. Hence users rarely, if ever, need to use the force option for mkfs.xfs.
Quoted 5 lines
So I
think that it got to the point where users will usually use mkfs.xfs -f all the time. And even if they did not and they would use a wrong device they would probably get the same warning even for the device they wanted to use in the first place.
I get a couple of queries a year from people saying they accidentally ran mkfs.ext4 on the wrong device and want to know if they can recover their XFS filesystem. The next question is usually "why didn't mkfs.ext4 warn me there was an existing filesystem on the device like mkfs.xfs does?".
That is why the "don't overwrite an existing filesystem by default" behaviour is important. Users like to be protected from mistakes they weren't aware they made, and far too few of our filesystem utilities provide that safety net.
A couple of users a year losing data like this is a couple of users too many. Especially when it would only take a couple of hours of your time to implement....
Quoted 3 lines
So even thoug it might help in some cases I do not think that we should go and change all file systems to do that as well, it would not be very useful anyway.
Tell that to the next user that trashes their data because a filesystem tool simply assumed in correctly that it owned the block device.
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
Quoted 70 lines · Dave Chinner
Date: Thu, 14 Feb 2013 22:04:23 +1100
From: Dave Chinner <[email protected]>
To: Lukáš Czerner <[email protected]>
Cc: Chris Murphy <[email protected]>, Karel Zak <[email protected]>,
[email protected], [email protected], Zach Brown <[email protected]>, [email protected]
Subject: Re: [PATCH]xfs_mkfs: wipe old signatures from the deviceQuoted 17 lines · Lukáš Czerner
Quoted 15 lines · Chris Murphy
Date: Thu, 14 Feb 2013 00:29:59 -0700
From: Chris Murphy <[email protected]>
To: Dave Chinner <[email protected]>
Cc: Karel Zak <[email protected]>, Lukáš Czerner <[email protected]>,
[email protected], [email protected], Zach Brown <[email protected]>, [email protected]
Subject: Re: [PATCH]xfs_mkfs: wipe old signatures from the deviceQuoted 4 lines · Dave Chinner
it is the responsibility of filesystem
tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.Quoted 6 lines
Quoted 3 lines
I appreciate this, and in particular that mkfs.xfs doesn't nerf a file system without the use of -f; even an existing XFS file system. Considering most data loss is user induced, I'd appreciate it if other file systems's tools weren't so easily made belligerent by (hopefully temporarily) confused apes wearing pants.
Chris Murphy
I would not be so optimistic about it. The reason being that there are almost _always_ old file system signatures on the device.
That assumption is way off the mark. What you do as a filesystem developer (remake filesystems on the same block device hundreds of times a day) is not at all typical, so you cannot extrapolate from your usage habits to what typically happens in production environments.
Admins don't tend to use "force" options by default (especially for destructive comands like mkfs) as 1) they are rarely necessary in the real world and 2) the consequences of errors are severe. The most common filesystem creation pattern in production systems (be it desktop, workstation or server) is that storage, devices and filesystems are set up once and then used for the entire lifetime ofthe system without ever having mkfs run on them again. i.e on pristine, empty hardware. Hence users rarely, if ever, need to use the force option for mkfs.xfs.
Quoted 5 lines
So I
think that it got to the point where users will usually use mkfs.xfs -f all the time. And even if they did not and they would use a wrong device they would probably get the same warning even for the device they wanted to use in the first place.I get a couple of queries a year from people saying they accidentally ran mkfs.ext4 on the wrong device and want to know if they can recover their XFS filesystem. The next question is usually "why didn't mkfs.ext4 warn me there was an existing filesystem on the device like mkfs.xfs does?".
That is why the "don't overwrite an existing filesystem by default" behaviour is important. Users like to be protected from mistakes they weren't aware they made, and far too few of our filesystem utilities provide that safety net.
A couple of users a year losing data like this is a couple of users too many. Especially when it would only take a couple of hours of your time to implement....
Quoted 7 lines
Quoted 3 lines
So even thoug it might help in some cases I do not think that we should go and change all file systems to do that as well, it would not be very useful anyway.
Tell that to the next user that trashes their data because a filesystem tool simply assumed in correctly that it owned the block device.
I am not sure whether using a single file system for the lifetime of the hardware is really the case, it seems unlikely to me, but you have more experience than I do... But I am not talking about remaking the file system hundreds of times a day, because once is just enough to hit this.
mkfs.ext4 actually does have 'undo' functionality which can store the changes made to the device on the file system itself. It is not enabled by default, but exactly for this reason I've been thinking of ways to make this default and possible delete the data when it's not likely to be needed anymore (first mount ? write ?).
I was also thinking about checking /etc/fstab prior the file system creation, this might rule out most of the false positives that mkfs.xfs can generate I guess.
-Lukas
Quoted 5 lines
Cheers,
Dave.
_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
Am Donnerstag, 14. Februar 2013 schrieb Dave Chinner:
Quoted 21 lines
Quoted 5 lines
So I
think that it got to the point where users will usually use mkfs.xfs -f all the time. And even if they did not and they would use a wrong device they would probably get the same warning even for the device they wanted to use in the first place.I get a couple of queries a year from people saying they accidentally ran mkfs.ext4 on the wrong device and want to know if they can recover their XFS filesystem. The next question is usually "why didn't mkfs.ext4 warn me there was an existing filesystem on the device like mkfs.xfs does?".
That is why the "don't overwrite an existing filesystem by default" behaviour is important. Users like to be protected from mistakes they weren't aware they made, and far too few of our filesystem utilities provide that safety net.
A couple of users a year losing data like this is a couple of users too many. Especially when it would only take a couple of hours of your time to implement....
Quoted 7 lines
Quoted 3 lines
So even thoug it might help in some cases I do not think that we should go and change all file systems to do that as well, it would not be very useful anyway.
Tell that to the next user that trashes their data because a filesystem tool simply assumed in correctly that it owned the block device.
Full ACK.
I always loved that mkfs.xfs asks in that case.
IMO its just sane to do so.
Signature · 2 lines
Martin 'Helios' Steigerwald - http://www.Lichtvoll.de GPG: 03B0 0D6C 0040 0710 4AFA B82F 991B EAAC A599 84C7
Quoted 30 lines · Martin Steigerwald
Am Donnerstag, 14. Februar 2013 schrieb Dave Chinner:
Quoted 21 lines
Quoted 5 lines
So I
think that it got to the point where users will usually use mkfs.xfs -f all the time. And even if they did not and they would use a wrong device they would probably get the same warning even for the device they wanted to use in the first place.I get a couple of queries a year from people saying they accidentally ran mkfs.ext4 on the wrong device and want to know if they can recover their XFS filesystem. The next question is usually "why didn't mkfs.ext4 warn me there was an existing filesystem on the device like mkfs.xfs does?".
That is why the "don't overwrite an existing filesystem by default" behaviour is important. Users like to be protected from mistakes they weren't aware they made, and far too few of our filesystem utilities provide that safety net.
A couple of users a year losing data like this is a couple of users too many. Especially when it would only take a couple of hours of your time to implement....
Quoted 7 lines
Quoted 3 lines
So even thoug it might help in some cases I do not think that we should go and change all file systems to do that as well, it would not be very useful anyway.
Tell that to the next user that trashes their data because a filesystem tool simply assumed in correctly that it owned the block device.
Quoted 5 lines
Full ACK.
I always loved that mkfs.xfs asks in that case.
IMO its just sane to do so.
I just sent a patch to do so for btrfs-progs, FWIW. :)
-Eric
Quoted 32 lines · Dave Chinner
Quoted 10 lines · Lukáš Czerner
Quoted 7 lines · Chris Murphy
Date: Thu, 14 Feb 2013 00:29:59 -0700
From: Chris Murphy <[email protected]>
To: Dave Chinner <[email protected]>
Cc: Karel Zak <[email protected]>, Lukáš Czerner <[email protected]>,
[email protected], [email protected], Zach Brown <[email protected]>, [email protected]
Subject: Re: [PATCH]xfs_mkfs: wipe old signatures from the deviceQuoted 5 lines
So I
think that it got to the point where users will usually use mkfs.xfs -f all the time. And even if they did not and they would use a wrong device they would probably get the same warning even for the device they wanted to use in the first place.I get a couple of queries a year from people saying they accidentally ran mkfs.ext4 on the wrong device and want to know if they can recover their XFS filesystem. The next question is usually "why didn't mkfs.ext4 warn me there was an existing filesystem on the device like mkfs.xfs does?".
That is why the "don't overwrite an existing filesystem by default" behaviour is important. Users like to be protected from mistakes they weren't aware they made, and far too few of our filesystem utilities provide that safety net.
A couple of users a year losing data like this is a couple of users too many. Especially when it would only take a couple of hours of your time to implement....
Quoted 7 lines
Quoted 3 lines
So even thoug it might help in some cases I do not think that we should go and change all file systems to do that as well, it would not be very useful anyway.
Tell that to the next user that trashes their data because a filesystem tool simply assumed in correctly that it owned the block device.
We had someone on IRC a day or two ago who had done exactly this.
They're not the only one -- I can recall seeing at least one other person who managed to mkfs.btrfs on an existing filesystem.
Hugo.
Signature · 4 lines
=== Hugo Mills: hugo@... carfax.org.uk | darksatanic.net | lug.org.uk ===
PGP key: 515C238D from wwwkeys.eu.pgp.net or http://www.carfax.org.uk
--- I am but mad north-north-west: when the wind is southerly, I ---
know a hawk from a handsaw. Quoted 17 lines · Lukáš Czerner
Quoted 15 lines · Chris Murphy
Date: Thu, 14 Feb 2013 00:29:59 -0700
From: Chris Murphy <[email protected]>
To: Dave Chinner <[email protected]>
Cc: Karel Zak <[email protected]>, Lukáš Czerner <[email protected]>,
[email protected], [email protected], Zach Brown <[email protected]>, [email protected]
Subject: Re: [PATCH]xfs_mkfs: wipe old signatures from the deviceQuoted 4 lines · Dave Chinner
it is the responsibility of filesystem
tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.
Quoted 8 lines
Quoted 3 lines
I appreciate this, and in particular that mkfs.xfs doesn't nerf a file system without the use of -f; even an existing XFS file system. Considering most data loss is user induced, I'd appreciate it if other file systems's tools weren't so easily made belligerent by (hopefully temporarily) confused apes wearing pants.
Chris Murphy
I would not be so optimistic about it. The reason being that there are almost _always_ old file system signatures on the device. So I think that it got to the point where users will usually use mkfs.xfs -f all the time.
I know I do ;) But as Dave points out, fs developers are odd ducks.
Quoted 3 lines
And even if they did not and they would use a wrong device they would probably get the same warning even for the device they wanted to use in the first place.
I was thinking, as annoying as it might be, requiring the user to use "-f $OLD_FS_TYPE" might require a bit more positive action and cognition on the admin's part.
OTOH, that could get annoying, and break old scripts. :)
-Eric
Quoted 6 lines
So even thoug it might help in some cases I do not think that we should go and change all file systems to do that as well, it would not be very useful anyway.
-Lukas
Quoted 6 lines · Eric Sandeen
I was thinking, as annoying as it might be, requiring the user to use "-f $OLD_FS_TYPE" might require a bit more positive action and cognition on the admin's part.
OTOH, that could get annoying, and break old scripts. :)
Yes, it's possible to get carried away with this: An "are you sure" prompt that requires typing "yes"; or even more deliciously irritating, only accepts as positive:
"primateconfirmspreviousfilesystemdestructionwontsubsequentlyaskaboutdatarecovery"
Way, way more people will copy/paste that from a list or man entry, than acquire muscle memory for -f.
Chris Murphy
_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
Quoted 8 lines · Dave Chinner
IOWs, it is the responsibility of the filesystem tools to correctly identify the filesystem being operated on, and not to modify/trash/recover anything unless specifically directed by the user. When a filesystem tool gets it wrong, then that specific tool needs to be fixed. i.e. it is the responsibility of filesystem tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.
As an average Joe user, I started to agree with this, but then as a developer I had second thoughts. It can't be right that every filesystem tool has to have code to recognize every other type of filesystem; that just doesn't scale. So each tool would need to call some API, which I suppose would need to access some kernel code that iterated for every filesystem type the kernel was configured to handle, or knew historically.
And then we have identical code in every tool, and that code is not serving the tool's primary purpose, so maybe it should be factored out on the one-tool-one-job philosophy. So perhaps there should be a single tool that tells what filesystem type is present on a device, which everybody runs before mkfs. Oh wait, df -T or other utilities already do that.
So I'm not sure the issue is black and white. As an average Joe, I expect mkfs to trash whatever I give it, so I'm pretty careful to check what is there first. And if I expected to be overwriting an existing filesystem and I did it often enough to know about the mkfs.xfs behaviour, I suppose I would always invoke it with the -f flag. Consistency among the various flavours of mkfs would be nice, though.
_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
Quoted 10 lines · Dave Howorth
Quoted 8 lines · Dave Chinner
IOWs, it is the responsibility of the filesystem tools to correctly identify the filesystem being operated on, and not to modify/trash/recover anything unless specifically directed by the user. When a filesystem tool gets it wrong, then that specific tool needs to be fixed. i.e. it is the responsibility of filesystem tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.
Quoted 4 lines
As an average Joe user, I started to agree with this, but then as a developer I had second thoughts. It can't be right that every filesystem tool has to have code to recognize every other type of filesystem; that just doesn't scale. So each tool would need to call some API, which I
Yep - it's called libblkid, it's very easy to use for this purpose. xfsprogs already does, and I just sent a patch for btrfs-progs as well.
So libblkid contains all the smarts, really, and it's not a code duplication problem.
Quoted 2 lines
suppose would need to access some kernel code that iterated for every filesystem type the kernel was configured to handle, or knew historically.
Nah, it doesn't matter what the running kernel knows about, IMHO, just what is recognized on the disk at hand.
-Eric
Quoted 14 lines
And then we have identical code in every tool, and that code is not serving the tool's primary purpose, so maybe it should be factored out on the one-tool-one-job philosophy. So perhaps there should be a single tool that tells what filesystem type is present on a device, which everybody runs before mkfs. Oh wait, df -T or other utilities already do that.
So I'm not sure the issue is black and white. As an average Joe, I expect mkfs to trash whatever I give it, so I'm pretty careful to check what is there first. And if I expected to be overwriting an existing filesystem and I did it often enough to know about the mkfs.xfs behaviour, I suppose I would always invoke it with the -f flag. Consistency among the various flavours of mkfs would be nice, though.
Quoted 5 lines
_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs