Skip to contentSkip to conversation

xfs_mkfs: wipe old signatures from the device

PATCH In review Lukas Czerner 9 participants 19 messages Feb 12, 2013 – Feb 14, 2013 Btrfs file system

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 xfs_mkfs: wipe old signatures from the device

We should wipe off all the signatures from the device prior the file system creation. It is because because some file systems (btrfs) may still have their signatures on the device which can be confusing for userspace possibly resulting in the unmountable file system.

This patch adds a function which uses libblkid library to wipe all the signatures from the device.

If user disables libblkid library or does not have it installed this new feature will not be used. This case can be implemented separately.

---
 mkfs/xfs_mkfs.c |  134 +++++++++++++++++++++++++++++++++++++++---------------
 1 files changed, 97 insertions(+), 37 deletions(-)
mkfs/xfs_mkfs.c +97 -37
index d636549..a889620 100644
@@ -286,18 +286,73 @@ calc_stripe_factors(
 }
 
 #ifdef ENABLE_BLKID
+static int
+wipe_signatures(
+	char		*device,
+	int		dryrun)
+{
+	blkid_probe	pr = NULL;
+	int		ret = 0;
+	int		fd;
+
+	pr = blkid_new_probe_from_filename(device);
+	if (!pr)
+		goto out;
+
+	fd = open(device, O_RDWR|O_CLOEXEC);
+	if (fd < 0) {
+		ret = -1;
+		goto out;
+	}
+	ret = blkid_probe_set_device(pr, fd, 0, 0);
+	if (ret)
+		goto out;
+
+	/* No need to check return values we know that 'pr' is initialized */
+	blkid_probe_enable_partitions(pr, 1);
+	blkid_probe_enable_superblocks(pr, 1);
+	blkid_probe_set_partitions_flags(pr, BLKID_PARTS_MAGIC);
+	blkid_probe_set_superblocks_flags(pr,
+					BLKID_SUBLKS_MAGIC | BLKID_SUBLKS_TYPE);
+
+	while (1) {
+		ret = blkid_do_probe(pr);
+		if (ret != 0)
+			break;
+
+		/* Wipe off the signature */
+		ret = blkid_do_wipe(pr, dryrun);
+		if (ret)
+			break;
+	}
+	close(fd);
+
+out:
+	if (pr)
+		blkid_free_probe(pr);
+	if (ret < 0)
+		fprintf(stderr,
+			_("%s: Cannot wipe off existing signatures\n"), progname);
+	return (ret < 0 ? ret : 0);
+}
+
 /*
  * Check for existing filesystem or partition table on device.
  * Returns:
  *	 1 for existing fs or partition
  *	 0 for nothing found
  *	-1 for internal error
+ * If wipe is set, we'll attempt to wipe of all the signatures from
+ * the device. In this case we never return 1 (existing fs or partition).
  */
 static int
 check_overwrite(
-	char		*device)
+	char		*device,
+	int		wipe,
+	int		dryrun)
 {
 	const char	*type;
+	const char	*name;
 	blkid_probe	pr = NULL;
 	int		ret;
 	int		fd;
@@ -325,40 +380,43 @@ check_overwrite(
 	if (!pr)
 		goto out;
 
-	ret = blkid_probe_enable_partitions(pr, 1);
-	if (ret < 0)
-		goto out;
+	/* No need to check return values we know that 'pr' is initialized */
+	blkid_probe_enable_partitions(pr, 1);
+	blkid_probe_enable_superblocks(pr, 1);
+	blkid_probe_set_partitions_flags(pr, BLKID_PARTS_MAGIC);
+	blkid_probe_set_superblocks_flags(pr,
+					BLKID_SUBLKS_MAGIC | BLKID_SUBLKS_TYPE);
 
-	ret = blkid_do_fullprobe(pr);
-	if (ret < 0)
-		goto out;
+	ret = 0;
+	while (1) {
+		int retval = 0;
 
-	/*
-	 * Blkid returns 1 for nothing found and 0 when it finds a signature,
-	 * but we want the exact opposite, so reverse the return value here.
-	 *
-	 * In addition print some useful diagnostics about what actually is
-	 * on the device.
-	 */
-	if (ret) {
-		ret = 0;
-		goto out;
-	}
+		retval = blkid_do_probe(pr);
+		if (retval != 0) {
+			ret = retval;
+			break;
+		}
 
-	if (!blkid_probe_lookup_value(pr, "TYPE", &type, NULL)) {
-		fprintf(stderr,
-			_("%s: %s appears to contain an existing "
-			"filesystem (%s).\n"), progname, device, type);
-	} else if (!blkid_probe_lookup_value(pr, "PTTYPE", &type, NULL)) {
-		fprintf(stderr,
-			_("%s: %s appears to contain a partition "
-			"table (%s).\n"), progname, device, type);
-	} else {
-		fprintf(stderr,
-			_("%s: %s appears to contain something weird "
-			"according to blkid\n"), progname, device);
+		if (blkid_probe_lookup_value(pr, "TYPE", &type, NULL) == 0)
+			name = "filesystem";
+
+		else if (blkid_probe_lookup_value(pr, "PTTYPE",
+						    &type, NULL) == 0)
+			name = "partition table";
+		else
+			continue;
+
+		if (wipe)
+			fprintf(stderr, _("%s: Wiping off %s signature.\n"),
+				progname, type);
+		else {
+			fprintf(stderr,
+				_("%s: %s appears to contain an existing "
+				"%s (%s).\n"), progname, device, name, type);
+			ret = 1;
+			continue;
+		}
 	}
-	ret = 1;
 
 out:
 	if (pr)
@@ -367,6 +425,8 @@ out:
 		fprintf(stderr,
 			_("%s: probe of %s failed, cannot detect "
 			  "existing filesystem.\n"), progname, device);
+	if (ret == 1 && wipe)
+		ret = wipe_signatures(device, dryrun);
 	return ret;
 }
 
@@ -464,7 +524,8 @@ static void get_topology(
 #else /* ENABLE_BLKID */
 static int
 check_overwrite(
-	char		*device)
+	char		*device,
+	int		wipe)
 {
 	char		*type;
 
@@ -1939,16 +2000,15 @@ _("block size %d cannot be smaller than logical sector size %d\n"),
 	xi.rtsize &= sector_mask;
 	xi.logBBsize &= (__uint64_t)-1 << (MAX(lsectorlog, 10) - BBSHIFT);
 
-	if (!force_overwrite) {
-		if (check_overwrite(dfile) ||
-		    check_overwrite(logfile) ||
-		    check_overwrite(xi.rtname)) {
+	if (check_overwrite(dfile, force_overwrite, Nflag) ||
+	    check_overwrite(logfile, force_overwrite, Nflag) ||
+	    check_overwrite(xi.rtname, force_overwrite, Nflag))
+		if (force_overwrite) {
 			fprintf(stderr,
 			_("%s: Use the -f option to force overwrite.\n"),
 				progname);
 			exit(1);
 		}
-	}
 
 	if (discard) {
 		discard_blocks(xi.ddev, xi.dsize);
Signature · 6 lines
1.7.7.6

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
#1 lore raw
Quoted 8 lines · Lukas Czerner
mkfs/xfs_mkfs.c +8 -0
+wipe_signatures(
+	char		*device,
+	int		dryrun)
+{
+	blkid_probe	pr = NULL;
+	int		ret = 0;
+	int		fd;
+

mkfs/xfs_mkfs.c calc_stripe_factors( · line 307

Quoted 10 lines
mkfs/xfs_mkfs.c +10 -0
+	pr = blkid_new_probe_from_filename(device);
+	if (!pr)
+		goto out;
+
+	fd = open(device, O_RDWR|O_CLOEXEC);
+	if (fd < 0) {
+		ret = -1;
+		goto out;
+	}
+	ret = blkid_probe_set_device(pr, fd, 0, 0);

This does not make sense. The blkid_new_probe_from_filename() also opens (read-only) the device ;-)

You need:

   pr = blkid_new_probe();
   if (!pr)
      goto out;

fd = open(device, O_RDWR|O_CLOEXEC);

   if (fd < 0) {
		ret = -1;
		goto out;

}
ret = blkid_probe_set_device(pr, fd, 0, 0);

Maybe you also need something like

AC_CHECK_LIB(blkid, blkid_do_wipe, [have_blkidwipe=yes], [have_blkidwipe=no])

or so, because old libblkid versions does not contain wipe stuff. See also m4/package_blkid.m4 in xfsprogs where is already check for blkid topology stuff.

Karel

Signature · 7 lines
 Karel Zak  <[email protected]>
 http://karelzak.blogspot.com

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
#2 lore raw
Quoted 20 lines · Karel Zak

Date: Tue, 12 Feb 2013 12:31:52 +0100
From: Karel Zak <[email protected]>
To: Lukas Czerner <[email protected]>
Cc: [email protected], [email protected]
Subject: Re: [PATCH] xfs_mkfs: wipe old signatures from the device

Quoted 13 lines · Lukas Czerner
mkfs/xfs_mkfs.c +13 -0
+wipe_signatures(
+	char		*device,
+	int		dryrun)
+{
+	blkid_probe	pr = NULL;
+	int		ret = 0;
+	int		fd;
+
+	pr = blkid_new_probe_from_filename(device);
+	if (!pr)
+		goto out;
+
+	fd = open(device, O_RDWR|O_CLOEXEC);
Quoted 8 lines

mkfs/xfs_mkfs.c calc_stripe_factors( · line 307

Quoted 5 lines
mkfs/xfs_mkfs.c +5 -0
+	if (fd < 0) {
+		ret = -1;
+		goto out;
+	}
+	ret = blkid_probe_set_device(pr, fd, 0, 0);

This does not make sense. The blkid_new_probe_from_filename() also opens (read-only) the device ;-)

Good to know thanks!

-Lukas

Quoted 17 lines

You need:

   pr = blkid_new_probe();
   if (!pr)
      goto out;
   fd = open(device, O_RDWR|O_CLOEXEC);
   if (fd < 0) {
		ret = -1;
		goto out;
   }
   ret = blkid_probe_set_device(pr, fd, 0, 0);

Maybe you also need something like

Quoted 10 lines

AC_CHECK_LIB(blkid, blkid_do_wipe, [have_blkidwipe=yes], [have_blkidwipe=no])

or so, because old libblkid versions does not contain wipe stuff. See also m4/package_blkid.m4 in xfsprogs where is already check for blkid topology stuff.

Karel

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

#3 lore raw
In reply to Lukas Czerner
Quoted 10 lines · Lukas Czerner

We should wipe off all the signatures from the device prior the file system creation. It is because because some file systems (btrfs) may still have their signatures on the device which can be confusing for userspace possibly resulting in the unmountable file system.

This patch adds a function which uses libblkid library to wipe all the signatures from the device.

If user disables libblkid library or does not have it installed this new feature will not be used. This case can be implemented separately.

mkfs.xfs already zeros the first part of the block device regardless of whether it is using libblkid or not:

....
/* amount (in bytes) we zero at the beginning and end of the device to
* remove traces of other filesystems, raid superblocks, etc. */
#define WHACK_SIZE (128 * 1024)
....

        /*
         * Zero out the beginning of the device, to obliterate any old
         * filesystem signatures out there.  This should take care of
         * swap (somewhere around the page size), jfs (32k),
         * ext[2,3] and reiserfs (64k) - and hopefully all else.
         */
        mp->m_ddev_targ.dev = xi.ddev;
        mp->m_ddev_targ.bt_mount = mp;
        buf = libxfs_getbuf(mp->m_ddev_targ, 0, BTOBB(WHACK_SIZE));
        memset(XFS_BUF_PTR(buf), 0, WHACK_SIZE);
        libxfs_writebuf(buf, LIBXFS_EXIT_ON_FAILURE);
        libxfs_purgebuf(buf);

Isn't 128k of zeroing enough to kill existing filesystem signatures? If not how much is, and why can't we just change WHACK_SIZE to reflect the size that will kill those signatures that are further offset into the device?

Cheers,

Dave.

Signature · 7 lines
Dave Chinner
[email protected]

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
#4 lore raw
Quoted 1 line · Dave Chinner

Isn't 128k of zeroing enough to kill existing filesystem signatures?

Really no.

Quoted 3 lines

If not how much is, and why can't we just change WHACK_SIZE to reflect the size that will kill those signatures that are further offset into the device?

btrfs: first superblock at 64KB, second at 64MB, third at 256GB.

zfs has at least 4 blocks at the begin and end of the device.

GPT has the backup table at the end of the device.

RAIDs have signatures also at the end of the device, etc.

Karel

Signature · 7 lines
 Karel Zak  <[email protected]>
 http://karelzak.blogspot.com

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
#5 lore raw
Quoted 9 lines · Karel Zak

Date: Wed, 13 Feb 2013 09:01:54 +0100
From: Karel Zak <[email protected]>
To: Dave Chinner <[email protected]>
Cc: Lukas Czerner <[email protected]>, [email protected], [email protected]
Subject: Re: [PATCH] xfs_mkfs: wipe old signatures from the device

Quoted 1 line · Dave Chinner

Isn't 128k of zeroing enough to kill existing filesystem signatures?

Quoted 7 lines

Really no.

Quoted 3 lines

If not how much is, and why can't we just change WHACK_SIZE to reflect the size that will kill those signatures that are further offset into the device?

btrfs: first superblock at 64KB, second at 64MB, third at 256GB.

We've been discussing this with Dave and others and it seems really odd that blkid would detect file system signature from backup superblock. It seems that it's doing this only for btrfs, ignoring backup superblocks for any other file system.

I know that btrfs user space is broken in a way that it would unconditionally use backup superblock, but it has to be fixed!

Also I think that the bug in btrfs is not a reason for blkid to treat btrfs backup superblocks as primary (which is what you're essentially doing). We should fix that first and then we can discuss whether fs utilities should be using blkid to wipe all the signatures from the device I think.

It seem to me that this is real problem and also nasty regression because:

mkfs.btrfs /dev/sda
mkfs.xfs -f /dev/sda

and the device is unmountable, even though it contains valid xfs file system.

However

mkfs.btrfs /dev/sda
mkfs.ext4 -F /dev/sda

works well, however I am not sure why that is. Is that some kind of mount(8) magic ?

So I think that liblkid _and_ btrfs tools has to be fixed not to treat backup superblocks as primary!

Thanks!
-Lukas

Quoted 11 lines

zfs has at least 4 blocks at the begin and end of the device.

GPT has the backup table at the end of the device.

RAIDs have signatures also at the end of the device, etc.

Karel

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

#6 lore raw
Quoted 7 lines · Lukáš Czerner

However

mkfs.btrfs /dev/sda
mkfs.ext4 -F /dev/sda

works well, however I am not sure why that is. Is that some kind of mount(8) magic ?

This is bug in libmount. Fixed in upstream tree. The libmount in some cases ignores the ambivalent probing result from libblkid and tries stuff from /etc/filesystems (where is for example ext4).

Quoted 2 lines

So I think that liblkid _and_ btrfs tools has to be fixed not to treat backup superblocks as primary!

The problem with additional btrfs superblocks has been already reported to btrfs guys:

http://www.mail-archive.com/[email protected]/msg20957.html

and I don't see a reply "yep, this is btrfs-progs bug" :-)

Karel

Signature · 2 lines
 Karel Zak  <[email protected]>
 http://karelzak.blogspot.com
#7 lore raw
Quoted 16 lines · Karel Zak
Quoted 7 lines · Lukáš Czerner

However

mkfs.btrfs /dev/sda
mkfs.ext4 -F /dev/sda

works well, however I am not sure why that is. Is that some kind of mount(8) magic ?

This is bug in libmount. Fixed in upstream tree. The libmount in some cases ignores the ambivalent probing result from libblkid and tries stuff from /etc/filesystems (where is for example ext4).

Quoted 2 lines

So I think that liblkid _and_ btrfs tools has to be fixed not to treat backup superblocks as primary!

Quoted 6 lines

The problem with additional btrfs superblocks has been already reported to btrfs guys:

   http://www.mail-archive.com/[email protected]/msg20957.html

and I don't see a reply "yep, this is btrfs-progs bug" :-)

Oh, it's most definitely a btrfs-progs bug. Using stale metadata data on the block device in preference to current, primary metadata identifying the device to belong to a different filesystem is, well, rather unfriendly.

In the above case, mkfs.ext4 overwrites the btrfs magic block field so that it is not zero (as it is after wipefs runs), and hence the btrfs tools see it as a broken superblock and continue to the next location for recovery rather than see the device as containing a valid but foreign filesystem.

None of the filesystem tools from ext4, btrfs and XFS handle this sort of thing consistently, even though it is their responsibility to do so. XFS is probably the best of them, in that *some* of the tools will refuse to run unless you provide a force flag when the device appears to contain primary identifiers for a different filesystem. e.g. mkfs.xfs won't overwrite existing filesystems without a force flag, but mkfs.ext4 and mkfs.btrfs will:

$ sudo mkfs.xfs /dev/vdb
mkfs.xfs: /dev/vdb appears to contain an existing filesystem (btrfs).
mkfs.xfs: Use the -f option to force overwrite.
$ sudo mkfs.xfs -f /dev/vdb
meta-data=/dev/vdb               isize=256    agcount=4, agsize=720896 blks
         =                       sectsz=512   attr=2, projid32bit=0
         =                       crc=0    
data     =                       bsize=4096   blocks=2883584, imaxpct=25
         =                       sunit=0      swidth=0 blks
naming   =version 2              bsize=4096   ascii-ci=0
log      =internal log           bsize=4096   blocks=2560, version=2
         =                       sectsz=512   sunit=0 blks, lazy-count=1
realtime =none                   extsz=4096   blocks=0, rtextents=0
$ sudo mkfs.xfs /dev/vdb
mkfs.xfs: /dev/vdb appears to contain an existing filesystem (xfs).
mkfs.xfs: Use the -f option to force overwrite.
$ sudo mkfs.btrfs /dev/vdb

WARNING! - Btrfs Btrfs v0.19 IS EXPERIMENTAL WARNING! - see http://btrfs.wiki.kernel.org before using

fs created label (null) on /dev/vdb

        nodesize 4096 leafsize 4096 sectorsize 4096 size 11.00GB
Btrfs Btrfs v0.19
$ sudo mkfs.ext4 /dev/vdb
mke2fs 1.42.5 (29-Jul-2012)
Filesystem label=
OS type: Linux
Block size=4096 (log=2)
Fragment size=4096 (log=2)
Stride=0 blocks, Stripe width=0 blocks
720896 inodes, 2883584 blocks
144179 blocks (5.00%) reserved for the super user
First data block=0
Maximum filesystem blocks=2952790016
88 block groups
32768 blocks per group, 32768 fragments per group
8192 inodes per group
Superblock backups stored on blocks: 
        32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208

Allocating group tables: done
Writing inode tables: done
Creating journal (32768 blocks): done
Writing superblocks and filesystem accounting information: done

$ sudo mkfs.xfs /dev/vdb
mkfs.xfs: /dev/vdb appears to contain an existing filesystem (ext4).
mkfs.xfs: Use the -f option to force overwrite.
$

IOWs, it is the responsibility of the filesystem tools to correctly identify the filesystem being operated on, and not to modify/trash/recover anything unless specifically directed by the user. When a filesystem tool gets it wrong, then that specific tool needs to be fixed. i.e. it is the responsibility of filesystem tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.

And that makes it a btrfs-progs bug that needs to be fixed.

Cheers,

Dave.

Signature · 7 lines
Dave Chinner
[email protected]

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
#8 lore raw
Quoted 4 lines · Dave Chinner

it is the responsibility of filesystem
tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.

I appreciate this, and in particular that mkfs.xfs doesn't nerf a file system without the use of -f; even an existing XFS file system. Considering most data loss is user induced, I'd appreciate it if other file systems's tools weren't so easily made belligerent by (hopefully temporarily) confused apes wearing pants.

Chris Murphy

#9 lore raw
Quoted 10 lines · Chris Murphy

Date: Thu, 14 Feb 2013 00:29:59 -0700
From: Chris Murphy <[email protected]>
To: Dave Chinner <[email protected]>
Cc: Karel Zak <[email protected]>, Lukáš Czerner <[email protected]>,
[email protected], [email protected], Zach Brown <[email protected]>, [email protected]
Subject: Re: [PATCH] xfs_mkfs: wipe old signatures from the device

On Feb 13, 2013, at 3:17 PM, Dave Chinner <[email protected]> wrote:

Quoted 8 lines
Quoted 4 lines

it is the responsibility of filesystem
tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.

I appreciate this, and in particular that mkfs.xfs doesn't nerf a file system without the use of -f; even an existing XFS file system. Considering most data loss is user induced, I'd appreciate it if other file systems's tools weren't so easily made belligerent by (hopefully temporarily) confused apes wearing pants.

Chris Murphy

I would not be so optimistic about it. The reason being that there are almost _always_ old file system signatures on the device. So I think that it got to the point where users will usually use mkfs.xfs -f all the time. And even if they did not and they would use a wrong device they would probably get the same warning even for the device they wanted to use in the first place.

So even thoug it might help in some cases I do not think that we should go and change all file systems to do that as well, it would not be very useful anyway.

-Lukas

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

#10 lore raw
Quoted 17 lines · Lukáš Czerner
Quoted 15 lines · Chris Murphy

Date: Thu, 14 Feb 2013 00:29:59 -0700
From: Chris Murphy <[email protected]>
To: Dave Chinner <[email protected]>
Cc: Karel Zak <[email protected]>, Lukáš Czerner <[email protected]>,
[email protected], [email protected], Zach Brown <[email protected]>, [email protected]
Subject: Re: [PATCH] xfs_mkfs: wipe old signatures from the device

Quoted 4 lines · Dave Chinner

it is the responsibility of filesystem
tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.

Quoted 6 lines
Quoted 3 lines

I appreciate this, and in particular that mkfs.xfs doesn't nerf a file system without the use of -f; even an existing XFS file system. Considering most data loss is user induced, I'd appreciate it if other file systems's tools weren't so easily made belligerent by (hopefully temporarily) confused apes wearing pants.

Chris Murphy

I would not be so optimistic about it. The reason being that there are almost _always_ old file system signatures on the device.

That assumption is way off the mark. What you do as a filesystem developer (remake filesystems on the same block device hundreds of times a day) is not at all typical, so you cannot extrapolate from your usage habits to what typically happens in production environments.

Admins don't tend to use "force" options by default (especially for destructive comands like mkfs) as 1) they are rarely necessary in the real world and 2) the consequences of errors are severe. The most common filesystem creation pattern in production systems (be it desktop, workstation or server) is that storage, devices and filesystems are set up once and then used for the entire lifetime ofthe system without ever having mkfs run on them again. i.e on pristine, empty hardware. Hence users rarely, if ever, need to use the force option for mkfs.xfs.

Quoted 5 lines

So I
think that it got to the point where users will usually use mkfs.xfs -f all the time. And even if they did not and they would use a wrong device they would probably get the same warning even for the device they wanted to use in the first place.

I get a couple of queries a year from people saying they accidentally ran mkfs.ext4 on the wrong device and want to know if they can recover their XFS filesystem. The next question is usually "why didn't mkfs.ext4 warn me there was an existing filesystem on the device like mkfs.xfs does?".

That is why the "don't overwrite an existing filesystem by default" behaviour is important. Users like to be protected from mistakes they weren't aware they made, and far too few of our filesystem utilities provide that safety net.

A couple of users a year losing data like this is a couple of users too many. Especially when it would only take a couple of hours of your time to implement....

Quoted 3 lines

So even thoug it might help in some cases I do not think that we should go and change all file systems to do that as well, it would not be very useful anyway.

Tell that to the next user that trashes their data because a filesystem tool simply assumed in correctly that it owned the block device.

Cheers,

Dave.

Signature · 2 lines
Dave Chinner
[email protected]
#11 lore raw
Quoted 70 lines · Dave Chinner

Date: Thu, 14 Feb 2013 22:04:23 +1100
From: Dave Chinner <[email protected]>
To: Lukáš Czerner <[email protected]>
Cc: Chris Murphy <[email protected]>, Karel Zak <[email protected]>,
[email protected], [email protected], Zach Brown <[email protected]>, [email protected]
Subject: Re: [PATCH] xfs_mkfs: wipe old signatures from the device

Quoted 17 lines · Lukáš Czerner
Quoted 15 lines · Chris Murphy

Date: Thu, 14 Feb 2013 00:29:59 -0700
From: Chris Murphy <[email protected]>
To: Dave Chinner <[email protected]>
Cc: Karel Zak <[email protected]>, Lukáš Czerner <[email protected]>,
[email protected], [email protected], Zach Brown <[email protected]>, [email protected]
Subject: Re: [PATCH] xfs_mkfs: wipe old signatures from the device

Quoted 4 lines · Dave Chinner

it is the responsibility of filesystem
tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.

Quoted 6 lines
Quoted 3 lines

I appreciate this, and in particular that mkfs.xfs doesn't nerf a file system without the use of -f; even an existing XFS file system. Considering most data loss is user induced, I'd appreciate it if other file systems's tools weren't so easily made belligerent by (hopefully temporarily) confused apes wearing pants.

Chris Murphy

I would not be so optimistic about it. The reason being that there are almost _always_ old file system signatures on the device.

That assumption is way off the mark. What you do as a filesystem developer (remake filesystems on the same block device hundreds of times a day) is not at all typical, so you cannot extrapolate from your usage habits to what typically happens in production environments.

Admins don't tend to use "force" options by default (especially for destructive comands like mkfs) as 1) they are rarely necessary in the real world and 2) the consequences of errors are severe. The most common filesystem creation pattern in production systems (be it desktop, workstation or server) is that storage, devices and filesystems are set up once and then used for the entire lifetime ofthe system without ever having mkfs run on them again. i.e on pristine, empty hardware. Hence users rarely, if ever, need to use the force option for mkfs.xfs.

Quoted 5 lines

So I
think that it got to the point where users will usually use mkfs.xfs -f all the time. And even if they did not and they would use a wrong device they would probably get the same warning even for the device they wanted to use in the first place.

I get a couple of queries a year from people saying they accidentally ran mkfs.ext4 on the wrong device and want to know if they can recover their XFS filesystem. The next question is usually "why didn't mkfs.ext4 warn me there was an existing filesystem on the device like mkfs.xfs does?".

That is why the "don't overwrite an existing filesystem by default" behaviour is important. Users like to be protected from mistakes they weren't aware they made, and far too few of our filesystem utilities provide that safety net.

A couple of users a year losing data like this is a couple of users too many. Especially when it would only take a couple of hours of your time to implement....

Quoted 7 lines
Quoted 3 lines

So even thoug it might help in some cases I do not think that we should go and change all file systems to do that as well, it would not be very useful anyway.

Tell that to the next user that trashes their data because a filesystem tool simply assumed in correctly that it owned the block device.

I am not sure whether using a single file system for the lifetime of the hardware is really the case, it seems unlikely to me, but you have more experience than I do... But I am not talking about remaking the file system hundreds of times a day, because once is just enough to hit this.

mkfs.ext4 actually does have 'undo' functionality which can store the changes made to the device on the file system itself. It is not enabled by default, but exactly for this reason I've been thinking of ways to make this default and possible delete the data when it's not likely to be needed anymore (first mount ? write ?).

I was also thinking about checking /etc/fstab prior the file system creation, this might rule out most of the false positives that mkfs.xfs can generate I guess.

-Lukas

Quoted 5 lines

Cheers,

Dave.

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

#12 lore raw
In reply to Dave Chinner

Am Donnerstag, 14. Februar 2013 schrieb Dave Chinner:

Quoted 21 lines
Quoted 5 lines

So I
think that it got to the point where users will usually use mkfs.xfs -f all the time. And even if they did not and they would use a wrong device they would probably get the same warning even for the device they wanted to use in the first place.

I get a couple of queries a year from people saying they accidentally ran mkfs.ext4 on the wrong device and want to know if they can recover their XFS filesystem. The next question is usually "why didn't mkfs.ext4 warn me there was an existing filesystem on the device like mkfs.xfs does?".

That is why the "don't overwrite an existing filesystem by default" behaviour is important. Users like to be protected from mistakes they weren't aware they made, and far too few of our filesystem utilities provide that safety net.

A couple of users a year losing data like this is a couple of users too many. Especially when it would only take a couple of hours of your time to implement....

Quoted 7 lines
Quoted 3 lines

So even thoug it might help in some cases I do not think that we should go and change all file systems to do that as well, it would not be very useful anyway.

Tell that to the next user that trashes their data because a filesystem tool simply assumed in correctly that it owned the block device.

Full ACK.

I always loved that mkfs.xfs asks in that case.

IMO its just sane to do so.

Signature · 2 lines
Martin 'Helios' Steigerwald - http://www.Lichtvoll.de
GPG: 03B0 0D6C 0040 0710 4AFA  B82F 991B EAAC A599 84C7
#13 lore raw
Quoted 30 lines · Martin Steigerwald

Am Donnerstag, 14. Februar 2013 schrieb Dave Chinner:

Quoted 21 lines
Quoted 5 lines

So I
think that it got to the point where users will usually use mkfs.xfs -f all the time. And even if they did not and they would use a wrong device they would probably get the same warning even for the device they wanted to use in the first place.

I get a couple of queries a year from people saying they accidentally ran mkfs.ext4 on the wrong device and want to know if they can recover their XFS filesystem. The next question is usually "why didn't mkfs.ext4 warn me there was an existing filesystem on the device like mkfs.xfs does?".

That is why the "don't overwrite an existing filesystem by default" behaviour is important. Users like to be protected from mistakes they weren't aware they made, and far too few of our filesystem utilities provide that safety net.

A couple of users a year losing data like this is a couple of users too many. Especially when it would only take a couple of hours of your time to implement....

Quoted 7 lines
Quoted 3 lines

So even thoug it might help in some cases I do not think that we should go and change all file systems to do that as well, it would not be very useful anyway.

Tell that to the next user that trashes their data because a filesystem tool simply assumed in correctly that it owned the block device.

Quoted 5 lines

Full ACK.

I always loved that mkfs.xfs asks in that case.

IMO its just sane to do so.

I just sent a patch to do so for btrfs-progs, FWIW. :)

-Eric

#14 lore raw
In reply to Dave Chinner
Quoted 32 lines · Dave Chinner
Quoted 10 lines · Lukáš Czerner
Quoted 7 lines · Chris Murphy

Date: Thu, 14 Feb 2013 00:29:59 -0700
From: Chris Murphy <[email protected]>
To: Dave Chinner <[email protected]>
Cc: Karel Zak <[email protected]>, Lukáš Czerner <[email protected]>,
[email protected], [email protected], Zach Brown <[email protected]>, [email protected]
Subject: Re: [PATCH] xfs_mkfs: wipe old signatures from the device

Quoted 5 lines

So I
think that it got to the point where users will usually use mkfs.xfs -f all the time. And even if they did not and they would use a wrong device they would probably get the same warning even for the device they wanted to use in the first place.

I get a couple of queries a year from people saying they accidentally ran mkfs.ext4 on the wrong device and want to know if they can recover their XFS filesystem. The next question is usually "why didn't mkfs.ext4 warn me there was an existing filesystem on the device like mkfs.xfs does?".

That is why the "don't overwrite an existing filesystem by default" behaviour is important. Users like to be protected from mistakes they weren't aware they made, and far too few of our filesystem utilities provide that safety net.

A couple of users a year losing data like this is a couple of users too many. Especially when it would only take a couple of hours of your time to implement....

Quoted 7 lines
Quoted 3 lines

So even thoug it might help in some cases I do not think that we should go and change all file systems to do that as well, it would not be very useful anyway.

Tell that to the next user that trashes their data because a filesystem tool simply assumed in correctly that it owned the block device.

We had someone on IRC a day or two ago who had done exactly this.
They're not the only one -- I can recall seeing at least one other person who managed to mkfs.btrfs on an existing filesystem.

Hugo.

Signature · 4 lines
=== Hugo Mills: hugo@... carfax.org.uk | darksatanic.net | lug.org.uk ===
  PGP key: 515C238D from wwwkeys.eu.pgp.net or http://www.carfax.org.uk
  --- I am but mad north-north-west:  when the wind is southerly, I ---  
                       know a hawk from a handsaw.                       
#15 lore raw
In reply to Lukáš Czerner
Quoted 17 lines · Lukáš Czerner
Quoted 15 lines · Chris Murphy

Date: Thu, 14 Feb 2013 00:29:59 -0700
From: Chris Murphy <[email protected]>
To: Dave Chinner <[email protected]>
Cc: Karel Zak <[email protected]>, Lukáš Czerner <[email protected]>,
[email protected], [email protected], Zach Brown <[email protected]>, [email protected]
Subject: Re: [PATCH] xfs_mkfs: wipe old signatures from the device

Quoted 4 lines · Dave Chinner

it is the responsibility of filesystem
tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.

Quoted 8 lines
Quoted 3 lines

I appreciate this, and in particular that mkfs.xfs doesn't nerf a file system without the use of -f; even an existing XFS file system. Considering most data loss is user induced, I'd appreciate it if other file systems's tools weren't so easily made belligerent by (hopefully temporarily) confused apes wearing pants.

Chris Murphy

I would not be so optimistic about it. The reason being that there are almost _always_ old file system signatures on the device. So I think that it got to the point where users will usually use mkfs.xfs -f all the time.

I know I do ;) But as Dave points out, fs developers are odd ducks.

Quoted 3 lines

And even if they did not and they would use a wrong device they would probably get the same warning even for the device they wanted to use in the first place.

I was thinking, as annoying as it might be, requiring the user to use "-f $OLD_FS_TYPE" might require a bit more positive action and cognition on the admin's part.

OTOH, that could get annoying, and break old scripts. :)

-Eric

Quoted 6 lines

So even thoug it might help in some cases I do not think that we should go and change all file systems to do that as well, it would not be very useful anyway.

-Lukas

#16 lore raw
Quoted 6 lines · Eric Sandeen

I was thinking, as annoying as it might be, requiring the user to use "-f $OLD_FS_TYPE" might require a bit more positive action and cognition on the admin's part.

OTOH, that could get annoying, and break old scripts. :)

Yes, it's possible to get carried away with this: An "are you sure" prompt that requires typing "yes"; or even more deliciously irritating, only accepts as positive:

"primateconfirmspreviousfilesystemdestructionwontsubsequentlyaskaboutdatarecovery"

Way, way more people will copy/paste that from a list or man entry, than acquire muscle memory for -f.

Chris Murphy
_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

#17 lore raw
In reply to Dave Chinner
Quoted 8 lines · Dave Chinner

IOWs, it is the responsibility of the filesystem tools to correctly identify the filesystem being operated on, and not to modify/trash/recover anything unless specifically directed by the user. When a filesystem tool gets it wrong, then that specific tool needs to be fixed. i.e. it is the responsibility of filesystem tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.

As an average Joe user, I started to agree with this, but then as a developer I had second thoughts. It can't be right that every filesystem tool has to have code to recognize every other type of filesystem; that just doesn't scale. So each tool would need to call some API, which I suppose would need to access some kernel code that iterated for every filesystem type the kernel was configured to handle, or knew historically.

And then we have identical code in every tool, and that code is not serving the tool's primary purpose, so maybe it should be factored out on the one-tool-one-job philosophy. So perhaps there should be a single tool that tells what filesystem type is present on a device, which everybody runs before mkfs. Oh wait, df -T or other utilities already do that.

So I'm not sure the issue is black and white. As an average Joe, I expect mkfs to trash whatever I give it, so I'm pretty careful to check what is there first. And if I expected to be overwriting an existing filesystem and I did it often enough to know about the mkfs.xfs behaviour, I suppose I would always invoke it with the -f flag. Consistency among the various flavours of mkfs would be nice, though.

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

#18 lore raw
Quoted 10 lines · Dave Howorth
Quoted 8 lines · Dave Chinner

IOWs, it is the responsibility of the filesystem tools to correctly identify the filesystem being operated on, and not to modify/trash/recover anything unless specifically directed by the user. When a filesystem tool gets it wrong, then that specific tool needs to be fixed. i.e. it is the responsibility of filesystem tools to behave sanely, not for the rest of the world to have to work around the dangerous behaviour of a specific filesystems' toolset.

Quoted 4 lines

As an average Joe user, I started to agree with this, but then as a developer I had second thoughts. It can't be right that every filesystem tool has to have code to recognize every other type of filesystem; that just doesn't scale. So each tool would need to call some API, which I

Yep - it's called libblkid, it's very easy to use for this purpose. xfsprogs already does, and I just sent a patch for btrfs-progs as well.

So libblkid contains all the smarts, really, and it's not a code duplication problem.

Quoted 2 lines

suppose would need to access some kernel code that iterated for every filesystem type the kernel was configured to handle, or knew historically.

Nah, it doesn't matter what the running kernel knows about, IMHO, just what is recognized on the disk at hand.

-Eric

Quoted 14 lines

And then we have identical code in every tool, and that code is not serving the tool's primary purpose, so maybe it should be factored out on the one-tool-one-job philosophy. So perhaps there should be a single tool that tells what filesystem type is present on a device, which everybody runs before mkfs. Oh wait, df -T or other utilities already do that.

So I'm not sure the issue is black and white. As an average Joe, I expect mkfs to trash whatever I give it, so I'm pretty careful to check what is there first. And if I expected to be overwriting an existing filesystem and I did it often enough to know about the mkfs.xfs behaviour, I suppose I would always invoke it with the -f flag. Consistency among the various flavours of mkfs would be nice, though.

Quoted 5 lines

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs