Skip to contentSkip to conversation

e2fsprogs: Using discard in e2fsprogs tools

PATCH Applied Lukas Czerner 2 participants 14 messages Nov 18, 2010 – Dec 2, 2010

lore
Share & tools
lore
#0 lore raw

v2 cover e2fsprogs: Using discard in e2fsprogs tools

Hi all,

We have came to consensus about using discard in e2fsprogs tools. These patches generalize use of discard in e2fsprogs tools introduce changes in mke2fs and e2fsck.

Short summary
-------------

[PATCH 1/6] e2fsprogs: Add discard function into struct_io_manager
* generalize use of discard in e2fsprogs and let any tool in e2fsprogs take advantage of it withou need to write its own BLKDISCARD wrappers.

[PATCH 2/6] e2fsprogs: Add CHANNEL_FLAGS_DISCARD_ZEROES flag for io_manager
* Give the opportunity for any io_manager to check if device discard support zeroes data and save the results into io_channel for use in any e2fsprofs tool.

[PATCH 3/6] e2fsck: Discard free data and inode blocks.
* In pass 5 after the group descriptors has been changed discard free data and inode blocks. The consensus was that it should be OFF by default, so it is.
* Introduce new paid of extended options discard/nodiscard.

[PATCH 4/6] mke2fs: Deprecate -K option, introduce discard/nodiscard
* To the same of consistency and to gain ability to easily default it the new pair of extended options has been added.
* The consensus was that it should stay ON by default, so it is.

[PATCH 5/6] mke2fs: Use unix_discard() for discards

[PATCH 6/6] mke2fs: Add discard option into mke2fs.conf

Any comments appreciated!

Thanks!

-Lukas

---
[PATCH 1/6] e2fsprogs: Add discard function into struct_io_manager [PATCH 2/6] e2fsprogs: Add CHANNEL_FLAGS_DISCARD_ZEROES flag for io_manager [PATCH 3/6] e2fsck: Discard free data and inode blocks.
[PATCH 4/6] mke2fs: Deprecate -K option, introduce discard/nodiscard [PATCH 5/6] mke2fs: Use unix_discard() for discards
[PATCH 6/6] mke2fs: Add discard option into mke2fs.conf

 e2fsck/e2fsck.8.in    |   14 +++++++
 e2fsck/e2fsck.h       |    1 +
 e2fsck/pass5.c        |   96 ++++++++++++++++++++++++++++++++++++++++++++++++
 e2fsck/unix.c         |   10 ++++-
 lib/ext2fs/ext2_io.h  |    5 ++
 lib/ext2fs/unix_io.c  |   45 ++++++++++++++++++++++-
 misc/mke2fs.8.in      |   18 +++++---
 misc/mke2fs.c         |   98 ++++++++++++++-----------------------------------
 misc/mke2fs.conf.5.in |    5 ++
 9 files changed, 213 insertions(+), 79 deletions(-)
#1 lore raw

PATCH v2 1/6 e2fsprogs: Add discard function into struct_io_manager

In order to provide generic "discard" function for all e2fsprogs tools add a discard function prototype into struct_io_manager. Specific function for specific io managers can be crated that way.

This commit also creates unix_discard function which uses BLKDISCARD ioctl to discard data blocks on the block device and bind it into unit_io_manager structure to be available for all e2fsprogs tools. Note that BLKDISCARD is still Linux specific ioctl, however other unix systems may provide similar functionality. So far the unix_discard() remains linux specific hence is embedded in #ifdef __linux__ macro.

---
 lib/ext2fs/ext2_io.h |    2 ++
 lib/ext2fs/unix_io.c |   31 +++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+), 0 deletions(-)
lib/ext2fs/ext2_io.h +2 -0
index ccc9c8b..c8ec8e5 100644
@@ -83,6 +83,8 @@ struct struct_io_manager {
 					int count, void *data);
 	errcode_t (*write_blk64)(io_channel channel, unsigned long long block,
 					int count, const void *data);
+	errcode_t (*discard)(io_channel channel, unsigned long long block,
+			     unsigned long long count, unsigned int blocksize);
 	long	reserved[16];
 };
 
lib/ext2fs/unix_io.c +31 -0
index 1df1fdd..991de50 100644
@@ -115,6 +115,8 @@ static errcode_t unix_read_blk64(io_channel channel, unsigned long long block,
 			       int count, void *data);
 static errcode_t unix_write_blk64(io_channel channel, unsigned long long block,
 				int count, const void *data);
+static errcode_t unix_discard(io_channel channel, unsigned long long block,
+			      unsigned long long count, unsigned int blocksize);
 
 static struct struct_io_manager struct_unix_manager = {
 	EXT2_ET_MAGIC_IO_MANAGER,
@@ -130,6 +132,7 @@ static struct struct_io_manager struct_unix_manager = {
 	unix_get_stats,
 	unix_read_blk64,
 	unix_write_blk64,
+	unix_discard,
 };
 
 io_manager unix_io_manager = &struct_unix_manager;
@@ -834,3 +837,31 @@ static errcode_t unix_set_option(io_channel channel, const char *option,
 	}
 	return EXT2_ET_INVALID_ARGUMENT;
 }
+
+#ifdef __linux__
+
+#ifndef BLKDISCARD
+#define BLKDISCARD	_IO(0x12,119)
+#endif
+
+static errcode_t unix_discard(io_channel channel, unsigned long long block,
+			     unsigned long long count, unsigned int blocksize)
+{
+	struct unix_private_data *data;
+	__uint64_t	range[2];
+
+	EXT2_CHECK_MAGIC(channel, EXT2_ET_MAGIC_IO_CHANNEL);
+	data = (struct unix_private_data *) channel->private_data;
+	EXT2_CHECK_MAGIC(data, EXT2_ET_MAGIC_UNIX_IO_CHANNEL);
+
+	range[0] = (__uint64_t)(block);
+	range[1] = (__uint64_t)(count);
+	range[0] *= (__uint64_t)(blocksize);
+	range[1] *= (__uint64_t)(blocksize);
+
+	return ioctl(data->dev, BLKDISCARD, &range);
+}
+
+#else
+#define unix_discard(channel, block, count, blocksize)	EXT2_ET_UNIMPLEMENTED
+#endif
Signature · 1 line
1.7.2.3
#2 lore raw

Re: [1/6] e2fsprogs: Add discard function into struct_io_manager

Quoted 12 lines · Lukas Czerner

In order to provide generic "discard" function for all e2fsprogs tools add a discard function prototype into struct_io_manager. Specific function for specific io managers can be crated that way.

This commit also creates unix_discard function which uses BLKDISCARD ioctl to discard data blocks on the block device and bind it into unit_io_manager structure to be available for all e2fsprogs tools. Note that BLKDISCARD is still Linux specific ioctl, however other unix systems may provide similar functionality. So far the unix_discard() remains linux specific hence is embedded in #ifdef __linux__ macro.

Quoted 1 line

I had to make a few cleanup changes before I could merge this into the master branch.

1) unix_discard() was defined as a macro in the !__linux__ case. Since unix_discard() gets used to fill in a function pointer in the io_channel_ops structure, this would have caused non-linux compiles to break.

2) I added a io_channel_discard() function which checks to see if io_channel->discard is non-NULL before dereferencing the function pointer (you had mke2fs and e2fsck blindly dereferencing the function pointer before checking to see if it was non-NULL; this is bad, because not all io_managers, in particular, not the test_io manager in your patches, support discard).

3) I added support for discard to the test_io manager, which makes debugging much easier (just compile with --enable-testio-debug, and then set the TEST_IO_FLAGS environment variable).

4) There was no need to pass the block size to the discard function. The blocksize is part of the io_channel abstraction, and is stored in the channel structure.

- Ted

The resulting patch looked like this.

commit e90a59ed434d6c5e38dd148aa4ba5b22b8f7eb24

Author: Lukas Czerner <[email protected]>
Date:   Thu Nov 18 03:38:36 2010 +0000

e2fsprogs: Add discard function into struct_io_manager

In order to provide generic "discard" function for all e2fsprogs tools add a discard function prototype into struct_io_manager. Specific function for specific io managers can be crated that way.

This commit also creates unix_discard function which uses BLKDISCARD ioctl to discard data blocks on the block device and bind it into unit_io_manager structure to be available for all e2fsprogs tools. Note that BLKDISCARD is still Linux specific ioctl, however other unix systems may provide similar functionality. So far the unix_discard() remains linux specific hence is embedded in #ifdef __linux__ macro.

lib/ext2fs/ext2_io.h +2 -0
index ccc9c8b..f26b569 100644
@@ -83,6 +83,8 @@ struct struct_io_manager {
 					int count, void *data);
 	errcode_t (*write_blk64)(io_channel channel, unsigned long long block,
 					int count, const void *data);
+	errcode_t (*discard)(io_channel channel, unsigned long long block,
+			     unsigned long long count);
 	long	reserved[16];
 };
 
lib/ext2fs/io_manager.c +11 -0
index 6d0e234..80f9dfc 100644
@@ -99,3 +99,14 @@ errcode_t io_channel_write_blk64(io_channel channel, unsigned long long block,
 	return (channel->manager->write_blk)(channel, (unsigned long) block,
 					     count, data);
 }
+
+errcode_t io_channel_discard(io_channel channel, unsigned long long block,
+			     unsigned long long count)
+{
+	EXT2_CHECK_MAGIC(channel, EXT2_ET_MAGIC_IO_CHANNEL);
+
+	if (channel->manager->discard)
+		return (channel->manager->discard)(channel, block, count);
+
+	return EXT2_ET_UNIMPLEMENTED;
+}
lib/ext2fs/test_io.c +22 -1
index 8d887a8..242d442 100644
@@ -73,7 +73,8 @@ static errcode_t test_write_byte(io_channel channel, unsigned long offset,
 static errcode_t test_set_option(io_channel channel, const char *option,
 				 const char *arg);
 static errcode_t test_get_stats(io_channel channel, io_stats *stats);
-
+static errcode_t test_discard(io_channel channel, unsigned long long block,
+			      unsigned long long count);
 
 static struct struct_io_manager struct_test_manager = {
 	EXT2_ET_MAGIC_IO_MANAGER,
@@ -89,6 +90,7 @@ static struct struct_io_manager struct_test_manager = {
 	test_get_stats,
 	test_read_blk64,
 	test_write_blk64,
+	test_discard,
 };
 
 io_manager test_io_manager = &struct_test_manager;
@@ -120,6 +122,7 @@ void (*test_io_cb_write_byte)
 #define TEST_FLAG_FLUSH			0x08
 #define TEST_FLAG_DUMP			0x10
 #define TEST_FLAG_SET_OPTION		0x20
+#define TEST_FLAG_DISCARD		0x40
 
 static void test_dump_block(io_channel channel,
 			    struct test_private_data *data,
@@ -495,3 +498,21 @@ static errcode_t test_get_stats(io_channel channel, io_stats *stats)
 	}
 	return retval;
 }
+
+static errcode_t test_discard(io_channel channel, unsigned long long block,
+			      unsigned long long count)
+{
+	struct test_private_data *data;
+	errcode_t	retval = 0;
+
+	EXT2_CHECK_MAGIC(channel, EXT2_ET_MAGIC_IO_CHANNEL);
+	data = (struct test_private_data *) channel->private_data;
+	EXT2_CHECK_MAGIC(data, EXT2_ET_MAGIC_TEST_IO_CHANNEL);
+
+	retval = io_channel_discard(channel, block, count);
+	if (data->flags & TEST_FLAG_DISCARD)
+		fprintf(data->outfile,
+			"Test_io: discard(%llu, %llu) returned %s\n",
+			block, count, retval ? error_message(retval) : "OK");
+	return retval;
+}
lib/ext2fs/unix_io.c +31 -0
index 1df1fdd..2302374 100644
@@ -115,6 +115,8 @@ static errcode_t unix_read_blk64(io_channel channel, unsigned long long block,
 			       int count, void *data);
 static errcode_t unix_write_blk64(io_channel channel, unsigned long long block,
 				int count, const void *data);
+static errcode_t unix_discard(io_channel channel, unsigned long long block,
+			      unsigned long long count);
 
 static struct struct_io_manager struct_unix_manager = {
 	EXT2_ET_MAGIC_IO_MANAGER,
@@ -130,6 +132,7 @@ static struct struct_io_manager struct_unix_manager = {
 	unix_get_stats,
 	unix_read_blk64,
 	unix_write_blk64,
+	unix_discard,
 };
 
 io_manager unix_io_manager = &struct_unix_manager;
@@ -834,3 +837,31 @@ static errcode_t unix_set_option(io_channel channel, const char *option,
 	}
 	return EXT2_ET_INVALID_ARGUMENT;
 }
+
+#if defined(__linux__) && !defined(BLKDISCARD)
+#define BLKDISCARD	_IO(0x12,119)
+#endif
+
+static errcode_t unix_discard(io_channel channel, unsigned long long block,
+			      unsigned long long count)
+{
+#ifdef BLKDISCARD
+	struct unix_private_data *data;
+	__uint64_t	range[2];
+	int		ret;
+
+	EXT2_CHECK_MAGIC(channel, EXT2_ET_MAGIC_IO_CHANNEL);
+	data = (struct unix_private_data *) channel->private_data;
+	EXT2_CHECK_MAGIC(data, EXT2_ET_MAGIC_UNIX_IO_CHANNEL);
+
+	range[0] = (__uint64_t)(block) * channel->block_size;
+	range[1] = (__uint64_t)(count) * channel->block_size;
+
+	ret = ioctl(data->dev, BLKDISCARD, &range);
+	if (ret < 0)
+		return errno;
+	return 0;
+#else
+	return EXT2_ET_UNIMPLEMENTED;
+#endif
+}
#3 lore raw

Re: [1/6] e2fsprogs: Add discard function into struct_io_manager

Quoted 34 lines · Ted Ts'o
Quoted 12 lines · Lukas Czerner

In order to provide generic "discard" function for all e2fsprogs tools add a discard function prototype into struct_io_manager. Specific function for specific io managers can be crated that way.

This commit also creates unix_discard function which uses BLKDISCARD ioctl to discard data blocks on the block device and bind it into unit_io_manager structure to be available for all e2fsprogs tools. Note that BLKDISCARD is still Linux specific ioctl, however other unix systems may provide similar functionality. So far the unix_discard() remains linux specific hence is embedded in #ifdef __linux__ macro.

Quoted 1 line

I had to make a few cleanup changes before I could merge this into the master branch.

1) unix_discard() was defined as a macro in the !__linux__ case. Since unix_discard() gets used to fill in a function pointer in the io_channel_ops structure, this would have caused non-linux compiles to break.

2) I added a io_channel_discard() function which checks to see if io_channel->discard is non-NULL before dereferencing the function pointer (you had mke2fs and e2fsck blindly dereferencing the function pointer before checking to see if it was non-NULL; this is bad, because not all io_managers, in particular, not the test_io manager in your patches, support discard).

3) I added support for discard to the test_io manager, which makes debugging much easier (just compile with --enable-testio-debug, and then set the TEST_IO_FLAGS environment variable).

Quoted 5 lines

4) There was no need to pass the block size to the discard function. The blocksize is part of the io_channel abstraction, and is stored in the channel structure.

- Ted

Hi Ted,

sorry for the delay. Thanks a lot for your review, fixes and merge, it looks good.

Thanks!

-Lukas

Quoted 162 lines

The resulting patch looked like this.

commit e90a59ed434d6c5e38dd148aa4ba5b22b8f7eb24

Author: Lukas Czerner <[email protected]>
Date:   Thu Nov 18 03:38:36 2010 +0000

e2fsprogs: Add discard function into struct_io_manager

In order to provide generic "discard" function for all e2fsprogs tools add a discard function prototype into struct_io_manager. Specific function for specific io managers can be crated that way.

This commit also creates unix_discard function which uses BLKDISCARD ioctl to discard data blocks on the block device and bind it into unit_io_manager structure to be available for all e2fsprogs tools. Note that BLKDISCARD is still Linux specific ioctl, however other unix systems may provide similar functionality. So far the unix_discard() remains linux specific hence is embedded in #ifdef __linux__ macro.

lib/ext2fs/ext2_io.h +2 -0
index ccc9c8b..f26b569 100644
@@ -83,6 +83,8 @@ struct struct_io_manager {
 					int count, void *data);
 	errcode_t (*write_blk64)(io_channel channel, unsigned long long block,
 					int count, const void *data);
+	errcode_t (*discard)(io_channel channel, unsigned long long block,
+			     unsigned long long count);
 	long	reserved[16];
 };
 
lib/ext2fs/io_manager.c +11 -0
index 6d0e234..80f9dfc 100644
@@ -99,3 +99,14 @@ errcode_t io_channel_write_blk64(io_channel channel, unsigned long long block,
 	return (channel->manager->write_blk)(channel, (unsigned long) block,
 					     count, data);
 }
+
+errcode_t io_channel_discard(io_channel channel, unsigned long long block,
+			     unsigned long long count)
+{
+	EXT2_CHECK_MAGIC(channel, EXT2_ET_MAGIC_IO_CHANNEL);
+
+	if (channel->manager->discard)
+		return (channel->manager->discard)(channel, block, count);
+
+	return EXT2_ET_UNIMPLEMENTED;
+}
lib/ext2fs/test_io.c +22 -1
index 8d887a8..242d442 100644
@@ -73,7 +73,8 @@ static errcode_t test_write_byte(io_channel channel, unsigned long offset,
 static errcode_t test_set_option(io_channel channel, const char *option,
 				 const char *arg);
 static errcode_t test_get_stats(io_channel channel, io_stats *stats);
-
+static errcode_t test_discard(io_channel channel, unsigned long long block,
+			      unsigned long long count);
 
 static struct struct_io_manager struct_test_manager = {
 	EXT2_ET_MAGIC_IO_MANAGER,
@@ -89,6 +90,7 @@ static struct struct_io_manager struct_test_manager = {
 	test_get_stats,
 	test_read_blk64,
 	test_write_blk64,
+	test_discard,
 };
 
 io_manager test_io_manager = &struct_test_manager;
@@ -120,6 +122,7 @@ void (*test_io_cb_write_byte)
 #define TEST_FLAG_FLUSH			0x08
 #define TEST_FLAG_DUMP			0x10
 #define TEST_FLAG_SET_OPTION		0x20
+#define TEST_FLAG_DISCARD		0x40
 
 static void test_dump_block(io_channel channel,
 			    struct test_private_data *data,
@@ -495,3 +498,21 @@ static errcode_t test_get_stats(io_channel channel, io_stats *stats)
 	}
 	return retval;
 }
+
+static errcode_t test_discard(io_channel channel, unsigned long long block,
+			      unsigned long long count)
+{
+	struct test_private_data *data;
+	errcode_t	retval = 0;
+
+	EXT2_CHECK_MAGIC(channel, EXT2_ET_MAGIC_IO_CHANNEL);
+	data = (struct test_private_data *) channel->private_data;
+	EXT2_CHECK_MAGIC(data, EXT2_ET_MAGIC_TEST_IO_CHANNEL);
+
+	retval = io_channel_discard(channel, block, count);
+	if (data->flags & TEST_FLAG_DISCARD)
+		fprintf(data->outfile,
+			"Test_io: discard(%llu, %llu) returned %s\n",
+			block, count, retval ? error_message(retval) : "OK");
+	return retval;
+}
lib/ext2fs/unix_io.c +31 -0
index 1df1fdd..2302374 100644
@@ -115,6 +115,8 @@ static errcode_t unix_read_blk64(io_channel channel, unsigned long long block,
 			       int count, void *data);
 static errcode_t unix_write_blk64(io_channel channel, unsigned long long block,
 				int count, const void *data);
+static errcode_t unix_discard(io_channel channel, unsigned long long block,
+			      unsigned long long count);
 
 static struct struct_io_manager struct_unix_manager = {
 	EXT2_ET_MAGIC_IO_MANAGER,
@@ -130,6 +132,7 @@ static struct struct_io_manager struct_unix_manager = {
 	unix_get_stats,
 	unix_read_blk64,
 	unix_write_blk64,
+	unix_discard,
 };
 
 io_manager unix_io_manager = &struct_unix_manager;
@@ -834,3 +837,31 @@ static errcode_t unix_set_option(io_channel channel, const char *option,
 	}
 	return EXT2_ET_INVALID_ARGUMENT;
 }
+
+#if defined(__linux__) && !defined(BLKDISCARD)
+#define BLKDISCARD	_IO(0x12,119)
+#endif
+
+static errcode_t unix_discard(io_channel channel, unsigned long long block,
+			      unsigned long long count)
+{
+#ifdef BLKDISCARD
+	struct unix_private_data *data;
+	__uint64_t	range[2];
+	int		ret;
+
+	EXT2_CHECK_MAGIC(channel, EXT2_ET_MAGIC_IO_CHANNEL);
+	data = (struct unix_private_data *) channel->private_data;
+	EXT2_CHECK_MAGIC(data, EXT2_ET_MAGIC_UNIX_IO_CHANNEL);
+
+	range[0] = (__uint64_t)(block) * channel->block_size;
+	range[1] = (__uint64_t)(count) * channel->block_size;
+
+	ret = ioctl(data->dev, BLKDISCARD, &range);
+	if (ret < 0)
+		return errno;
+	return 0;
+#else
+	return EXT2_ET_UNIMPLEMENTED;
+#endif
+}
#4 lore raw

PATCH v2 2/6 e2fsprogs: Add CHANNEL_FLAGS_DISCARD_ZEROES flag for io_manager

In reply to Lukas Czerner

When the device have discard support and simultaneously discard zeroes data (and it is properly advertised), then we can take advantage of such behavior in several e2fsprogs tools.

Add new flag CHANNEL_FLAGS_DISCARD_ZEROES for struct_io_channel so each io_manager can take advantage of this. The flag is properly set according to BLKDISCARDZEROES ioctl in unix_open.

Also remove old mke2fs_discard_zeroes_data() function and substitute it with helper which test this flag.

---
 lib/ext2fs/ext2_io.h |    3 +++
 lib/ext2fs/unix_io.c |   14 +++++++++++++-
 misc/mke2fs.c        |   23 +----------------------
 3 files changed, 17 insertions(+), 23 deletions(-)
lib/ext2fs/ext2_io.h +3 -0
index c8ec8e5..ec5d2d1 100644
@@ -29,6 +29,9 @@ typedef struct struct_io_channel *io_channel;
 typedef struct struct_io_stats *io_stats;
 
 #define CHANNEL_FLAGS_WRITETHROUGH	0x01
+#define CHANNEL_FLAGS_DISCARD_ZEROES	0x02
+
+#define ext2fs_discard_zeroes_data(i) (i->flags & CHANNEL_FLAGS_DISCARD_ZEROES)
 
 struct struct_io_channel {
 	errcode_t	magic;
lib/ext2fs/unix_io.c +13 -1
index 991de50..703df86 100644
@@ -425,12 +425,18 @@ static errcode_t flush_cached_blocks(io_channel channel,
 }
 #endif /* NO_IO_CACHE */
 
+#ifdef __linux__
+#ifndef BLKDISCARDZEROES
+#define BLKDISCARDZEROES _IO(0x12,124)
+#endif
+#endif
+
 static errcode_t unix_open(const char *name, int flags, io_channel *channel)
 {
 	io_channel	io = NULL;
 	struct unix_private_data *data = NULL;
 	errcode_t	retval;
-	int		open_flags;
+	int		open_flags, zeroes = 0;
 	struct stat	st;
 #ifdef __linux__
 	struct 		utsname ut;
@@ -487,6 +493,12 @@ static errcode_t unix_open(const char *name, int flags, io_channel *channel)
 	}
 #endif
 
+#ifdef BLKDISCARDZEROES
+	ioctl(data->dev, BLKDISCARDZEROES, &zeroes);
+	if (zeroes)
+		io->flags |= CHANNEL_FLAGS_DISCARD_ZEROES;
+#endif
+
 #if defined(__CYGWIN__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 	/*
 	 * Some operating systems require that the buffers be aligned,
misc/mke2fs.c +0 -11
index 0980045..3c5d8b7 100644
@@ -1903,10 +1903,6 @@ static int mke2fs_setup_tdb(const char *name, io_manager *io_ptr)
 #define BLKDISCARD	_IO(0x12,119)
 #endif
 
-#ifndef BLKDISCARDZEROES
-#define BLKDISCARDZEROES _IO(0x12,124)
-#endif
-
 /*
  * Return zero if the discard succeeds, and -1 if the discard fails.
  */
@@ -1941,23 +1937,6 @@ static int mke2fs_discard_blocks(ext2_filsys fs)
 	return ret;
 }
 
-static int mke2fs_discard_zeroes_data(ext2_filsys fs)
-{
-	int fd;
-	int ret;
-	int discard_zeroes_data = 0;
-
-	fd = open64(fs->device_name, O_RDWR);
#5 lore raw

Re: [2/6] e2fsprogs: Add CHANNEL_FLAGS_DISCARD_ZEROES flag for io_manager

Quoted 4 lines · Lukas Czerner

When the device have discard support and simultaneously discard zeroes data (and it is properly advertised), then we can take advantage of such behavior in several e2fsprogs tools.

Quoted 8 lines

Add new flag CHANNEL_FLAGS_DISCARD_ZEROES for struct_io_channel so each io_manager can take advantage of this. The flag is properly set according to BLKDISCARDZEROES ioctl in unix_open.

Also remove old mke2fs_discard_zeroes_data() function and substitute it with helper which test this flag.

I merged this patch with a minor change; I renamed ext2fs_discard_zeroes_data() to io_channel_discard_zeros_data().

- Ted

#6 lore raw

PATCH v2 3/6 e2fsck: Discard free data and inode blocks.

In reply to Lukas Czerner

In Pass 5 when we are checking block and inode bitmaps we have great opportunity to discard free space and unused inodes on the device, because bitmaps has just been verified as valid. This commit takes advantage of this opportunity and discards both, all free space and unused inodes.

I have added new set of options, 'nodiscard' and 'discard'. When the underlying devices does not support discard, or discard ends with an error, or when any kind of error occurs on the filesystem, no further discard attempt will be made and the e2fsck will behave as it would with nodiscard option provided.

As an addition, when there is any not-yet-zeroed inode table and discard zeroes data, then inode table is marked as zeroed.

---
 e2fsck/e2fsck.8.in |   14 +++++++
 e2fsck/e2fsck.h    |    1 +
 e2fsck/pass5.c     |   96 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 e2fsck/unix.c      |   10 +++++-
 4 files changed, 120 insertions(+), 1 deletions(-)
e2fsck/e2fsck.8.in +14 -0
index 3fb15e6..4c2dd98 100644
@@ -189,6 +189,20 @@ be 1 or 2.  The default extended attribute version format is 2.
 .BI fragcheck
 During pass 1, print a detailed report of any discontiguous blocks for
 files in the filesystem.
+.TP
+.BI discard
+Attempt to discard free blocks and unused inode blocks after the full
+filesystem check (discarding blocks is useful on solid state devices and sparse
+/ thin-provisioned storage). Note that discard is done in pass 5 AFTER the
+filesystem has been fully checked and only if it does not contain recognizable
+errors. However there might be cases where
+.B e2fsck
+does not fully recognise a problem and hence in this case this
+option may prevent you from further manual data recovery.
+.TP
+.BI nodiscard
+Do not attempt to discard free blocks and unused inode blocks. This option is
+exacly the opposite of discard option. This is set as default.
 .RE
 .TP
 .B \-f
e2fsck/e2fsck.h +1 -0
index d4df5f3..a377246 100644
@@ -155,6 +155,7 @@ struct resource_track {
 #define E2F_OPT_WRITECHECK	0x0200
 #define E2F_OPT_COMPRESS_DIRS	0x0400
 #define E2F_OPT_FRAGCHECK	0x0800
+#define E2F_OPT_DISCARD		0x1000
 
 /*
  * E2fsck flags
e2fsck/pass5.c +96 -0

156 lines not on this page · raw message

e2fsck/unix.c +9 -0
index 7eb269c..4cf55a9 100644
@@ -594,6 +594,12 @@ static void parse_extended_opts(e2fsck_t ctx, const char *opts)
 		} else if (strcmp(token, "fragcheck") == 0) {
 			ctx->options |= E2F_OPT_FRAGCHECK;
 			continue;
+		} else if (strcmp(token, "discard") == 0) {
+			ctx->options |= E2F_OPT_DISCARD;
+			continue;
+		} else if (strcmp(token, "nodiscard") == 0) {
+			ctx->options &= ~E2F_OPT_DISCARD;
+			continue;
 		} else {
 			fprintf(stderr, _("Unknown extended option: %s\n"),
 				token);
@@ -609,6 +615,8 @@ static void parse_extended_opts(e2fsck_t ctx, const char *opts)
 		       "Valid extended options are:\n"), stderr);
 		fputs(("\tea_ver=<ea_version (1 or 2)>\n"), stderr);
 		fputs(("\tfragcheck\n"), stderr);
+		fputs(("\tdiscard\n"), stderr);
+		fputs(("\tnodiscard\n"), stderr);
 		fputc('\n', stderr);
 		exit(1);
 	}
@@ -667,6 +675,7 @@ static errcode_t PRS(int argc, char *argv[], e2fsck_t *ret_ctx)
 		ctx->program_name = *argv;
 	else
 		ctx->program_name = "e2fsck";
+
 	while ((c = getopt (argc, argv, "panyrcC:B:dE:fvtFVM:b:I:j:P:l:L:N:SsDk")) != EOF)
 		switch (c) {
 		case 'C':
@@ -943,7 +952,6 @@ static errcode_t try_open_fs(e2fsck_t ctx, int flags, io_manager io_ptr,
 	return retval;
 }
 
#7 lore raw

Re: [3/6] e2fsck: Discard free data and inode blocks.

Quoted 11 lines · Lukas Czerner

In Pass 5 when we are checking block and inode bitmaps we have great opportunity to discard free space and unused inodes on the device, because bitmaps has just been verified as valid. This commit takes advantage of this opportunity and discards both, all free space and unused inodes.

I have added new set of options, 'nodiscard' and 'discard'. When the underlying devices does not support discard, or discard ends with an error, or when any kind of error occurs on the filesystem, no further discard attempt will be made and the e2fsck will behave as it would with nodiscard option provided.

Unfortunately, your patch set the ABORT flag and then returned an error, which then cause pass 5 processing to stop. I've fixed in my patches.

Quoted 5 lines

As an addition, when there is any not-yet-zeroed inode table and discard zeroes data, then inode table is marked as zeroed.

- Ted

commit efa1a355a1e4142b2d057be06931eb8fc0903ba3

Author: Lukas Czerner <[email protected]>
Date:   Thu Nov 18 03:38:38 2010 +0000

e2fsck: Discard free data and inode blocks.

In Pass 5 when we are checking block and inode bitmaps we have great opportunity to discard free space and unused inodes on the device, because bitmaps has just been verified as valid. This commit takes advantage of this opportunity and discards both, all free space and unused inodes.

I have added new set of options, 'nodiscard' and 'discard'. When the underlying devices does not support discard, or discard ends with an error, or when any kind of error occurs on the filesystem, no further discard attempt will be made and the e2fsck will behave as it would with nodiscard option provided.

As an addition, when there is any not-yet-zeroed inode table and discard zeroes data, then inode table is marked as zeroed.

e2fsck/e2fsck.8.in +14 -0
index 3fb15e6..4c2dd98 100644
@@ -189,6 +189,20 @@ be 1 or 2.  The default extended attribute version format is 2.
 .BI fragcheck
 During pass 1, print a detailed report of any discontiguous blocks for
 files in the filesystem.
+.TP
+.BI discard
+Attempt to discard free blocks and unused inode blocks after the full
+filesystem check (discarding blocks is useful on solid state devices and sparse
+/ thin-provisioned storage). Note that discard is done in pass 5 AFTER the
+filesystem has been fully checked and only if it does not contain recognizable
+errors. However there might be cases where
+.B e2fsck
+does not fully recognise a problem and hence in this case this
+option may prevent you from further manual data recovery.
+.TP
+.BI nodiscard
+Do not attempt to discard free blocks and unused inode blocks. This option is
+exacly the opposite of discard option. This is set as default.
 .RE
 .TP
 .B \-f
e2fsck/e2fsck.h +1 -0
index d4df5f3..a377246 100644
@@ -155,6 +155,7 @@ struct resource_track {
 #define E2F_OPT_WRITECHECK	0x0200
 #define E2F_OPT_COMPRESS_DIRS	0x0400
 #define E2F_OPT_FRAGCHECK	0x0800
+#define E2F_OPT_DISCARD		0x1000
 
 /*
  * E2fsck flags
e2fsck/pass5.c +84 -0

144 lines not on this page · raw message

e2fsck/unix.c +9 -0
index 7eb269c..4cf55a9 100644
@@ -594,6 +594,12 @@ static void parse_extended_opts(e2fsck_t ctx, const char *opts)
 		} else if (strcmp(token, "fragcheck") == 0) {
 			ctx->options |= E2F_OPT_FRAGCHECK;
 			continue;
+		} else if (strcmp(token, "discard") == 0) {
+			ctx->options |= E2F_OPT_DISCARD;
+			continue;
+		} else if (strcmp(token, "nodiscard") == 0) {
+			ctx->options &= ~E2F_OPT_DISCARD;
+			continue;
 		} else {
 			fprintf(stderr, _("Unknown extended option: %s\n"),
 				token);
@@ -609,6 +615,8 @@ static void parse_extended_opts(e2fsck_t ctx, const char *opts)
 		       "Valid extended options are:\n"), stderr);
 		fputs(("\tea_ver=<ea_version (1 or 2)>\n"), stderr);
 		fputs(("\tfragcheck\n"), stderr);
+		fputs(("\tdiscard\n"), stderr);
+		fputs(("\tnodiscard\n"), stderr);
 		fputc('\n', stderr);
 		exit(1);
 	}
@@ -667,6 +675,7 @@ static errcode_t PRS(int argc, char *argv[], e2fsck_t *ret_ctx)
 		ctx->program_name = *argv;
 	else
 		ctx->program_name = "e2fsck";
+
 	while ((c = getopt (argc, argv, "panyrcC:B:dE:fvtFVM:b:I:j:P:l:L:N:SsDk")) != EOF)
 		switch (c) {
 		case 'C':
@@ -943,7 +952,6 @@ static errcode_t try_open_fs(e2fsck_t ctx, int flags, io_manager io_ptr,
 	return retval;
 }
 
#8 lore raw

PATCH v2 4/6 mke2fs: Deprecate -K option, introduce discard/nodiscard

In reply to Lukas Czerner

It would be nice to have consistent "discard" options in every system tool (mount, fsck, mkfs) taking advantage of discards. Also "discard" and "nodiscard" is more descriptive instead of just "-K" and can be easily defaulted and it is something we can not do with "-K".

With this commit you need to specify extended option like this:

./mke2fs -T <fstype> -E nodiscard <device>

in order make a filesystem without discarding the device first. And

./mke2fs -T <fstype> -E discard <device>

respectively.

-K option is with this commit deprecated and should not be used anymore.

---
 misc/mke2fs.8.in |   18 +++++++++++-------
 misc/mke2fs.c    |   12 +++++++++++-
 2 files changed, 22 insertions(+), 8 deletions(-)
misc/mke2fs.8.in +11 -7
index b46e7e2..c587d81 100644
@@ -45,9 +45,6 @@ mke2fs \- create an ext2/ext3/ext4 filesystem
 .I journal-options
 ]
 [
-.B \-K
-]
-[
 .B \-N
 .I number-of-inodes
 ]
@@ -240,6 +237,17 @@ enable lazy inode table initialization.
 .B test_fs
 Set a flag in the filesystem superblock indicating that it may be
 mounted using experimental kernel code, such as the ext4dev filesystem.
+.TP
+.BI discard
+Attempt to discard blocks at mkfs time (discarding blocks initially is useful
+on solid state devices and sparse / thin-provisioned storage). When the device
+advertises that discard also zeroes data (any subsequent read after the discard
+and before write returns zero), then mark all not-yet-zeroed inode tables as
+zeroed. This significantly speeds up filesystem initialization. This is set
+as default.
+.TP
+.BI nodiscard
+Do not attempt to discard blocks at mkfs time. This is the default.
 .RE
 .TP
 .BI \-f " fragment-size"
@@ -369,10 +377,6 @@ and may be no more than 102,400 filesystem blocks.
 @[email protected] size " or " device
 @JDEV@options can be given for a filesystem.
 .TP
-.BI \-K
-Keep, do not attempt to discard blocks at mkfs time (discarding blocks initially
-is useful on solid state devices and sparse / thin-provisioned storage).
-.TP
 .BI \-l " filename"
 Read the bad blocks list from
 .IR filename .
misc/mke2fs.c +11 -1
index 3c5d8b7..d63c27f 100644
@@ -753,6 +753,10 @@ static void parse_extended_opts(struct ext2_super_block *param,
 				lazy_itable_init = strtoul(arg, &p, 0);
 			else
 				lazy_itable_init = 1;
+		} else if (!strcmp(token, "discard")) {
+			discard = 1;
+		} else if (!strcmp(token, "nodiscard")) {
+			discard = 0;
 		} else {
 			r_usage++;
 			badopt = token;
@@ -768,7 +772,9 @@ static void parse_extended_opts(struct ext2_super_block *param,
 			"\tstripe-width=<RAID stride * data disks in blocks>\n"
 			"\tresize=<resize maximum size in blocks>\n"
 			"\tlazy_itable_init=<0 to disable, 1 to enable>\n"
-			"\ttest_fs\n\n"),
+			"\ttest_fs\n"
+			"\tdiscard\n"
+			"\tnodiscard\n\n"),
 			badopt ? badopt : "");
 		free(buf);
 		exit(1);
@@ -1248,6 +1254,10 @@ static void PRS(int argc, char *argv[])
 			parse_journal_opts(optarg);
 			break;
 		case 'K':
+			fprintf(stderr, _("Warning: -K option is deprecated and "
+					  "should not be used anymore. Use "
+					  "\'-E nodiscard\' extended option "
+					  "instead!\n"));
 			discard = 0;
 			break;
 		case 'j':
Signature · 1 line
1.7.2.3
Applied
#9 lore raw

Re: [4/6] mke2fs: Deprecate -K option, introduce discard/nodiscard

Quoted 11 lines · Lukas Czerner

It would be nice to have consistent "discard" options in every system tool (mount, fsck, mkfs) taking advantage of discards. Also "discard" and "nodiscard" is more descriptive instead of just "-K" and can be easily defaulted and it is something we can not do with "-K".

With this commit you need to specify extended option like this:

./mke2fs -T <fstype> -E nodiscard <device>

in order make a filesystem without discarding the device first. And

Quoted 7 lines

./mke2fs -T <fstype> -E discard <device>

respectively.

-K option is with this commit deprecated and should not be used anymore.

Applied to the maint branch of e2fsprogs, so it will make 1.41.13.

- Ted

#10 lore raw

PATCH v2 5/6 mke2fs: Use unix_discard() for discards

In reply to Lukas Czerner

There is generic discard function in struct_io_manager, or in unix_io_manager to be specific. So use this instead of mke2fs_discard_blocks().

Since mke2fs_discard_blocks() is not used anymore (and should not be) remove it.

---
 misc/mke2fs.c |   54 +++++++++++-------------------------------------------
 1 files changed, 11 insertions(+), 43 deletions(-)
misc/mke2fs.c +11 -43
index d63c27f..e000058 100644
@@ -1907,48 +1907,6 @@ static int mke2fs_setup_tdb(const char *name, io_manager *io_ptr)
 	return retval;
 }
 
-#ifdef __linux__
-
-#ifndef BLKDISCARD
-#define BLKDISCARD	_IO(0x12,119)
-#endif
-
-/*
- * Return zero if the discard succeeds, and -1 if the discard fails.
- */
-static int mke2fs_discard_blocks(ext2_filsys fs)
-{
-	int fd;
-	int ret;
-	int blocksize;
-	__u64 blocks;
-	__uint64_t range[2];
-
-	blocks = ext2fs_blocks_count(fs->super);
-	blocksize = EXT2_BLOCK_SIZE(fs->super);
-	range[0] = 0;
-	range[1] = blocks * blocksize;
-
-	fd = open64(fs->device_name, O_RDWR);
-
-	if (fd > 0) {
-		ret = ioctl(fd, BLKDISCARD, &range);
-		if (verbose) {
-			printf(_("Calling BLKDISCARD from %llu to %llu "),
-			       (unsigned long long) range[0],
-			       (unsigned long long) range[1]);
-			if (ret)
-				printf(_("failed.\n"));
-			else
-				printf(_("succeeded.\n"));
-		}
-		close(fd);
-	}
-	return ret;
-}
-
-#endif
-
 int main (int argc, char *argv[])
 {
 	errcode_t	retval = 0;
@@ -2009,7 +1967,17 @@ int main (int argc, char *argv[])
 
 	/* Can't undo discard ... */
 	if (discard && (io_ptr != undo_io_manager)) {
-		retval = mke2fs_discard_blocks(fs);
+		blk64_t blocks = ext2fs_blocks_count(fs->super);
+		retval = io_ptr->discard(fs->io, 0, blocks, fs->blocksize);
+
+		if (verbose) {
+			printf(_("Calling BLKDISCARD from 0 to %llu "),
+			       (unsigned long long) blocks);
+			if (retval)
+				printf(_("failed.\n"));
+			else
+				printf(_("succeeded.\n"));
+		}
 
 		if (!retval && ext2fs_discard_zeroes_data(fs->io)) {
 			if (verbose)
Signature · 1 line
1.7.2.3
#11 lore raw

Re: [5/6] mke2fs: Use unix_discard() for discards

Quoted 8 lines · Lukas Czerner

There is generic discard function in struct_io_manager, or in unix_io_manager to be specific. So use this instead of mke2fs_discard_blocks().

Since mke2fs_discard_blocks() is not used anymore (and should not be) remove it.

Changed to use io_channel_discard() instead of derferencing the function pointer directly. I've applied this to the master branch.

- Ted

#12 lore raw

PATCH v2 6/6 mke2fs: Add discard option into mke2fs.conf

In reply to Lukas Czerner

Allow to specify discard in mke2fs.conf. Also change the way how to specify default value for lazy_itable_init. It is better to have all this defaulting done in the same place so do it in definition (as we do with discard).

---
 misc/mke2fs.c         |    9 +++++----
 misc/mke2fs.conf.5.in |    5 +++++
 2 files changed, 10 insertions(+), 4 deletions(-)
misc/mke2fs.c +5 -4
index e000058..2b423d7 100644
@@ -83,12 +83,12 @@ int	cflag;
 int	verbose;
 int	quiet;
 int	super_only;
-int	discard = 1;
+int	discard = 1;	/* attempt  to discard device before fs creation */
 int	force;
 int	noaction;
 int	journal_size;
 int	journal_flags;
-int	lazy_itable_init;	/* use lazy inode table init */
+int	lazy_itable_init = 0;	/* do not use lazy inode table init */
 char	*bad_blocks_filename;
 __u32	fs_stride;
 
@@ -1686,8 +1686,9 @@ static void PRS(int argc, char *argv[])
 
 	blocksize = EXT2_BLOCK_SIZE(&fs_param);
 
-	lazy_itable_init = get_bool_from_profile(fs_types,
-						 "lazy_itable_init", 0);
+	lazy_itable_init = get_bool_from_profile(fs_types, "lazy_itable_init",
+						 lazy_itable_init);
+	discard = get_bool_from_profile(fs_types, "discard" , discard);
 
 	/* Get options from profile */
 	for (cpp = fs_types; *cpp; cpp++) {
misc/mke2fs.conf.5.in +5 -0
index d212947..e9913cb 100644
@@ -360,6 +360,11 @@ option.  This can be used to configure the default extended options used
 by
 .BR mke2fs (8)
 on a per-filesystem type basis.
+.TP
+.I discard
+This relation is a boolean which specifies whether the
+.BR mke2fs (8)
+should attempt to discard device prior to filesystem creation.
 .SH FILES
 .TP
 .I /etc/mke2fs.conf
Signature · 1 line
1.7.2.3
Applied
#13 lore raw

Re: [6/6] mke2fs: Add discard option into mke2fs.conf

Quoted 6 lines · Lukas Czerner

Allow to specify discard in mke2fs.conf. Also change the way how to specify default value for lazy_itable_init. It is better to have all this defaulting done in the same place so do it in definition (as we do with discard).

Applied to the maint branch, so it will git e2fsprogs 1.41.13.

- Ted