Skip to contentSkip to conversation

common/filter: add _filter_bash()

PATCH In review Theodore Ts'o 2 participants 6 messages Dec 14, 2021 – Dec 19, 2021

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 common/filter: add _filter_bash()

This is needed to account for bash 5.1 adding line number annotation when a command like "bash -c /etc/passwd" fails, e.g., with

bash: line 1: /etc/passwd: Permission denied

instead of:

bash: /etc/passwd: Permission denied

---
 common/filter     | 11 +++++++++++
 tests/generic/572 | 33 +++++++++++++++++++--------------
 2 files changed, 30 insertions(+), 14 deletions(-)
common/filter +11 -0
index 2efbbd99..4b250e8b 100644
@@ -661,5 +661,16 @@ _filter_quota_report()
 		s|^(.*?) (\d+) (\d+) (\d+)|$1 @{[$2 * 1024 /'$bsize']} @{[$3 * 1024 /'$bsize']} @{[$4 * 1024 /'$bsize']}|'
 }
 
+#
+# Bash 5.1+ adds "line 1: " when printing an error running an executable
+# for example, "bash -c /etc/passwd" will result in the error
+# "bash: line 1: /etc/passwd: Permission denied" where as earlier
+# versions of bash will omit the "line 1: " annotation.
+#
+_filter_bash()
+{
+	sed -e "s/^bash: line 1: /bash: /"
+}
+
 # make sure this script returns success
 /bin/true
tests/generic/572 +19 -14
index f131c7ed..cded9ac6 100755
@@ -36,6 +36,11 @@ _scratch_mount
 fsv_orig_file=$SCRATCH_MNT/file
 fsv_file=$SCRATCH_MNT/file.fsv
 
+filter_output()
+{
+    _filter_bash | _filter_scratch
+}
+
 verify_data_readable()
 {
 	local file=$1
@@ -48,41 +53,41 @@ verify_data_unreadable()
 	local file=$1
 
 	# try both reading just the first data block, and reading until EOF
-	head -c $FSV_BLOCK_SIZE $file 2>&1 >/dev/null | _filter_scratch
-	md5sum $file |& _filter_scratch
+	head -c $FSV_BLOCK_SIZE $file 2>&1 >/dev/null | filter_output
+	md5sum $file |& filter_output
 }
 
 _fsv_scratch_begin_subtest "Enabling verity on file with verity already enabled fails with EEXIST"
 _fsv_create_enable_file $fsv_file
 echo "(trying again)"
-_fsv_enable $fsv_file |& _filter_scratch
+_fsv_enable $fsv_file |& filter_output
 
 _fsv_scratch_begin_subtest "Enabling verity with invalid hash algorithm fails with EINVAL"
-_fsv_create_enable_file $fsv_file --hash-alg=257 |& _filter_scratch
+_fsv_create_enable_file $fsv_file --hash-alg=257 |& filter_output
 verify_data_readable $fsv_file
 
 _fsv_scratch_begin_subtest "Enabling verity with invalid block size fails with EINVAL"
-_fsv_create_enable_file $fsv_file --block-size=1 |& _filter_scratch
+_fsv_create_enable_file $fsv_file --block-size=1 |& filter_output
 verify_data_readable $fsv_file
 
 _fsv_scratch_begin_subtest "Enabling verity on directory fails with EISDIR"
 mkdir $SCRATCH_MNT/dir
-_fsv_enable $SCRATCH_MNT/dir |& _filter_scratch
+_fsv_enable $SCRATCH_MNT/dir |& filter_output
 
 _fsv_scratch_begin_subtest "Enabling verity with too-long salt fails with EMSGSIZE"
-_fsv_create_enable_file $fsv_file --salt=$(perl -e 'print "A" x 1000') |& _filter_scratch
+_fsv_create_enable_file $fsv_file --salt=$(perl -e 'print "A" x 1000') |& filter_output
 verify_data_readable $fsv_file
 
 _fsv_scratch_begin_subtest "Enabling verity on file on read-only filesystem fails with EROFS"
 echo foo > $fsv_file
 _scratch_remount ro
-_fsv_enable $fsv_file |& _filter_scratch
+_fsv_enable $fsv_file |& filter_output
 _scratch_remount rw
 
 _fsv_scratch_begin_subtest "Enabling verity on file open for writing fails with ETXTBSY"
 echo foo > $fsv_file
 exec 3<> $fsv_file
-_fsv_enable $fsv_file |& _filter_scratch
+_fsv_enable $fsv_file |& filter_output
 exec 3<&-
 verify_data_readable $fsv_file
 
@@ -103,7 +108,7 @@ dd if=/dev/zero of=$fsv_file bs=1 count=0 seek=$((1 << 34)) status=none
 start_time=$(date +%s)
 $FSVERITY_PROG enable $fsv_file &
 sleep 0.5
-_fsv_enable $fsv_file |& _filter_scratch
+_fsv_enable $fsv_file |& filter_output
 kill %1
 wait
 
@@ -112,11 +117,11 @@ _fsv_create_enable_file $fsv_file >> $seqres.full
 echo "* reading"
 $XFS_IO_PROG -r $fsv_file -c ''
 echo "* xfs_io writing, should be O_RDWR"
-$XFS_IO_PROG $fsv_file -c '' |& _filter_scratch
+$XFS_IO_PROG $fsv_file -c '' |& filter_output
 echo "* bash >>, should be O_APPEND"
-bash -c "echo >> $fsv_file" |& _filter_scratch
+bash -c "echo >> $fsv_file" |& filter_output
 echo "* bash >, should be O_WRONLY|O_CREAT|O_TRUNC"
-bash -c "echo > $fsv_file" |& _filter_scratch
+bash -c "echo > $fsv_file" |& filter_output
 
 _fsv_scratch_begin_subtest "verity file can be read"
 _fsv_create_enable_file $fsv_file >> $seqres.full
@@ -160,7 +165,7 @@ _get_filesize $fsv_file
 
 _fsv_scratch_begin_subtest "Trying to measure non-verity file fails with ENODATA"
 echo foo > $fsv_file
-_fsv_measure $fsv_file |& _filter_scratch
+_fsv_measure $fsv_file |& filter_output
 verify_data_readable $fsv_file
 
 # Test files <= 1 block in size.  These are a bit of a special case since there
Signature · 1 line
2.31.0
#1 lore raw

Hi Ted,

Quoted 24 lines · Theodore Ts'o

This is needed to account for bash 5.1 adding line number annotation when a command like "bash -c /etc/passwd" fails, e.g., with

bash: line 1: /etc/passwd: Permission denied

instead of:

bash: /etc/passwd: Permission denied

---
 common/filter     | 11 +++++++++++
 tests/generic/572 | 33 +++++++++++++++++++--------------
 2 files changed, 30 insertions(+), 14 deletions(-)
common/filter +1 -0
index 2efbbd99..4b250e8b 100644
@@ -661,5 +661,16 @@ _filter_quota_report()
 		s|^(.*?) (\d+) (\d+) (\d+)|$1 @{[$2 * 1024 /'$bsize']} @{[$3 * 1024 /'$bsize']} @{[$4 * 1024 /'$bsize']}|'
 }
 
+#

common/filter _filter_quota_report() · line 674

Quoted 10 lines
common/filter +10 -0
+# Bash 5.1+ adds "line 1: " when printing an error running an executable
+# for example, "bash -c /etc/passwd" will result in the error
+# "bash: line 1: /etc/passwd: Permission denied" where as earlier
+# versions of bash will omit the "line 1: " annotation.
+#
+_filter_bash()
+{
+	sed -e "s/^bash: line 1: /bash: /"
+}
+

Doesn't this happen with any shell command passed to -c, not just commands that run an executable? In the commands below which are causing the problem, it is actually the 'echo' built-in being used, not an executable.

common/filter _filter_quota_report()

Quoted 5 lines
common/filter +2 -2
-bash -c "echo >> $fsv_file" |& _filter_scratch
+bash -c "echo >> $fsv_file" |& filter_output
 echo "* bash >, should be O_WRONLY|O_CREAT|O_TRUNC"
-bash -c "echo > $fsv_file" |& _filter_scratch
+bash -c "echo > $fsv_file" |& filter_output

Otherwise this patch looks good, thanks!

- Eric

#2 lore raw
Quoted 4 lines · Eric Biggers

Doesn't this happen with any shell command passed to -c, not just commands that run an executable? In the commands below which are causing the problem, it is actually the 'echo' built-in being used, not an executable.

Well.... it happens with all executables and *some* built-in commands which returns an error. (See the postscript for a case where the line number is printed.) In the case of an error opening the redirected standard output, such as permission denied, the difference does occur:

Compare:

% schroot -c buster-amd64 -- bash -c "echo foo >> /bin/bash"
bash: /bin/bash: Permission denied
% schroot -c buster-amd64 -- bash --version
GNU bash, version 5.0.3(1)-release (x86_64-pc-linux-gnu) Copyright (C) 2019 Free Software Foundation, Inc.
...

with

% schroot -c bullseye-amd64 -- bash -c "echo foo >> /bin/bash"
bash: line 1: /bin/bash: Permission denied
% schroot -c bullseye-amd64 -- bash --version
GNU bash, version 5.1.4(1)-release (x86_64-pc-linux-gnu)
...

I just used the example of "bash -c /etc/passwd" because it was simpler, and because it would be easy to verify regardless of whether the command was executed as root or not. For example:

% schroot -c bullseye-amd64 -u root -- bash -c "echo foo >> /bin/bash"

would have succeeded (and corrupted /bin/bash in my chroot :-).

I suppose I could have used the "schroot -c ..." example in the commit description, but that's a debian-specific command, and I chose to err on the side of something simpler and easier to replicate.

If Eryu would prefer, I can resend with a modified commit description, or he can feel free to edit the commit description with the above example if he thinks it's clearer.

Cheers,

- Ted

P.S. The 5.1 behavior is actually a bit more consistent, since other errors involving built-ins do actualy print a line number (although not a *consistent* line number between bash 5.0 and 5.1:

% schroot -c buster-amd64 -- bash -c "jobs --bad-option"
bash: line 0: jobs: --: invalid option
...

vs

% schroot -c bullseye-amd64 -- bash -c "jobs --bad-option"
bash: line 1: jobs: --: invalid option
...

So the changes in Bash 5.1 error reporting does have something to recommend itself in terms of self-consistency, but it is a bit of pain for xfstests, where any changes in output requires extra filter hacking!

#3 lore raw
Quoted 28 lines · Theodore Ts'o
Quoted 4 lines · Eric Biggers

Doesn't this happen with any shell command passed to -c, not just commands that run an executable? In the commands below which are causing the problem, it is actually the 'echo' built-in being used, not an executable.

Well.... it happens with all executables and *some* built-in commands which returns an error. (See the postscript for a case where the line number is printed.) In the case of an error opening the redirected standard output, such as permission denied, the difference does occur:

Compare:

% schroot -c buster-amd64 -- bash -c "echo foo >> /bin/bash"
bash: /bin/bash: Permission denied
% schroot -c buster-amd64 -- bash --version
GNU bash, version 5.0.3(1)-release (x86_64-pc-linux-gnu) Copyright (C) 2019 Free Software Foundation, Inc.
...

with

% schroot -c bullseye-amd64 -- bash -c "echo foo >> /bin/bash"
bash: line 1: /bin/bash: Permission denied
% schroot -c bullseye-amd64 -- bash --version
GNU bash, version 5.1.4(1)-release (x86_64-pc-linux-gnu)
...

Quoted 7 lines

I just used the example of "bash -c /etc/passwd" because it was simpler, and because it would be easy to verify regardless of whether the command was executed as root or not. For example:

% schroot -c bullseye-amd64 -u root -- bash -c "echo foo >> /bin/bash"

would have succeeded (and corrupted /bin/bash in my chroot :-).

A good example that doesn't involve an executable would be

bash -c "echo foo > /"

... since that command redirection will always fail.

Quoted 7 lines

I suppose I could have used the "schroot -c ..." example in the commit description, but that's a debian-specific command, and I chose to err on the side of something simpler and easier to replicate.

If Eryu would prefer, I can resend with a modified commit description, or he can feel free to edit the commit description with the above example if he thinks it's clearer.

I'm more concerned about the misleading comment above _filter_bash(), not the commit message which is less important. Anyway, it's not too important, but I thought it was worth pointing out.

- Eric

#4 lore raw

PATCH 1/1 common/filter: add _filter_bash()

This is needed to account for bash 5.1 adding line number annotation when executing a command via the -c option and it fails. For example, "bash -c 'echo foo > /'" will cause bash 5.1 to report:

bash: line 1: /: Is a directory

instead of:

bash: /: Is a directory

---
 common/filter     | 11 +++++++++++
 tests/generic/572 | 33 +++++++++++++++++++--------------
 2 files changed, 30 insertions(+), 14 deletions(-)
common/filter +11 -0
index 2efbbd99..c3db7a56 100644
@@ -661,5 +661,16 @@ _filter_quota_report()
 		s|^(.*?) (\d+) (\d+) (\d+)|$1 @{[$2 * 1024 /'$bsize']} @{[$3 * 1024 /'$bsize']} @{[$4 * 1024 /'$bsize']}|'
 }
 
+#
+# Bash 5.1+ adds "line 1: " when reporting an error when running a
+# command via the -c option.  For example, "bash -c 'echo foo > /'"
+# will result in the error "bash: line 1: /: Is a directory" when
+# earlier versions of bash would omit the "line 1: " annotation.
+#
+_filter_bash()
+{
+	sed -e "s/^bash: line 1: /bash: /"
+}
+
 # make sure this script returns success
 /bin/true
tests/generic/572 +19 -14
index f131c7ed..cded9ac6 100755
@@ -36,6 +36,11 @@ _scratch_mount
 fsv_orig_file=$SCRATCH_MNT/file
 fsv_file=$SCRATCH_MNT/file.fsv
 
+filter_output()
+{
+    _filter_bash | _filter_scratch
+}
+
 verify_data_readable()
 {
 	local file=$1
@@ -48,41 +53,41 @@ verify_data_unreadable()
 	local file=$1
 
 	# try both reading just the first data block, and reading until EOF
-	head -c $FSV_BLOCK_SIZE $file 2>&1 >/dev/null | _filter_scratch
-	md5sum $file |& _filter_scratch
+	head -c $FSV_BLOCK_SIZE $file 2>&1 >/dev/null | filter_output
+	md5sum $file |& filter_output
 }
 
 _fsv_scratch_begin_subtest "Enabling verity on file with verity already enabled fails with EEXIST"
 _fsv_create_enable_file $fsv_file
 echo "(trying again)"
-_fsv_enable $fsv_file |& _filter_scratch
+_fsv_enable $fsv_file |& filter_output
 
 _fsv_scratch_begin_subtest "Enabling verity with invalid hash algorithm fails with EINVAL"
-_fsv_create_enable_file $fsv_file --hash-alg=257 |& _filter_scratch
+_fsv_create_enable_file $fsv_file --hash-alg=257 |& filter_output
 verify_data_readable $fsv_file
 
 _fsv_scratch_begin_subtest "Enabling verity with invalid block size fails with EINVAL"
-_fsv_create_enable_file $fsv_file --block-size=1 |& _filter_scratch
+_fsv_create_enable_file $fsv_file --block-size=1 |& filter_output
 verify_data_readable $fsv_file
 
 _fsv_scratch_begin_subtest "Enabling verity on directory fails with EISDIR"
 mkdir $SCRATCH_MNT/dir
-_fsv_enable $SCRATCH_MNT/dir |& _filter_scratch
+_fsv_enable $SCRATCH_MNT/dir |& filter_output
 
 _fsv_scratch_begin_subtest "Enabling verity with too-long salt fails with EMSGSIZE"
-_fsv_create_enable_file $fsv_file --salt=$(perl -e 'print "A" x 1000') |& _filter_scratch
+_fsv_create_enable_file $fsv_file --salt=$(perl -e 'print "A" x 1000') |& filter_output
 verify_data_readable $fsv_file
 
 _fsv_scratch_begin_subtest "Enabling verity on file on read-only filesystem fails with EROFS"
 echo foo > $fsv_file
 _scratch_remount ro
-_fsv_enable $fsv_file |& _filter_scratch
+_fsv_enable $fsv_file |& filter_output
 _scratch_remount rw
 
 _fsv_scratch_begin_subtest "Enabling verity on file open for writing fails with ETXTBSY"
 echo foo > $fsv_file
 exec 3<> $fsv_file
-_fsv_enable $fsv_file |& _filter_scratch
+_fsv_enable $fsv_file |& filter_output
 exec 3<&-
 verify_data_readable $fsv_file
 
@@ -103,7 +108,7 @@ dd if=/dev/zero of=$fsv_file bs=1 count=0 seek=$((1 << 34)) status=none
 start_time=$(date +%s)
 $FSVERITY_PROG enable $fsv_file &
 sleep 0.5
-_fsv_enable $fsv_file |& _filter_scratch
+_fsv_enable $fsv_file |& filter_output
 kill %1
 wait
 
@@ -112,11 +117,11 @@ _fsv_create_enable_file $fsv_file >> $seqres.full
 echo "* reading"
 $XFS_IO_PROG -r $fsv_file -c ''
 echo "* xfs_io writing, should be O_RDWR"
-$XFS_IO_PROG $fsv_file -c '' |& _filter_scratch
+$XFS_IO_PROG $fsv_file -c '' |& filter_output
 echo "* bash >>, should be O_APPEND"
-bash -c "echo >> $fsv_file" |& _filter_scratch
+bash -c "echo >> $fsv_file" |& filter_output
 echo "* bash >, should be O_WRONLY|O_CREAT|O_TRUNC"
-bash -c "echo > $fsv_file" |& _filter_scratch
+bash -c "echo > $fsv_file" |& filter_output
 
 _fsv_scratch_begin_subtest "verity file can be read"
 _fsv_create_enable_file $fsv_file >> $seqres.full
@@ -160,7 +165,7 @@ _get_filesize $fsv_file
 
 _fsv_scratch_begin_subtest "Trying to measure non-verity file fails with ENODATA"
 echo foo > $fsv_file
-_fsv_measure $fsv_file |& _filter_scratch
+_fsv_measure $fsv_file |& filter_output
 verify_data_readable $fsv_file
 
 # Test files <= 1 block in size.  These are a bit of a special case since there
Signature · 1 line
2.31.0
Reviewed-by on v1 1/1
#5 lore raw
Quoted 8 lines · Theodore Ts'o

This is needed to account for bash 5.1 adding line number annotation when executing a command via the -c option and it fails. For example, "bash -c 'echo foo > /'" will cause bash 5.1 to report:

bash: line 1: /: Is a directory

instead of:

Quoted 7 lines

bash: /: Is a directory

---
 common/filter     | 11 +++++++++++
 tests/generic/572 | 33 +++++++++++++++++++--------------
 2 files changed, 30 insertions(+), 14 deletions(-)

- Eric