lib/oe/gpg_sign.py: Avoid race when creating .sig files in detach_sign
PATCH 1/1 lib/oe/gpg_sign.py: Avoid race when creating .sig files in detach_sign
Move the signature file into place only after it is successfully signed. This to avoid race and corrupted .sig files in cases multiple onging builds write to a shared sstate-cache dir.
--- meta/lib/oe/gpg_sign.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-)
meta/lib/oe/gpg_sign.py +15 -10
@@ -5,11 +5,12 @@ # """Helper module for GPG signing""" -import os import bb -import subprocess +import os import shlex +import subprocess +import tempfile class LocalSigner(object): """Class for handling local (on the build host) signing"""
@@ -73,8 +74,6 @@ class LocalSigner(object): cmd += ['--homedir', self.gpg_path] if armor: cmd += ['--armor'] - if output_suffix: - cmd += ['-o', input_file + "." + output_suffix] if use_sha256: cmd += ['--digest-algo', "SHA256"]
@@ -83,19 +82,25 @@ class LocalSigner(object): if self.gpg_version > (2,1,): cmd += ['--pinentry-mode', 'loopback'] - cmd += [input_file] - try: if passphrase_file: with open(passphrase_file) as fobj: passphrase = fobj.readline(); - job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) - (_, stderr) = job.communicate(passphrase.encode("utf-8")) + output_file = input_file + "." + (output_suffix or 'sig') + with tempfile.TemporaryDirectory(dir=os.path.dirname(output_file)) as tmp_dir: + tmp_file = os.path.join(tmp_dir, os.path.basename(output_file)) + cmd += ['-o', tmp_file] + + cmd += [input_file] + + job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + (_, stderr) = job.communicate(passphrase.encode("utf-8")) - if job.returncode: - bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8"))) + if job.returncode: + bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8"))) + os.rename(tmp_file, output_file) except IOError as e: bb.error("IO error (%s): %s" % (e.errno, e.strerror)) raise Exception("Failed to sign '%s'" % input_file)
Signature · 1 line
2.30.2
Quoted 41 lines · Tobias Hagelborn
Move the signature file into place only after it is successfully signed. This to avoid race and corrupted .sig files in cases multiple onging builds write to a shared sstate-cache dir.
Signed-off-by Tobias Hagelborn <[email protected]>--- meta/lib/oe/gpg_sign.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-)meta/lib/oe/gpg_sign.py +3
-4@@ -5,11 +5,12 @@ # """Helper module for GPG signing""" -import os import bb -import subprocess +import os import shlex +import subprocess +import tempfile class LocalSigner(object): """Class for handling local (on the build host) signing"""@@ -73,8 +74,6 @@ class LocalSigner(object): cmd += ['--homedir', self.gpg_path] if armor: cmd += ['--armor'] - if output_suffix: - cmd += ['-o', input_file + "." + output_suffix] if use_sha256: cmd += ['--digest-algo', "SHA256"]@@ -83,19 +82,25 @@ class LocalSigner(object): if self.gpg_version > (2,1,): cmd += ['--pinentry-mode', 'loopback']
meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 90
Quoted 10 lines
meta/lib/oe/gpg_sign.py +1
-4- cmd += [input_file] - try: if passphrase_file: with open(passphrase_file) as fobj: passphrase = fobj.readline(); - job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) - (_, stderr) = job.communicate(passphrase.encode("utf-8")) + output_file = input_file + "." + (output_suffix or 'sig')
This doesn't match the behaviour of output_suffix as used above where it defaults to None. This forces it to a default of "sig" instead?
If that intentional that should be in the commit message.
Quoted 8 lines
meta/lib/oe/gpg_sign.py +8
-0+ with tempfile.TemporaryDirectory(dir=os.path.dirname(output_file)) as tmp_dir: + tmp_file = os.path.join(tmp_dir, os.path.basename(output_file)) + cmd += ['-o', tmp_file] + + cmd += [input_file] + + job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + (_, stderr) = job.communicate(passphrase.encode("utf-8"))
meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 106
Quoted 10 lines
meta/lib/oe/gpg_sign.py +3
-2- if job.returncode: - bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8"))) + if job.returncode: + bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8"))) + os.rename(tmp_file, output_file) except IOError as e: bb.error("IO error (%s): %s" % (e.errno, e.strerror)) raise Exception("Failed to sign '%s'" % input_file)
Cheers,
Richard
A message that is not in the archive (sent off-list or not yet received): [email protected]
Quoted 73 lines · Richard Purdie via lists.openembedded.org
Quoted 41 lines · Tobias Hagelborn
Move the signature file into place only after it is successfully signed. This to avoid race and corrupted .sig files in cases multiple onging builds write to a shared sstate-cache dir.
Signed-off-by Tobias Hagelborn <[email protected]>--- meta/lib/oe/gpg_sign.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-)meta/lib/oe/gpg_sign.py +3
-4@@ -5,11 +5,12 @@ # """Helper module for GPG signing""" -import os import bb -import subprocess +import os import shlex +import subprocess +import tempfile class LocalSigner(object): """Class for handling local (on the build host) signing"""@@ -73,8 +74,6 @@ class LocalSigner(object): cmd += ['--homedir', self.gpg_path] if armor: cmd += ['--armor'] - if output_suffix: - cmd += ['-o', input_file + "." + output_suffix] if use_sha256: cmd += ['--digest-algo', "SHA256"]@@ -83,19 +82,25 @@ class LocalSigner(object): if self.gpg_version > (2,1,): cmd += ['--pinentry-mode', 'loopback']meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 90
Quoted 10 lines
meta/lib/oe/gpg_sign.py +1
-4- cmd += [input_file] - try: if passphrase_file: with open(passphrase_file) as fobj: passphrase = fobj.readline(); - job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) - (_, stderr) = job.communicate(passphrase.encode("utf-8")) + output_file = input_file + "." + (output_suffix or 'sig')This doesn't match the behaviour of
output_suffixas used above where it defaults to None. This forces it to a default of "sig" instead?If that intentional that should be in the commit message.
Quoted 14 lines
meta/lib/oe/gpg_sign.py +10
-2+ with tempfile.TemporaryDirectory(dir=os.path.dirname(output_file)) as tmp_dir: + tmp_file = os.path.join(tmp_dir, os.path.basename(output_file)) + cmd += ['-o', tmp_file] + + cmd += [input_file] + + job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + (_, stderr) = job.communicate(passphrase.encode("utf-8")) - if job.returncode: - bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8"))) + if job.returncode: + bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))
Quoted 4 lines
meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 106
Quoted 4 lines
meta/lib/oe/gpg_sign.py +1
-0+ os.rename(tmp_file, output_file) except IOError as e: bb.error("IO error (%s): %s" % (e.errno, e.strerror)) raise Exception("Failed to sign '%s'" % input_file)
I've been struggling to confirm it but have now done so. This does cause an oe-selftest failure as here:
https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/4950/steps/14/logs/stdio
i.e. from
oe-selftest -r runtime_test.TestImage.test_testimage_dnf
Cheers,
Richard
Thanks for the test feedback Richard!
I have not run the oe selftest properly on this one and will look in to the self-test results. (We have not used the signed rpm part for instance)
Cheers
Tobias
________________________________
From: Richard Purdie <[email protected]>
Sent: Thursday, March 30, 2023 11:10 AM
To: Tobias Hagelborn <[email protected]>; [email protected] <[email protected]>
Subject: Re: [OE-core] [PATCH] lib/oe/gpg_sign.py: Avoid race when creating .sig files in detach_sign
Quoted 73 lines · Richard Purdie via lists.openembedded.org
Quoted 41 lines · Tobias Hagelborn
Move the signature file into place only after it is successfully signed. This to avoid race and corrupted .sig files in cases multiple onging builds write to a shared sstate-cache dir.
Signed-off-by Tobias Hagelborn <[email protected]>--- meta/lib/oe/gpg_sign.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-)meta/lib/oe/gpg_sign.py +3
-4@@ -5,11 +5,12 @@ # """Helper module for GPG signing""" -import os import bb -import subprocess +import os import shlex +import subprocess +import tempfile class LocalSigner(object): """Class for handling local (on the build host) signing"""@@ -73,8 +74,6 @@ class LocalSigner(object): cmd += ['--homedir', self.gpg_path] if armor: cmd += ['--armor'] - if output_suffix: - cmd += ['-o', input_file + "." + output_suffix] if use_sha256: cmd += ['--digest-algo', "SHA256"]@@ -83,19 +82,25 @@ class LocalSigner(object): if self.gpg_version > (2,1,): cmd += ['--pinentry-mode', 'loopback']meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 90
Quoted 10 lines
meta/lib/oe/gpg_sign.py +1
-4- cmd += [input_file] - try: if passphrase_file: with open(passphrase_file) as fobj: passphrase = fobj.readline(); - job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) - (_, stderr) = job.communicate(passphrase.encode("utf-8")) + output_file = input_file + "." + (output_suffix or 'sig')This doesn't match the behaviour of
output_suffixas used above where it defaults to None. This forces it to a default of "sig" instead?If that intentional that should be in the commit message.
Quoted 14 lines
meta/lib/oe/gpg_sign.py +10
-2+ with tempfile.TemporaryDirectory(dir=os.path.dirname(output_file)) as tmp_dir: + tmp_file = os.path.join(tmp_dir, os.path.basename(output_file)) + cmd += ['-o', tmp_file] + + cmd += [input_file] + + job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + (_, stderr) = job.communicate(passphrase.encode("utf-8")) - if job.returncode: - bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8"))) + if job.returncode: + bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))
Quoted 4 lines
meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 106
Quoted 4 lines
meta/lib/oe/gpg_sign.py +1
-0+ os.rename(tmp_file, output_file) except IOError as e: bb.error("IO error (%s): %s" % (e.errno, e.strerror)) raise Exception("Failed to sign '%s'" % input_file)
I've been struggling to confirm it but have now done so. This does cause an oe-selftest failure as here:
https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/4950/steps/14/logs/stdio
i.e. from
oe-selftest -r runtime_test.TestImage.test_testimage_dnf
Cheers,
Richard
Quoted 83 lines · Richard Purdie
Quoted 73 lines · Richard Purdie via lists.openembedded.org
Quoted 41 lines · Tobias Hagelborn
Move the signature file into place only after it is successfully signed. This to avoid race and corrupted .sig files in cases multiple onging builds write to a shared sstate-cache dir.
Signed-off-by Tobias Hagelborn <[email protected]>--- meta/lib/oe/gpg_sign.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-)meta/lib/oe/gpg_sign.py +3
-4@@ -5,11 +5,12 @@ # """Helper module for GPG signing""" -import os import bb -import subprocess +import os import shlex +import subprocess +import tempfile class LocalSigner(object): """Class for handling local (on the build host) signing"""@@ -73,8 +74,6 @@ class LocalSigner(object): cmd += ['--homedir', self.gpg_path] if armor: cmd += ['--armor'] - if output_suffix: - cmd += ['-o', input_file + "." + output_suffix] if use_sha256: cmd += ['--digest-algo', "SHA256"]@@ -83,19 +82,25 @@ class LocalSigner(object): if self.gpg_version > (2,1,): cmd += ['--pinentry-mode', 'loopback']meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 90
Quoted 10 lines
meta/lib/oe/gpg_sign.py +1
-4- cmd += [input_file] - try: if passphrase_file: with open(passphrase_file) as fobj: passphrase = fobj.readline(); - job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) - (_, stderr) = job.communicate(passphrase.encode("utf-8")) + output_file = input_file + "." + (output_suffix or 'sig')This doesn't match the behaviour of
output_suffixas used above where it defaults to None. This forces it to a default of "sig" instead?If that intentional that should be in the commit message.
Quoted 14 lines
meta/lib/oe/gpg_sign.py +10
-2+ with tempfile.TemporaryDirectory(dir=os.path.dirname(output_file)) as tmp_dir: + tmp_file = os.path.join(tmp_dir, os.path.basename(output_file)) + cmd += ['-o', tmp_file] + + cmd += [input_file] + + job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + (_, stderr) = job.communicate(passphrase.encode("utf-8")) - if job.returncode: - bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8"))) + if job.returncode: + bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))Quoted 4 lines
meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 106
Quoted 4 lines
meta/lib/oe/gpg_sign.py +1
-0+ os.rename(tmp_file, output_file) except IOError as e: bb.error("IO error (%s): %s" % (e.errno, e.strerror)) raise Exception("Failed to sign '%s'" % input_file)I've been struggling to confirm it but have now done so. This does cause an oe-selftest failure as here:
Quoted 7 lines
https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/4950/steps/14/logs/stdio
i.e. from
oe-selftest -r runtime_test.TestImage.test_testimage_dnf
I realize just now that I forgot to send an email telling exactly that, sorry!
Quoted 15 lines
Cheers,
Richard
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#179308): https://lists.openembedded.org/g/openembedded-core/message/179308
Mute This Topic: https://lists.openembedded.org/mt/97797700/3617179
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin Embedded Linux and Kernel engineering https://bootlin.com
A message that is not in the archive (sent off-list or not yet received): [email protected]
Quoted 83 lines · Richard Purdie via lists.openembedded.org
Quoted 73 lines · Richard Purdie via lists.openembedded.org
Quoted 41 lines · Tobias Hagelborn
Move the signature file into place only after it is successfully signed. This to avoid race and corrupted .sig files in cases multiple onging builds write to a shared sstate-cache dir.
Signed-off-by Tobias Hagelborn <[email protected]>--- meta/lib/oe/gpg_sign.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-)meta/lib/oe/gpg_sign.py +3
-4@@ -5,11 +5,12 @@ # """Helper module for GPG signing""" -import os import bb -import subprocess +import os import shlex +import subprocess +import tempfile class LocalSigner(object): """Class for handling local (on the build host) signing"""@@ -73,8 +74,6 @@ class LocalSigner(object): cmd += ['--homedir', self.gpg_path] if armor: cmd += ['--armor'] - if output_suffix: - cmd += ['-o', input_file + "." + output_suffix] if use_sha256: cmd += ['--digest-algo', "SHA256"]@@ -83,19 +82,25 @@ class LocalSigner(object): if self.gpg_version > (2,1,): cmd += ['--pinentry-mode', 'loopback']meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 90
Quoted 10 lines
meta/lib/oe/gpg_sign.py +1
-4- cmd += [input_file] - try: if passphrase_file: with open(passphrase_file) as fobj: passphrase = fobj.readline(); - job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) - (_, stderr) = job.communicate(passphrase.encode("utf-8")) + output_file = input_file + "." + (output_suffix or 'sig')This doesn't match the behaviour of
output_suffixas used above where it defaults to None. This forces it to a default of "sig" instead?If that intentional that should be in the commit message.
Quoted 14 lines
meta/lib/oe/gpg_sign.py +10
-2+ with tempfile.TemporaryDirectory(dir=os.path.dirname(output_file)) as tmp_dir: + tmp_file = os.path.join(tmp_dir, os.path.basename(output_file)) + cmd += ['-o', tmp_file] + + cmd += [input_file] + + job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + (_, stderr) = job.communicate(passphrase.encode("utf-8")) - if job.returncode: - bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8"))) + if job.returncode: + bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))Quoted 4 lines
meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 106
Quoted 4 lines
meta/lib/oe/gpg_sign.py +1
-0+ os.rename(tmp_file, output_file) except IOError as e: bb.error("IO error (%s): %s" % (e.errno, e.strerror)) raise Exception("Failed to sign '%s'" % input_file)I've been struggling to confirm it but have now done so. This does cause an oe-selftest failure as here:
Quoted 6 lines
https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/4950/steps/14/logs/stdio
i.e. from
oe-selftest -r runtime_test.TestImage.test_testimage_dnf
I should also mention, this test fails with oe-core's nodistro for other reasons but does work ok with poky. I filed a bug for that:
https://bugzilla.yoctoproject.org/show_bug.cgi?id=15086
since it was confusing my debugging! You'd therefore need to use poky for testing your change until we fix that.
Cheers,
Richard