Skip to contentSkip to conversation

lib/oe/gpg_sign.py: Avoid race when creating .sig files in detach_sign

PATCH In review Tobias Hagelborn 4 participants 6 messages Mar 23, 2023 – Mar 30, 2023 2 messages not in the archive

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 lib/oe/gpg_sign.py: Avoid race when creating .sig files in detach_sign

Move the signature file into place only after it is successfully signed. This to avoid race and corrupted .sig files in cases multiple onging builds write to a shared sstate-cache dir.

---
 meta/lib/oe/gpg_sign.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)
meta/lib/oe/gpg_sign.py +15 -10
index 613dab8561..868846cdc5 100644
@@ -5,11 +5,12 @@
 #
 
 """Helper module for GPG signing"""
-import os
 
 import bb
-import subprocess
+import os
 import shlex
+import subprocess
+import tempfile
 
 class LocalSigner(object):
     """Class for handling local (on the build host) signing"""
@@ -73,8 +74,6 @@ class LocalSigner(object):
             cmd += ['--homedir', self.gpg_path]
         if armor:
             cmd += ['--armor']
-        if output_suffix:
-            cmd += ['-o', input_file + "." + output_suffix]
         if use_sha256:
             cmd += ['--digest-algo', "SHA256"]
 
@@ -83,19 +82,25 @@ class LocalSigner(object):
         if self.gpg_version > (2,1,):
             cmd += ['--pinentry-mode', 'loopback']
 
-        cmd += [input_file]
-
         try:
             if passphrase_file:
                 with open(passphrase_file) as fobj:
                     passphrase = fobj.readline();
 
-            job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
-            (_, stderr) = job.communicate(passphrase.encode("utf-8"))
+            output_file = input_file + "." + (output_suffix or 'sig')
+            with tempfile.TemporaryDirectory(dir=os.path.dirname(output_file)) as tmp_dir:
+                tmp_file = os.path.join(tmp_dir, os.path.basename(output_file))
+                cmd += ['-o', tmp_file]
+
+                cmd += [input_file]
+
+                job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+                (_, stderr) = job.communicate(passphrase.encode("utf-8"))
 
-            if job.returncode:
-                bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))
+                if job.returncode:
+                    bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))
 
+                os.rename(tmp_file, output_file)
         except IOError as e:
             bb.error("IO error (%s): %s" % (e.errno, e.strerror))
             raise Exception("Failed to sign '%s'" % input_file)
Signature · 1 line
2.30.2
#1 lore raw
Quoted 41 lines · Tobias Hagelborn

Move the signature file into place only after it is successfully signed. This to avoid race and corrupted .sig files in cases multiple onging builds write to a shared sstate-cache dir.

---
 meta/lib/oe/gpg_sign.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)
meta/lib/oe/gpg_sign.py +3 -4
index 613dab8561..868846cdc5 100644
@@ -5,11 +5,12 @@
 #
 
 """Helper module for GPG signing"""
-import os
 
 import bb
-import subprocess
+import os
 import shlex
+import subprocess
+import tempfile
 
 class LocalSigner(object):
     """Class for handling local (on the build host) signing"""
@@ -73,8 +74,6 @@ class LocalSigner(object):
             cmd += ['--homedir', self.gpg_path]
         if armor:
             cmd += ['--armor']
-        if output_suffix:
-            cmd += ['-o', input_file + "." + output_suffix]
         if use_sha256:
             cmd += ['--digest-algo', "SHA256"]
 
@@ -83,19 +82,25 @@ class LocalSigner(object):
         if self.gpg_version > (2,1,):
             cmd += ['--pinentry-mode', 'loopback']
 

meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 90

Quoted 10 lines
meta/lib/oe/gpg_sign.py +1 -4
-        cmd += [input_file]
-
         try:
             if passphrase_file:
                 with open(passphrase_file) as fobj:
                     passphrase = fobj.readline();
 
-            job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
-            (_, stderr) = job.communicate(passphrase.encode("utf-8"))
+            output_file = input_file + "." + (output_suffix or 'sig')

This doesn't match the behaviour of output_suffix as used above where it defaults to None. This forces it to a default of "sig" instead?

If that intentional that should be in the commit message.

Quoted 8 lines
meta/lib/oe/gpg_sign.py +8 -0
+            with tempfile.TemporaryDirectory(dir=os.path.dirname(output_file)) as tmp_dir:
+                tmp_file = os.path.join(tmp_dir, os.path.basename(output_file))
+                cmd += ['-o', tmp_file]
+
+                cmd += [input_file]
+
+                job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+                (_, stderr) = job.communicate(passphrase.encode("utf-8"))

meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 106

Quoted 10 lines
meta/lib/oe/gpg_sign.py +3 -2
 
-            if job.returncode:
-                bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))
+                if job.returncode:
+                    bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))
 
+                os.rename(tmp_file, output_file)
         except IOError as e:
             bb.error("IO error (%s): %s" % (e.errno, e.strerror))
             raise Exception("Failed to sign '%s'" % input_file)

Cheers,

Richard

A message that is not in the archive (sent off-list or not yet received): [email protected]

#3 lore raw
Quoted 73 lines · Richard Purdie via lists.openembedded.org
Quoted 41 lines · Tobias Hagelborn

Move the signature file into place only after it is successfully signed. This to avoid race and corrupted .sig files in cases multiple onging builds write to a shared sstate-cache dir.

---
 meta/lib/oe/gpg_sign.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)
meta/lib/oe/gpg_sign.py +3 -4
index 613dab8561..868846cdc5 100644
@@ -5,11 +5,12 @@
 #
 
 """Helper module for GPG signing"""
-import os
 
 import bb
-import subprocess
+import os
 import shlex
+import subprocess
+import tempfile
 
 class LocalSigner(object):
     """Class for handling local (on the build host) signing"""
@@ -73,8 +74,6 @@ class LocalSigner(object):
             cmd += ['--homedir', self.gpg_path]
         if armor:
             cmd += ['--armor']
-        if output_suffix:
-            cmd += ['-o', input_file + "." + output_suffix]
         if use_sha256:
             cmd += ['--digest-algo', "SHA256"]
 
@@ -83,19 +82,25 @@ class LocalSigner(object):
         if self.gpg_version > (2,1,):
             cmd += ['--pinentry-mode', 'loopback']
 

meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 90

Quoted 10 lines
meta/lib/oe/gpg_sign.py +1 -4
-        cmd += [input_file]
-
         try:
             if passphrase_file:
                 with open(passphrase_file) as fobj:
                     passphrase = fobj.readline();
 
-            job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
-            (_, stderr) = job.communicate(passphrase.encode("utf-8"))
+            output_file = input_file + "." + (output_suffix or 'sig')

This doesn't match the behaviour of output_suffix as used above where it defaults to None. This forces it to a default of "sig" instead?

If that intentional that should be in the commit message.

Quoted 14 lines
meta/lib/oe/gpg_sign.py +10 -2
+            with tempfile.TemporaryDirectory(dir=os.path.dirname(output_file)) as tmp_dir:
+                tmp_file = os.path.join(tmp_dir, os.path.basename(output_file))
+                cmd += ['-o', tmp_file]
+
+                cmd += [input_file]
+
+                job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+                (_, stderr) = job.communicate(passphrase.encode("utf-8"))
 
-            if job.returncode:
-                bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))
+                if job.returncode:
+                    bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))
 
Quoted 4 lines

meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 106

Quoted 4 lines
meta/lib/oe/gpg_sign.py +1 -0
+                os.rename(tmp_file, output_file)
         except IOError as e:
             bb.error("IO error (%s): %s" % (e.errno, e.strerror))
             raise Exception("Failed to sign '%s'" % input_file)

I've been struggling to confirm it but have now done so. This does cause an oe-selftest failure as here:

https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/4950/steps/14/logs/stdio

i.e. from

oe-selftest -r runtime_test.TestImage.test_testimage_dnf

Cheers,

Richard

#4 lore raw

Thanks for the test feedback Richard!

I have not run the oe selftest properly on this one and will look in to the self-test results. (We have not used the signed rpm part for instance)

Cheers
Tobias

________________________________
From: Richard Purdie <[email protected]>
Sent: Thursday, March 30, 2023 11:10 AM
To: Tobias Hagelborn <[email protected]>; [email protected] <[email protected]>
Subject: Re: [OE-core] [PATCH] lib/oe/gpg_sign.py: Avoid race when creating .sig files in detach_sign

Quoted 73 lines · Richard Purdie via lists.openembedded.org
Quoted 41 lines · Tobias Hagelborn

Move the signature file into place only after it is successfully signed. This to avoid race and corrupted .sig files in cases multiple onging builds write to a shared sstate-cache dir.

---
 meta/lib/oe/gpg_sign.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)
meta/lib/oe/gpg_sign.py +3 -4
index 613dab8561..868846cdc5 100644
@@ -5,11 +5,12 @@
 #
 
 """Helper module for GPG signing"""
-import os
 
 import bb
-import subprocess
+import os
 import shlex
+import subprocess
+import tempfile
 
 class LocalSigner(object):
     """Class for handling local (on the build host) signing"""
@@ -73,8 +74,6 @@ class LocalSigner(object):
             cmd += ['--homedir', self.gpg_path]
         if armor:
             cmd += ['--armor']
-        if output_suffix:
-            cmd += ['-o', input_file + "." + output_suffix]
         if use_sha256:
             cmd += ['--digest-algo', "SHA256"]
 
@@ -83,19 +82,25 @@ class LocalSigner(object):
         if self.gpg_version > (2,1,):
             cmd += ['--pinentry-mode', 'loopback']
 

meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 90

Quoted 10 lines
meta/lib/oe/gpg_sign.py +1 -4
-        cmd += [input_file]
-
         try:
             if passphrase_file:
                 with open(passphrase_file) as fobj:
                     passphrase = fobj.readline();
 
-            job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
-            (_, stderr) = job.communicate(passphrase.encode("utf-8"))
+            output_file = input_file + "." + (output_suffix or 'sig')

This doesn't match the behaviour of output_suffix as used above where it defaults to None. This forces it to a default of "sig" instead?

If that intentional that should be in the commit message.

Quoted 14 lines
meta/lib/oe/gpg_sign.py +10 -2
+            with tempfile.TemporaryDirectory(dir=os.path.dirname(output_file)) as tmp_dir:
+                tmp_file = os.path.join(tmp_dir, os.path.basename(output_file))
+                cmd += ['-o', tmp_file]
+
+                cmd += [input_file]
+
+                job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+                (_, stderr) = job.communicate(passphrase.encode("utf-8"))
 
-            if job.returncode:
-                bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))
+                if job.returncode:
+                    bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))
 
Quoted 4 lines

meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 106

Quoted 4 lines
meta/lib/oe/gpg_sign.py +1 -0
+                os.rename(tmp_file, output_file)
         except IOError as e:
             bb.error("IO error (%s): %s" % (e.errno, e.strerror))
             raise Exception("Failed to sign '%s'" % input_file)

I've been struggling to confirm it but have now done so. This does cause an oe-selftest failure as here:

https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/4950/steps/14/logs/stdio

i.e. from

oe-selftest -r runtime_test.TestImage.test_testimage_dnf

Cheers,

Richard

#5 lore raw
In reply to Richard Purdie
Quoted 83 lines · Richard Purdie
Quoted 73 lines · Richard Purdie via lists.openembedded.org
Quoted 41 lines · Tobias Hagelborn

Move the signature file into place only after it is successfully signed. This to avoid race and corrupted .sig files in cases multiple onging builds write to a shared sstate-cache dir.

---
 meta/lib/oe/gpg_sign.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)
meta/lib/oe/gpg_sign.py +3 -4
index 613dab8561..868846cdc5 100644
@@ -5,11 +5,12 @@
 #
 
 """Helper module for GPG signing"""
-import os
 
 import bb
-import subprocess
+import os
 import shlex
+import subprocess
+import tempfile
 
 class LocalSigner(object):
     """Class for handling local (on the build host) signing"""
@@ -73,8 +74,6 @@ class LocalSigner(object):
             cmd += ['--homedir', self.gpg_path]
         if armor:
             cmd += ['--armor']
-        if output_suffix:
-            cmd += ['-o', input_file + "." + output_suffix]
         if use_sha256:
             cmd += ['--digest-algo', "SHA256"]
 
@@ -83,19 +82,25 @@ class LocalSigner(object):
         if self.gpg_version > (2,1,):
             cmd += ['--pinentry-mode', 'loopback']
 

meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 90

Quoted 10 lines
meta/lib/oe/gpg_sign.py +1 -4
-        cmd += [input_file]
-
         try:
             if passphrase_file:
                 with open(passphrase_file) as fobj:
                     passphrase = fobj.readline();
 
-            job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
-            (_, stderr) = job.communicate(passphrase.encode("utf-8"))
+            output_file = input_file + "." + (output_suffix or 'sig')

This doesn't match the behaviour of output_suffix as used above where it defaults to None. This forces it to a default of "sig" instead?

If that intentional that should be in the commit message.

Quoted 14 lines
meta/lib/oe/gpg_sign.py +10 -2
+            with tempfile.TemporaryDirectory(dir=os.path.dirname(output_file)) as tmp_dir:
+                tmp_file = os.path.join(tmp_dir, os.path.basename(output_file))
+                cmd += ['-o', tmp_file]
+
+                cmd += [input_file]
+
+                job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+                (_, stderr) = job.communicate(passphrase.encode("utf-8"))
 
-            if job.returncode:
-                bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))
+                if job.returncode:
+                    bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))
 
Quoted 4 lines

meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 106

Quoted 4 lines
meta/lib/oe/gpg_sign.py +1 -0
+                os.rename(tmp_file, output_file)
         except IOError as e:
             bb.error("IO error (%s): %s" % (e.errno, e.strerror))
             raise Exception("Failed to sign '%s'" % input_file)

I've been struggling to confirm it but have now done so. This does cause an oe-selftest failure as here:

Quoted 7 lines

https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/4950/steps/14/logs/stdio

i.e. from

oe-selftest -r runtime_test.TestImage.test_testimage_dnf

I realize just now that I forgot to send an email telling exactly that, sorry!

Quoted 15 lines

Cheers,

Richard

-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#179308): https://lists.openembedded.org/g/openembedded-core/message/179308
Mute This Topic: https://lists.openembedded.org/mt/97797700/3617179
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

Signature · 3 lines
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

A message that is not in the archive (sent off-list or not yet received): [email protected]

#7 lore raw
Quoted 83 lines · Richard Purdie via lists.openembedded.org
Quoted 73 lines · Richard Purdie via lists.openembedded.org
Quoted 41 lines · Tobias Hagelborn

Move the signature file into place only after it is successfully signed. This to avoid race and corrupted .sig files in cases multiple onging builds write to a shared sstate-cache dir.

---
 meta/lib/oe/gpg_sign.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)
meta/lib/oe/gpg_sign.py +3 -4
index 613dab8561..868846cdc5 100644
@@ -5,11 +5,12 @@
 #
 
 """Helper module for GPG signing"""
-import os
 
 import bb
-import subprocess
+import os
 import shlex
+import subprocess
+import tempfile
 
 class LocalSigner(object):
     """Class for handling local (on the build host) signing"""
@@ -73,8 +74,6 @@ class LocalSigner(object):
             cmd += ['--homedir', self.gpg_path]
         if armor:
             cmd += ['--armor']
-        if output_suffix:
-            cmd += ['-o', input_file + "." + output_suffix]
         if use_sha256:
             cmd += ['--digest-algo', "SHA256"]
 
@@ -83,19 +82,25 @@ class LocalSigner(object):
         if self.gpg_version > (2,1,):
             cmd += ['--pinentry-mode', 'loopback']
 

meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 90

Quoted 10 lines
meta/lib/oe/gpg_sign.py +1 -4
-        cmd += [input_file]
-
         try:
             if passphrase_file:
                 with open(passphrase_file) as fobj:
                     passphrase = fobj.readline();
 
-            job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
-            (_, stderr) = job.communicate(passphrase.encode("utf-8"))
+            output_file = input_file + "." + (output_suffix or 'sig')

This doesn't match the behaviour of output_suffix as used above where it defaults to None. This forces it to a default of "sig" instead?

If that intentional that should be in the commit message.

Quoted 14 lines
meta/lib/oe/gpg_sign.py +10 -2
+            with tempfile.TemporaryDirectory(dir=os.path.dirname(output_file)) as tmp_dir:
+                tmp_file = os.path.join(tmp_dir, os.path.basename(output_file))
+                cmd += ['-o', tmp_file]
+
+                cmd += [input_file]
+
+                job = subprocess.Popen(cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+                (_, stderr) = job.communicate(passphrase.encode("utf-8"))
 
-            if job.returncode:
-                bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))
+                if job.returncode:
+                    bb.fatal("GPG exited with code %d: %s" % (job.returncode, stderr.decode("utf-8")))
 
Quoted 4 lines

meta/lib/oe/gpg_sign.py class LocalSigner(object): · line 106

Quoted 4 lines
meta/lib/oe/gpg_sign.py +1 -0
+                os.rename(tmp_file, output_file)
         except IOError as e:
             bb.error("IO error (%s): %s" % (e.errno, e.strerror))
             raise Exception("Failed to sign '%s'" % input_file)

I've been struggling to confirm it but have now done so. This does cause an oe-selftest failure as here:

Quoted 6 lines

https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/4950/steps/14/logs/stdio

i.e. from

oe-selftest -r runtime_test.TestImage.test_testimage_dnf

I should also mention, this test fails with oe-core's nodistro for other reasons but does work ok with poky. I filed a bug for that:

https://bugzilla.yoctoproject.org/show_bug.cgi?id=15086

since it was confusing my debugging! You'd therefore need to use poky for testing your change until we fix that.

Cheers,

Richard