Skip to contentSkip to conversation

package_manager: install versioned postinst scripts

PATCH In review Anton Kachalov 3 participants 6 messages Mar 22, 2021 – Mar 23, 2021 1 message not in the archive

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 package_manager: install versioned postinst scripts

From: "Anton D. Kachalov" <[email protected]>

When running on read-only rootfs backed with overlayfs, the processed scripts are being marked as deleted on upperdir of overlayfs. When it comes to upgrade the main read-only image, it might contain new postinst scripts that are different from the already processed. Introduced suffix (sha256 of the content) allows to distinct updated scripts of the same package.

---
 meta/lib/oe/package_manager/deb/rootfs.py   | 11 ++++++++++-
 meta/lib/oe/package_manager/ipk/rootfs.py   | 11 ++++++++++-
 meta/lib/oe/package_manager/rpm/__init__.py |  4 +++-
 3 files changed, 23 insertions(+), 3 deletions(-)
meta/lib/oe/package_manager/deb/rootfs.py +10 -1
index 8fbaca11d6..704d213626 100644
@@ -4,6 +4,7 @@
 
 import re
 import shutil
+from hashlib import sha256
 from oe.rootfs import Rootfs
 from oe.manifest import Manifest
 from oe.utils import execute_pre_post_process
@@ -115,8 +116,16 @@ class DpkgOpkgRootfs(Rootfs):
             bb.utils.mkdirhier(dst_postinst_dir)
 
             if os.path.exists(os.path.join(src_postinst_dir, p + ".postinst")):
+                csum = sha256()
+                with open(src_postinst_dir, p + ".postinst", "rb") as f:
+                    while True:
+                        data = f.read(65536)
+                        if not data:
+                            break
+                        csum.update(data)
+
                 shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"),
-                            os.path.join(dst_postinst_dir, "%03d-%s" % (num, p)))
+                            os.path.join(dst_postinst_dir, "%03d-%s.%s" % (num, p, csum.hexdigest())))
 
             num += 1
 
meta/lib/oe/package_manager/ipk/rootfs.py +10 -1
index 26dbee6f6a..6ead6ddef3 100644
@@ -5,6 +5,7 @@
 import re
 import filecmp
 import shutil
+from hashlib import sha256
 from oe.rootfs import Rootfs
 from oe.manifest import Manifest
 from oe.utils import execute_pre_post_process
@@ -116,8 +117,16 @@ class DpkgOpkgRootfs(Rootfs):
             bb.utils.mkdirhier(dst_postinst_dir)
 
             if os.path.exists(os.path.join(src_postinst_dir, p + ".postinst")):
+                csum = sha256()
+                with open(src_postinst_dir, p + ".postinst", "rb") as f:
+                    while True:
+                        data = f.read(65536)
+                        if not data:
+                            break
+                        csum.update(data)
+
                 shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"),
-                            os.path.join(dst_postinst_dir, "%03d-%s" % (num, p)))
+                            os.path.join(dst_postinst_dir, "%03d-%s.%s" % (num, p, csum.hexdigest())))
 
             num += 1
 
meta/lib/oe/package_manager/rpm/__init__.py +3 -1
index 6df0092281..4746e96ce8 100644
@@ -4,6 +4,7 @@
 
 import shutil
 import subprocess
+from hashlib import sha256
 from oe.package_manager import *
 
 class RpmIndexer(Indexer):
@@ -359,7 +360,8 @@ class RpmPM(PackageManager):
         target_path = oe.path.join(self.target_rootfs, self.d.expand('${sysconfdir}/rpm-postinsts/'))
         bb.utils.mkdirhier(target_path)
         num = self._script_num_prefix(target_path)
-        saved_script_name = oe.path.join(target_path, "%d-%s" % (num, pkg))
+        csum = sha256(output.encode("utf-8")).hexdigest()
+        saved_script_name = oe.path.join(target_path, "%d-%s.%s" % (num, pkg, csum))
         open(saved_script_name, 'w').write(output)
         os.chmod(saved_script_name, 0o755)
 
Signature · 1 line
2.31.0.rc2.261.g7f71774620-goog
#1 lore raw
Quoted 8 lines · Anton Kachalov via lists.openembedded.org

From: "Anton D. Kachalov" <[email protected]>

When running on read-only rootfs backed with overlayfs, the processed scripts are being marked as deleted on upperdir of overlayfs. When it comes to upgrade the main read-only image, it might contain new postinst scripts that are different from the already processed. Introduced suffix (sha256 of the content) allows to distinct updated scripts of the same package.

Quoted 6 lines

---

 meta/lib/oe/package_manager/deb/rootfs.py   | 11 ++++++++++-
 meta/lib/oe/package_manager/ipk/rootfs.py   | 11 ++++++++++-
 meta/lib/oe/package_manager/rpm/__init__.py |  4 +++-
 3 files changed, 23 insertions(+), 3 deletions(-)

Isn't this working around an issue with overlayfs?

I'm a bit worried we're adding complexity to all users of the project for a very specific use case where the issue is really somewhere else?

Cheers,

Richard

#2 lore raw
In reply to Anton Kachalov

Two remarks below...

Met vriendelijke groet / kind regards,

Mike Looijmans
System Expert

TOPIC Embedded Products B.V.
Materiaalweg 4, 5681 RJ Best
The Netherlands

T: +31 (0) 499 33 69 69
E: [email protected]
W: www.topic.nl

Please consider the environment before printing this e-mail

Quoted 7 lines · Anton Kachalov via lists.openembedded.org

From: "Anton D. Kachalov" <[email protected]>

When running on read-only rootfs backed with overlayfs, the processed scripts are being marked as deleted on upperdir of overlayfs. When it comes to upgrade the main read-only image, it might contain new postinst scripts that are different from the already processed. Introduced suffix (sha256 of the content) allows to distinct updated scripts of the same package.

So your usecase is that you have a read-only rootfs with an overlay on top, and then just replace the rootfs. And then expect things to still work?

That doesn't sound valid to me - normally I'd wipe the overlay clean when updating the underlying read-only part. I'd be very interested in what you're actually trying to accomplish here?

And further, I don't think appending 64 characters to filenames is going to make anyone happy.

In your case, I'd opt for a script that just resets the postinst scripts (and maybe more?) in the overlay. Or have 3 overlays. Whenever the package manager runs, activate the second layer. The "user" stuff in the third layer is normally active. Then when you upgrade the system, wipe the second layer, to get rid of obsolete packages. Or something like that. Again, tell us what you're really building...

Quoted 93 lines
---
  meta/lib/oe/package_manager/deb/rootfs.py   | 11 ++++++++++-
  meta/lib/oe/package_manager/ipk/rootfs.py   | 11 ++++++++++-
  meta/lib/oe/package_manager/rpm/__init__.py |  4 +++-
  3 files changed, 23 insertions(+), 3 deletions(-)
meta/lib/oe/package_manager/deb/rootfs.py +10 -1
index 8fbaca11d6..704d213626 100644
@@ -4,6 +4,7 @@
 
  import re
  import shutil
+from hashlib import sha256
  from oe.rootfs import Rootfs
  from oe.manifest import Manifest
  from oe.utils import execute_pre_post_process
@@ -115,8 +116,16 @@ class DpkgOpkgRootfs(Rootfs):
              bb.utils.mkdirhier(dst_postinst_dir)
 
              if os.path.exists(os.path.join(src_postinst_dir, p + ".postinst")):
+                csum = sha256()
+                with open(src_postinst_dir, p + ".postinst", "rb") as f:
+                    while True:
+                        data = f.read(65536)
+                        if not data:
+                            break
+                        csum.update(data)
+
                  shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"),
-                            os.path.join(dst_postinst_dir, "%03d-%s" % (num, p)))
+                            os.path.join(dst_postinst_dir, "%03d-%s.%s" % (num, p, csum.hexdigest())))
 
              num += 1
 
meta/lib/oe/package_manager/ipk/rootfs.py +10 -1
index 26dbee6f6a..6ead6ddef3 100644
@@ -5,6 +5,7 @@
  import re
  import filecmp
  import shutil
+from hashlib import sha256
  from oe.rootfs import Rootfs
  from oe.manifest import Manifest
  from oe.utils import execute_pre_post_process
@@ -116,8 +117,16 @@ class DpkgOpkgRootfs(Rootfs):
              bb.utils.mkdirhier(dst_postinst_dir)
 
              if os.path.exists(os.path.join(src_postinst_dir, p + ".postinst")):
+                csum = sha256()
+                with open(src_postinst_dir, p + ".postinst", "rb") as f:
+                    while True:
+                        data = f.read(65536)
+                        if not data:
+                            break
+                        csum.update(data)
+
                  shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"),
-                            os.path.join(dst_postinst_dir, "%03d-%s" % (num, p)))
+                            os.path.join(dst_postinst_dir, "%03d-%s.%s" % (num, p, csum.hexdigest())))
 
              num += 1
 
meta/lib/oe/package_manager/rpm/__init__.py +3 -1
index 6df0092281..4746e96ce8 100644
@@ -4,6 +4,7 @@
 
  import shutil
  import subprocess
+from hashlib import sha256
  from oe.package_manager import *
 
  class RpmIndexer(Indexer):
@@ -359,7 +360,8 @@ class RpmPM(PackageManager):
          target_path = oe.path.join(self.target_rootfs, self.d.expand('${sysconfdir}/rpm-postinsts/'))
          bb.utils.mkdirhier(target_path)
          num = self._script_num_prefix(target_path)
-        saved_script_name = oe.path.join(target_path, "%d-%s" % (num, pkg))
+        csum = sha256(output.encode("utf-8")).hexdigest()
+        saved_script_name = oe.path.join(target_path, "%d-%s.%s" % (num, pkg, csum))
          open(saved_script_name, 'w').write(output)
          os.chmod(saved_script_name, 0o755)
 
Signature · 1 line
Mike Looijmans
#3 lore raw

Hi, Mike.

That is how OpenBMC is designed. Most of the targets are read-only rootfs (squashfs) and overlayfs for RW.
It is not expected to overwrite binaries through overlayfs and there are plans to scope RW areas (e.g. /etc).

Those files with 64-chars suffix are mostly hidden from regular users: the files are removed during the first startup. No one will notice them :)

Anyway, alternatively within the openbmc update script it's possible to force clean the deleted files on overlayfs upperdir. However, it will mean that all the scripts regardless if they are updated or not, will be re-executed after rootfs update.

Quoted 160 lines · Mike Looijmans

Two remarks below...

Met vriendelijke groet / kind regards,

Mike Looijmans
System Expert

TOPIC Embedded Products B.V.
Materiaalweg 4, 5681 RJ Best
The Netherlands

T: +31 (0) 499 33 69 69 <+31%20499%20336%20969>
E: [email protected]
W: www.topic.nl

Please consider the environment before printing this e-mail

Quoted 3 lines · Anton Kachalov via lists.openembedded.org

From: "Anton D. Kachalov" <[email protected]>

When running on read-only rootfs backed with overlayfs, the processed

scripts

Quoted 2 lines

are being marked as deleted on upperdir of overlayfs. When it comes to upgrade the main read-only image, it might contain new postinst scripts

that

Quoted 2 lines

are different from the already processed. Introduced suffix (sha256 of the content) allows to distinct updated scripts of the same package.

So your usecase is that you have a read-only rootfs with an overlay on top, and then just replace the rootfs. And then expect things to still work?

That doesn't sound valid to me - normally I'd wipe the overlay clean when updating the underlying read-only part. I'd be very interested in what you're actually trying to accomplish here?

And further, I don't think appending 64 characters to filenames is going to make anyone happy.

In your case, I'd opt for a script that just resets the postinst scripts (and maybe more?) in the overlay. Or have 3 overlays. Whenever the package manager runs, activate the second layer. The "user" stuff in the third layer is normally active. Then when you upgrade the system, wipe the second layer, to get rid of obsolete packages. Or something like that. Again, tell us what you're really building...

Review

Quoted 9 lines
---
  meta/lib/oe/package_manager/deb/rootfs.py   | 11 ++++++++++-
  meta/lib/oe/package_manager/ipk/rootfs.py   | 11 ++++++++++-
  meta/lib/oe/package_manager/rpm/__init__.py |  4 +++-
  3 files changed, 23 insertions(+), 3 deletions(-)
(quoted diff) +0 -0

b/meta/lib/oe/package_manager/deb/rootfs.py

Quoted 12 lines

index 8fbaca11d6..704d213626 100644

meta/lib/oe/package_manager/deb/rootfs.py +1 -0
@@ -4,6 +4,7 @@
 
  import re
  import shutil
+from hashlib import sha256
  from oe.rootfs import Rootfs
  from oe.manifest import Manifest
  from oe.utils import execute_pre_post_process

meta/lib/oe/package_manager/deb/rootfs.py class DpkgOpkgRootfs(Rootfs): · line 118

Quoted 3 lines
meta/lib/oe/package_manager/deb/rootfs.py +0 -0
@@ -115,8 +116,16 @@ class DpkgOpkgRootfs(Rootfs):
              bb.utils.mkdirhier(dst_postinst_dir)
 
              if os.path.exists(os.path.join(src_postinst_dir, p +

".postinst")):

meta/lib/oe/package_manager/deb/rootfs.py class DpkgOpkgRootfs(Rootfs):

Quoted 9 lines
meta/lib/oe/package_manager/deb/rootfs.py +8 -0
+                csum = sha256()
+                with open(src_postinst_dir, p + ".postinst", "rb") as f:
+                    while True:
+                        data = f.read(65536)
+                        if not data:
+                            break
+                        csum.update(data)
+
                  shutil.copy(os.path.join(src_postinst_dir, p +

".postinst"),

meta/lib/oe/package_manager/deb/rootfs.py class DpkgOpkgRootfs(Rootfs): · line 119

Quoted 1 line
meta/lib/oe/package_manager/deb/rootfs.py +0 -1
-                            os.path.join(dst_postinst_dir, "%03d-%s" %

(num, p)))

meta/lib/oe/package_manager/deb/rootfs.py class DpkgOpkgRootfs(Rootfs): · line 128

Quoted 1 line
meta/lib/oe/package_manager/deb/rootfs.py +1 -0
+                            os.path.join(dst_postinst_dir, "%03d-%s.%s"

% (num, p, csum.hexdigest())))

Review

Quoted 4 lines
meta/lib/oe/package_manager/deb/rootfs.py +0 -0
              num += 1
 
(quoted diff) +0 -0

b/meta/lib/oe/package_manager/ipk/rootfs.py

Quoted 12 lines

index 26dbee6f6a..6ead6ddef3 100644

meta/lib/oe/package_manager/ipk/rootfs.py +1 -0
@@ -5,6 +5,7 @@
  import re
  import filecmp
  import shutil
+from hashlib import sha256
  from oe.rootfs import Rootfs
  from oe.manifest import Manifest
  from oe.utils import execute_pre_post_process

meta/lib/oe/package_manager/ipk/rootfs.py class DpkgOpkgRootfs(Rootfs): · line 119

Quoted 3 lines
meta/lib/oe/package_manager/ipk/rootfs.py +0 -0
@@ -116,8 +117,16 @@ class DpkgOpkgRootfs(Rootfs):
              bb.utils.mkdirhier(dst_postinst_dir)
 
              if os.path.exists(os.path.join(src_postinst_dir, p +

".postinst")):

meta/lib/oe/package_manager/ipk/rootfs.py class DpkgOpkgRootfs(Rootfs): · line 128

Quoted 9 lines
meta/lib/oe/package_manager/ipk/rootfs.py +8 -0
+                csum = sha256()
+                with open(src_postinst_dir, p + ".postinst", "rb") as f:
+                    while True:
+                        data = f.read(65536)
+                        if not data:
+                            break
+                        csum.update(data)
+
                  shutil.copy(os.path.join(src_postinst_dir, p +

".postinst"),

meta/lib/oe/package_manager/ipk/rootfs.py class DpkgOpkgRootfs(Rootfs): · line 120

Quoted 1 line
meta/lib/oe/package_manager/ipk/rootfs.py +0 -1
-                            os.path.join(dst_postinst_dir, "%03d-%s" %

(num, p)))

meta/lib/oe/package_manager/ipk/rootfs.py class DpkgOpkgRootfs(Rootfs): · line 129

Quoted 1 line
meta/lib/oe/package_manager/ipk/rootfs.py +1 -0
+                            os.path.join(dst_postinst_dir, "%03d-%s.%s"

% (num, p, csum.hexdigest())))

Review

Quoted 4 lines
meta/lib/oe/package_manager/ipk/rootfs.py +0 -0
              num += 1
 
(quoted diff) +0 -0

b/meta/lib/oe/package_manager/rpm/__init__.py

meta/lib/oe/package_manager/rpm/__init__.py class RpmPM(PackageManager): · line 360

Quoted 13 lines

index 6df0092281..4746e96ce8 100644

meta/lib/oe/package_manager/rpm/__init__.py +1 -0
@@ -4,6 +4,7 @@
 
  import shutil
  import subprocess
+from hashlib import sha256
  from oe.package_manager import *
 
  class RpmIndexer(Indexer):
@@ -359,7 +360,8 @@ class RpmPM(PackageManager):
          target_path = oe.path.join(self.target_rootfs,

self.d.expand('${sysconfdir}/rpm-postinsts/'))

meta/lib/oe/package_manager/rpm/__init__.py class RpmPM(PackageManager):

Quoted 3 lines
meta/lib/oe/package_manager/rpm/__init__.py +0 -1
          bb.utils.mkdirhier(target_path)
          num = self._script_num_prefix(target_path)
-        saved_script_name = oe.path.join(target_path, "%d-%s" % (num,

pkg))

meta/lib/oe/package_manager/rpm/__init__.py class RpmPM(PackageManager): · line 364

Quoted 2 lines
meta/lib/oe/package_manager/rpm/__init__.py +2 -0
+        csum = sha256(output.encode("utf-8")).hexdigest()
+        saved_script_name = oe.path.join(target_path, "%d-%s.%s" %

(num, pkg, csum))

meta/lib/oe/package_manager/rpm/__init__.py class RpmPM(PackageManager): · line 367

Quoted 6 lines
meta/lib/oe/package_manager/rpm/__init__.py +0 -0
          open(saved_script_name, 'w').write(output)
          os.chmod(saved_script_name, 0o755)
 

--
Mike Looijmans

#4 lore raw

So it's something exclusively for OpenBMC.

As for running the scripts, you cannot simply conclude that a script doesn't need to run if its contents did not change. It may very well call the executable from its package to do the actual work and that may have changed as well.

Met vriendelijke groet / kind regards,

Mike Looijmans
System Expert

TOPIC Embedded Products B.V.
Materiaalweg 4, 5681 RJ Best
The Netherlands

T: +31 (0) 499 33 69 69
E: [email protected]
W: www.topic.nl

Please consider the environment before printing this e-mail

Quoted 163 lines · Anton Kachalov

Hi, Mike.

That is how OpenBMC is designed. Most of the targets are read-only rootfs (squashfs) and overlayfs for RW. It is not expected to overwrite binaries through overlayfs and there are plans to scope RW areas (e.g. /etc).

Those files with 64-chars suffix are mostly hidden from regular users: the files are removed during the first startup. No one will notice them :)

Anyway, alternatively within the openbmc update script it's possible to force clean the deleted files on overlayfs upperdir. However, it will mean that all the scripts regardless if they are updated or not, will be re-executed after rootfs update.

On Mon, 22 Mar 2021 at 17:08, Mike Looijmans <[email protected] <mailto:[email protected]>> wrote:

Two remarks below...
On 22-03-2021 16:45, Anton Kachalov via lists.openembedded.org
<http://lists.openembedded.org> wrote:
> From: "Anton D. Kachalov" <[email protected] <mailto:[email protected]>>
>
> When running on read-only rootfs backed with overlayfs, the processed scripts
> are being marked as deleted on upperdir of overlayfs. When it comes to
> upgrade the main read-only image, it might contain new postinst scripts that
> are different from the already processed. Introduced suffix (sha256 of
> the content) allows to distinct updated scripts of the same package.

So your usecase is that you have a read-only rootfs with an overlay on
top, and then just replace the rootfs. And then expect things to still
work?

That doesn't sound valid to me - normally I'd wipe the overlay clean when updating the underlying read-only part. I'd be very interested in
what you're actually trying to accomplish here?

And further, I don't think appending 64 characters to filenames is going
to make anyone happy.

In your case, I'd opt for a script that just resets the postinst scripts
(and maybe more?) in the overlay. Or have 3 overlays. Whenever the package manager runs, activate the second layer. The "user" stuff in the
third layer is normally active. Then when you upgrade the system, wipe
the second layer, to get rid of obsolete packages. Or something like that. Again, tell us what you're really building...

>
> Signed-off-by: Anton D. Kachalov <[email protected] <mailto:[email protected]>>
> ---
>   meta/lib/oe/package_manager/deb/rootfs.py   | 11 ++++++++++- >   meta/lib/oe/package_manager/ipk/rootfs.py   | 11 ++++++++++- >   meta/lib/oe/package_manager/rpm/__init__.py |  4 +++- >   3 files changed, 23 insertions(+), 3 deletions(-)
>
> diff --git a/meta/lib/oe/package_manager/deb/rootfs.py b/meta/lib/oe/package_manager/deb/rootfs.py
> index 8fbaca11d6..704d213626 100644
> --- a/meta/lib/oe/package_manager/deb/rootfs.py
> +++ b/meta/lib/oe/package_manager/deb/rootfs.py
> @@ -4,6 +4,7 @@
>
>   import re
>   import shutil
> +from hashlib import sha256
>   from oe.rootfs import Rootfs
>   from oe.manifest import Manifest
>   from oe.utils import execute_pre_post_process
> @@ -115,8 +116,16 @@ class DpkgOpkgRootfs(Rootfs):
>               bb.utils.mkdirhier(dst_postinst_dir)
>
>               if os.path.exists(os.path.join(src_postinst_dir, p
+ ".postinst")):
> +                csum = sha256()
> +                with open(src_postinst_dir, p + ".postinst", "rb") as f:
> +                    while True:
> +                        data = f.read(65536)
> +                        if not data:
> +                            break
> +                        csum.update(data)
> +
>  shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"), > - os.path.join(dst_postinst_dir, "%03d-%s" % (num, p))) > + os.path.join(dst_postinst_dir, "%03d-%s.%s" % (num, p, csum.hexdigest())))
>
>               num += 1
>
> diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py b/meta/lib/oe/package_manager/ipk/rootfs.py
> index 26dbee6f6a..6ead6ddef3 100644
> --- a/meta/lib/oe/package_manager/ipk/rootfs.py
> +++ b/meta/lib/oe/package_manager/ipk/rootfs.py
> @@ -5,6 +5,7 @@
>   import re
>   import filecmp
>   import shutil
> +from hashlib import sha256
>   from oe.rootfs import Rootfs
>   from oe.manifest import Manifest
>   from oe.utils import execute_pre_post_process
> @@ -116,8 +117,16 @@ class DpkgOpkgRootfs(Rootfs):
>               bb.utils.mkdirhier(dst_postinst_dir)
>
>               if os.path.exists(os.path.join(src_postinst_dir, p
+ ".postinst")):
> +                csum = sha256()
> +                with open(src_postinst_dir, p + ".postinst", "rb") as f:
> +                    while True:
> +                        data = f.read(65536)
> +                        if not data:
> +                            break
> +                        csum.update(data)
> +
>  shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"), > - os.path.join(dst_postinst_dir, "%03d-%s" % (num, p))) > + os.path.join(dst_postinst_dir, "%03d-%s.%s" % (num, p, csum.hexdigest())))
>
>               num += 1
>
> diff --git a/meta/lib/oe/package_manager/rpm/__init__.py b/meta/lib/oe/package_manager/rpm/__init__.py
> index 6df0092281..4746e96ce8 100644
> --- a/meta/lib/oe/package_manager/rpm/__init__.py
> +++ b/meta/lib/oe/package_manager/rpm/__init__.py
> @@ -4,6 +4,7 @@
>
>   import shutil
>   import subprocess
> +from hashlib import sha256
>   from oe.package_manager import *
>
>   class RpmIndexer(Indexer):
> @@ -359,7 +360,8 @@ class RpmPM(PackageManager):
>           target_path = oe.path.join(self.target_rootfs, self.d.expand('${sysconfdir}/rpm-postinsts/'))
>           bb.utils.mkdirhier(target_path)
>           num = self._script_num_prefix(target_path) > -        saved_script_name = oe.path.join(target_path, "%d-%s" % (num, pkg))
> +        csum = sha256(output.encode("utf-8")).hexdigest() > +        saved_script_name = oe.path.join(target_path, "%d-%s.%s" % (num, pkg, csum))
>           open(saved_script_name, 'w').write(output) >           os.chmod(saved_script_name, 0o755)
>
>
>
>

--
Mike Looijmans

Signature · 1 line
Mike Looijmans
#5 lore raw

Yes, this use case makes sense. Then the reliable way is to cleanup upperdir of overlayfs during image upgrade and make postinsts scripts re-entrant.

Quoted 201 lines · On Tue, Mar 23, 2021, 07:56 Mike Looijmans <[email protected]> wrote:

So it's something exclusively for OpenBMC.

As for running the scripts, you cannot simply conclude that a script doesn't need to run if its contents did not change. It may very well call the executable from its package to do the actual work and that may have changed as well.

Met vriendelijke groet / kind regards,

Mike Looijmans
System Expert

TOPIC Embedded Products B.V.
Materiaalweg 4, 5681 RJ Best
The Netherlands

T: +31 (0) 499 33 69 69
E: [email protected]
W: www.topic.nl

Please consider the environment before printing this e-mail

Quoted 9 lines · Anton Kachalov

Hi, Mike.

That is how OpenBMC is designed. Most of the targets are read-only rootfs (squashfs) and overlayfs for RW.
It is not expected to overwrite binaries through overlayfs and there are plans to scope RW areas (e.g. /etc).

Those files with 64-chars suffix are mostly hidden from regular users:
the files are removed during the first startup. No one will notice them

:)

Quoted 15 lines

Anyway, alternatively within the openbmc update script it's possible to force clean the deleted files on overlayfs upperdir. However, it will mean that all the scripts regardless if they are updated or not, will be re-executed after rootfs update.

On Mon, 22 Mar 2021 at 17:08, Mike Looijmans <[email protected] <mailto:[email protected]>> wrote:

Two remarks below...
On 22-03-2021 16:45, Anton Kachalov via lists.openembedded.org
<http://lists.openembedded.org> wrote:
> From: "Anton D. Kachalov" <[email protected] <mailto:[email protected]>>
>

Quoted 9 lines

> When running on read-only rootfs backed with overlayfs, the processed scripts
> are being marked as deleted on upperdir of overlayfs. When it comes to
> upgrade the main read-only image, it might contain new postinst scripts that
> are different from the already processed. Introduced suffix (sha256 of
> the content) allows to distinct updated scripts of the same

package.

Quoted 134 lines

So your usecase is that you have a read-only rootfs with an overlay on
top, and then just replace the rootfs. And then expect things to still
work?

That doesn't sound valid to me - normally I'd wipe the overlay clean when updating the underlying read-only part. I'd be very interested in
what you're actually trying to accomplish here?

And further, I don't think appending 64 characters to filenames is going
to make anyone happy.

In your case, I'd opt for a script that just resets the postinst scripts
(and maybe more?) in the overlay. Or have 3 overlays. Whenever the package manager runs, activate the second layer. The "user" stuff in the
third layer is normally active. Then when you upgrade the system, wipe
the second layer, to get rid of obsolete packages. Or something like that. Again, tell us what you're really building...

>
> Signed-off-by: Anton D. Kachalov <[email protected] <mailto:[email protected]>>
> ---
> meta/lib/oe/package_manager/deb/rootfs.py | 11 ++++++++++-
> meta/lib/oe/package_manager/ipk/rootfs.py | 11 ++++++++++-
> meta/lib/oe/package_manager/rpm/__init__.py | 4 +++-
> 3 files changed, 23 insertions(+), 3 deletions(-)
>
> diff --git a/meta/lib/oe/package_manager/deb/rootfs.py b/meta/lib/oe/package_manager/deb/rootfs.py
> index 8fbaca11d6..704d213626 100644
> --- a/meta/lib/oe/package_manager/deb/rootfs.py
> +++ b/meta/lib/oe/package_manager/deb/rootfs.py
> @@ -4,6 +4,7 @@
>
> import re
> import shutil
> +from hashlib import sha256
> from oe.rootfs import Rootfs
> from oe.manifest import Manifest
> from oe.utils import execute_pre_post_process
> @@ -115,8 +116,16 @@ class DpkgOpkgRootfs(Rootfs):
> bb.utils.mkdirhier(dst_postinst_dir)
>
> if os.path.exists(os.path.join(src_postinst_dir, p
+ ".postinst")):
> + csum = sha256()
> + with open(src_postinst_dir, p + ".postinst",
"rb") as f:
> + while True:
> + data = f.read(65536)
> + if not data:
> + break
> + csum.update(data)
> +
> shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"), > - os.path.join(dst_postinst_dir, "%03d-%s" % (num, p))) > + os.path.join(dst_postinst_dir, "%03d-%s.%s" % (num, p, csum.hexdigest())))
>
> num += 1
>
> diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py b/meta/lib/oe/package_manager/ipk/rootfs.py
> index 26dbee6f6a..6ead6ddef3 100644
> --- a/meta/lib/oe/package_manager/ipk/rootfs.py
> +++ b/meta/lib/oe/package_manager/ipk/rootfs.py
> @@ -5,6 +5,7 @@
> import re
> import filecmp
> import shutil
> +from hashlib import sha256
> from oe.rootfs import Rootfs
> from oe.manifest import Manifest
> from oe.utils import execute_pre_post_process
> @@ -116,8 +117,16 @@ class DpkgOpkgRootfs(Rootfs):
> bb.utils.mkdirhier(dst_postinst_dir)
>
> if os.path.exists(os.path.join(src_postinst_dir, p
+ ".postinst")):
> + csum = sha256()
> + with open(src_postinst_dir, p + ".postinst",
"rb") as f:
> + while True:
> + data = f.read(65536)
> + if not data:
> + break
> + csum.update(data)
> +
> shutil.copy(os.path.join(src_postinst_dir, p + ".postinst"), > - os.path.join(dst_postinst_dir, "%03d-%s" % (num, p))) > + os.path.join(dst_postinst_dir, "%03d-%s.%s" % (num, p, csum.hexdigest())))
>
> num += 1
>
> diff --git a/meta/lib/oe/package_manager/rpm/__init__.py b/meta/lib/oe/package_manager/rpm/__init__.py
> index 6df0092281..4746e96ce8 100644
> --- a/meta/lib/oe/package_manager/rpm/__init__.py > +++ b/meta/lib/oe/package_manager/rpm/__init__.py > @@ -4,6 +4,7 @@
>
> import shutil
> import subprocess
> +from hashlib import sha256
> from oe.package_manager import *
>
> class RpmIndexer(Indexer):
> @@ -359,7 +360,8 @@ class RpmPM(PackageManager):
> target_path = oe.path.join(self.target_rootfs,
self.d.expand('${sysconfdir}/rpm-postinsts/'))
> bb.utils.mkdirhier(target_path)
> num = self._script_num_prefix(target_path)
> - saved_script_name = oe.path.join(target_path, "%d-%s" %
(num, pkg))
> + csum = sha256(output.encode("utf-8")).hexdigest()
> + saved_script_name = oe.path.join(target_path,
"%d-%s.%s" % (num, pkg, csum))
> open(saved_script_name, 'w').write(output)
> os.chmod(saved_script_name, 0o755)
>
>
>
>

Quoted 3 lines

--
Mike Looijmans

--
Mike Looijmans