Add package managers as a plugin
OE-core today has three different package managers, the well-known formats deb and rpm is supported as well as ipkg that is good for embedded devices.
When building and having a good cache hit, a significant amount of time is spent in the phase of generating a rootfs, which is really about the performance of the package manager. To save build time and also get a package manager that is suitanle for use on targets where flash memory is a concern, support for apk is suggested.
However, it might or might not be what's wanted for OE-core since it increases the test matrix. Therefore I will refactor the package management code to allow a layer to add a new package manager without editing the meta layer.
This refactor will be divided into multiple patch series to be easier to review, this is the second patch series.
PATCH v3 1/3 Move rpm manifest to its own subdir
This is a part of a refactor that will split the package manager code so that it's possible to use other package managers in other layers.
--- meta/lib/oe/manifest.py | 51 +----------------- meta/lib/oe/package_managers/rpm/__init__.py | 3 ++ meta/lib/oe/package_managers/rpm/manifest.py | 56 ++++++++++++++++++++ meta/lib/oe/rootfs.py | 2 + 4 files changed, 62 insertions(+), 50 deletions(-) create mode 100644 meta/lib/oe/package_managers/rpm/__init__.py create mode 100644 meta/lib/oe/package_managers/rpm/manifest.py
meta/lib/oe/manifest.py +1 -50
@@ -189,56 +189,6 @@ class Manifest(object, metaclass=ABCMeta): return installed_pkgs -class RpmManifest(Manifest): - """ - Returns a dictionary object with mip and mlp packages. - """ - def _split_multilib(self, pkg_list): - pkgs = dict() - - for pkg in pkg_list.split(): - pkg_type = self.PKG_TYPE_MUST_INSTALL - - ml_variants = self.d.getVar('MULTILIB_VARIANTS').split() - - for ml_variant in ml_variants: - if pkg.startswith(ml_variant + '-'): - pkg_type = self.PKG_TYPE_MULTILIB - - if not pkg_type in pkgs: - pkgs[pkg_type] = pkg - else: - pkgs[pkg_type] += " " + pkg - - return pkgs - - def create_initial(self): - pkgs = dict() - - with open(self.initial_manifest, "w+") as manifest: - manifest.write(self.initial_manifest_file_header) - - for var in self.var_maps[self.manifest_type]: - if var in self.vars_to_split: - split_pkgs = self._split_multilib(self.d.getVar(var)) - if split_pkgs is not None: - pkgs = dict(list(pkgs.items()) + list(split_pkgs.items())) - else: - pkg_list = self.d.getVar(var) - if pkg_list is not None: - pkgs[self.var_maps[self.manifest_type][var]] = self.d.getVar(var) - - for pkg_type in pkgs: - for pkg in pkgs[pkg_type].split(): - manifest.write("%s,%s\n" % (pkg_type, pkg)) - - def create_final(self): - pass - - def create_full(self, pm): - pass - - class OpkgManifest(Manifest): """ Returns a dictionary object with mip and mlp packages.
@@ -332,6 +282,7 @@ class DpkgManifest(Manifest): def create_manifest(d, final_manifest=False, manifest_dir=None, manifest_type=Manifest.MANIFEST_TYPE_IMAGE): + from oe.package_managers.rpm.manifest import RpmManifest manifest_map = {'rpm': RpmManifest, 'ipk': OpkgManifest, 'deb': DpkgManifest}
meta/lib/oe/package_managers/rpm/__init__.py new file +3 -0
@@ -0,0 +1,3 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +#
meta/lib/oe/package_managers/rpm/manifest.py new file +56 -0
56 lines not on this page · raw message
meta/lib/oe/rootfs.py +2 -0
@@ -359,6 +359,8 @@ class RpmRootfs(Rootfs): self.log_check_regex = r'(unpacking of archive failed|Cannot find package'\ r'|exit 1|ERROR: |Error: |Error |ERROR '\ r'|Failed |Failed: |Failed$|Failed\(\d+\):)' + + from oe.package_managers.rpm.manifest import RpmManifest self.manifest = RpmManifest(d, manifest_dir) self.pm = RpmPM(d,
Signature · 1 line
2.20.1
PATCH v3 2/3 Move ipk manifest to its own subdir
In reply to Fredrik GustafssonThis is a part of a refactor that will split the package manager code so that it's possible to use other package managers in other layers.
--- meta/lib/oe/manifest.py | 71 +----------------- meta/lib/oe/package_managers/ipk/__init__.py | 3 + meta/lib/oe/package_managers/ipk/manifest.py | 75 ++++++++++++++++++++ meta/lib/oe/rootfs.py | 1 + 4 files changed, 80 insertions(+), 70 deletions(-) create mode 100644 meta/lib/oe/package_managers/ipk/__init__.py create mode 100644 meta/lib/oe/package_managers/ipk/manifest.py
meta/lib/oe/manifest.py +1 -70
@@ -188,76 +188,6 @@ class Manifest(object, metaclass=ABCMeta): return installed_pkgs - -class OpkgManifest(Manifest): - """ - Returns a dictionary object with mip and mlp packages. - """ - def _split_multilib(self, pkg_list): - pkgs = dict() - - for pkg in pkg_list.split(): - pkg_type = self.PKG_TYPE_MUST_INSTALL - - ml_variants = self.d.getVar('MULTILIB_VARIANTS').split() - - for ml_variant in ml_variants: - if pkg.startswith(ml_variant + '-'): - pkg_type = self.PKG_TYPE_MULTILIB - - if not pkg_type in pkgs: - pkgs[pkg_type] = pkg - else: - pkgs[pkg_type] += " " + pkg - - return pkgs - - def create_initial(self): - pkgs = dict() - - with open(self.initial_manifest, "w+") as manifest: - manifest.write(self.initial_manifest_file_header) - - for var in self.var_maps[self.manifest_type]: - if var in self.vars_to_split: - split_pkgs = self._split_multilib(self.d.getVar(var)) - if split_pkgs is not None: - pkgs = dict(list(pkgs.items()) + list(split_pkgs.items())) - else: - pkg_list = self.d.getVar(var) - if pkg_list is not None: - pkgs[self.var_maps[self.manifest_type][var]] = self.d.getVar(var) - - for pkg_type in sorted(pkgs): - for pkg in sorted(pkgs[pkg_type].split()): - manifest.write("%s,%s\n" % (pkg_type, pkg)) - - def create_final(self): - pass - - def create_full(self, pm): - if not os.path.exists(self.initial_manifest): - self.create_initial() - - initial_manifest = self.parse_initial_manifest() - pkgs_to_install = list() - for pkg_type in initial_manifest: - pkgs_to_install += initial_manifest[pkg_type] - if len(pkgs_to_install) == 0: - return - - output = pm.dummy_install(pkgs_to_install) - - with open(self.full_manifest, 'w+') as manifest: - pkg_re = re.compile('^Installing ([^ ]+) [^ ].*') - for line in set(output.split('\n')): - m = pkg_re.match(line) - if m: - manifest.write(m.group(1) + '\n') - - return - - class DpkgManifest(Manifest): def create_initial(self): with open(self.initial_manifest, "w+") as manifest:
@@ -283,6 +213,7 @@ class DpkgManifest(Manifest): def create_manifest(d, final_manifest=False, manifest_dir=None, manifest_type=Manifest.MANIFEST_TYPE_IMAGE): from oe.package_managers.rpm.manifest import RpmManifest + from oe.package_managers.ipk.manifest import OpkgManifest manifest_map = {'rpm': RpmManifest, 'ipk': OpkgManifest, 'deb': DpkgManifest}
meta/lib/oe/package_managers/ipk/__init__.py new file +3 -0
@@ -0,0 +1,3 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +#
meta/lib/oe/package_managers/ipk/manifest.py new file +75 -0
75 lines not on this page · raw message
meta/lib/oe/rootfs.py +1 -0
@@ -699,6 +699,7 @@ class OpkgRootfs(DpkgOpkgRootfs): super(OpkgRootfs, self).__init__(d, progress_reporter, logcatcher) self.log_check_regex = '(exit 1|Collected errors)' + from oe.package_managers.ipk.manifest import OpkgManifest self.manifest = OpkgManifest(d, manifest_dir) self.opkg_conf = self.d.getVar("IPKGCONF_TARGET") self.pkg_archs = self.d.getVar("ALL_MULTILIB_PACKAGE_ARCHS")
Signature · 1 line
2.20.1
PATCH v3 3/3 Move deb manifest to its own subdir
In reply to Fredrik GustafssonThis is a part of a refactor that will split the package manager code so that it's possible to use other package managers in other layers.
--- meta/lib/oe/manifest.py | 21 +--------------- meta/lib/oe/package_managers/deb/__init__.py | 3 +++ meta/lib/oe/package_managers/deb/manifest.py | 26 ++++++++++++++++++++ meta/lib/oe/rootfs.py | 1 + 4 files changed, 31 insertions(+), 20 deletions(-) create mode 100644 meta/lib/oe/package_managers/deb/__init__.py create mode 100644 meta/lib/oe/package_managers/deb/manifest.py
meta/lib/oe/manifest.py +1 -20
@@ -188,32 +188,13 @@ class Manifest(object, metaclass=ABCMeta): return installed_pkgs -class DpkgManifest(Manifest): - def create_initial(self): - with open(self.initial_manifest, "w+") as manifest: - manifest.write(self.initial_manifest_file_header) - - for var in self.var_maps[self.manifest_type]: - pkg_list = self.d.getVar(var) - - if pkg_list is None: - continue - - for pkg in pkg_list.split(): - manifest.write("%s,%s\n" % - (self.var_maps[self.manifest_type][var], pkg)) - - def create_final(self): - pass - - def create_full(self, pm): - pass def create_manifest(d, final_manifest=False, manifest_dir=None, manifest_type=Manifest.MANIFEST_TYPE_IMAGE): from oe.package_managers.rpm.manifest import RpmManifest from oe.package_managers.ipk.manifest import OpkgManifest + from oe.package_managers.deb.manifest import DpkgManifest manifest_map = {'rpm': RpmManifest, 'ipk': OpkgManifest, 'deb': DpkgManifest}
meta/lib/oe/package_managers/deb/__init__.py new file +3 -0
@@ -0,0 +1,3 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +#
meta/lib/oe/package_managers/deb/manifest.py new file +26 -0
@@ -0,0 +1,26 @@ +# +# SPDX-License-Identifier: GPL-2.0-only +# + +from oe.manifest import Manifest + +class DpkgManifest(Manifest): + def create_initial(self): + with open(self.initial_manifest, "w+") as manifest: + manifest.write(self.initial_manifest_file_header) + + for var in self.var_maps[self.manifest_type]: + pkg_list = self.d.getVar(var) + + if pkg_list is None: + continue + + for pkg in pkg_list.split(): + manifest.write("%s,%s\n" % + (self.var_maps[self.manifest_type][var], pkg)) + + def create_final(self): + pass + + def create_full(self, pm): + pass
meta/lib/oe/rootfs.py +1 -0
@@ -615,6 +615,7 @@ class DpkgRootfs(DpkgOpkgRootfs): bb.utils.remove(self.image_rootfs, True) bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS'), True) + from oe.package_managers.deb.manifest import DpkgManifest self.manifest = DpkgManifest(d, manifest_dir) self.pm = DpkgPM(d, d.getVar('IMAGE_ROOTFS'), d.getVar('PACKAGE_ARCHS'),
Signature · 1 line
2.20.1
✗ patchtest: failure for "[v3] Move rpm manifest to its ..." and 2 more
In reply to Fredrik Gustafsson== Series Details ==
Series: "[v3] Move rpm manifest to its ..." and 2 more
Revision: 1
URL : https://patchwork.openembedded.org/series/24970/
State : failure
== Summary ==
Thank you for submitting this patch series to OpenEmbedded Core. This is an automated response. Several tests have been executed on the proposed series by patchtest resulting in the following failures:
* Patch [v3,1/3] Move rpm manifest to its own subdir Issue Shortlog does not follow expected format [test_shortlog_format] Suggested fix Commit shortlog (first line of commit message) should follow the format "<target>: <summary>"
If you believe any of these test results are incorrect, please reply to the mailing list ([email protected]) raising your concerns. Otherwise we would appreciate you correcting the issues and submitting a new version of the patchset if applicable. Please ensure you add/increment the version number when sending the new version (i.e. [PATCH] -> [PATCH v2] -> [PATCH v3] -> ...).
---
Guidelines: https://www.openembedded.org/wiki/Commit_Patch_Message_Guidelines Test framework: http://git.yoctoproject.org/cgit/cgit.cgi/patchtest Test suite: http://git.yoctoproject.org/cgit/cgit.cgi/patchtest-oe
Re: ✗ patchtest: failure for "[v3] Move rpm manifest to its ..." and 2 more
Since I've edited four files, which one should be used as <target>?
/Fredrik
________________________________________
From: Patchwork <[email protected]>
Sent: Thursday, July 2, 2020 9:32 PM
To: Fredrik Gustafsson
Cc: [email protected]
Subject: ✗ patchtest: failure for "[v3] Move rpm manifest to its ..." and 2 more
== Series Details ==
Series: "[v3] Move rpm manifest to its ..." and 2 more
Revision: 1
URL : https://patchwork.openembedded.org/series/24970/
State : failure
== Summary ==
Thank you for submitting this patch series to OpenEmbedded Core. This is an automated response. Several tests have been executed on the proposed series by patchtest resulting in the following failures:
* Patch [v3,1/3] Move rpm manifest to its own subdir Issue Shortlog does not follow expected format [test_shortlog_format] Suggested fix Commit shortlog (first line of commit message) should follow the format "<target>: <summary>"
If you believe any of these test results are incorrect, please reply to the mailing list ([email protected]) raising your concerns. Otherwise we would appreciate you correcting the issues and submitting a new version of the patchset if applicable. Please ensure you add/increment the version number when sending the new version (i.e. [PATCH] -> [PATCH v2] -> [PATCH v3] -> ...).
---
Guidelines: https://www.openembedded.org/wiki/Commit_Patch_Message_Guidelines Test framework: http://git.yoctoproject.org/cgit/cgit.cgi/patchtest Test suite: http://git.yoctoproject.org/cgit/cgit.cgi/patchtest-oe
Re: [OE-core] ✗ patchtest: failure for "[v3] Move rpm manifest to its ..." and 2 more
Quoted 1 line · Fredrik Gustafsson
Since I've edited four files, which one should be used as <target>?
A suitable shortlog here would be something like:
lib/oe/{manifest,package_manager/XXX}: Move XXX manifest to...
Cheers,
Richard
Quoted 10 lines · Fredrik Gustafsson
OE-core today has three different package managers, the well-known formats deb and rpm is supported as well as ipkg that is good for embedded devices.
When building and having a good cache hit, a significant amount of time is spent in the phase of generating a rootfs, which is really about the performance of the package manager. To save build time and also get a package manager that is suitanle for use on targets where flash memory is a concern, support for apk is suggested.
Quoted 6 lines
However, it might or might not be what's wanted for OE-core since it increases the test matrix. Therefore I will refactor the package management code to allow a layer to add a new package manager without editing the meta layer.
This refactor will be divided into multiple patch series to be easier to review, this is the second patch series.
Sorry to be a pain but I think this series suffers from the opposite problem to the previous version - it's only a partial move of some of the code. I'd much rather review a series which does the complete refactor of the existing code into separate modules without making any functional changes. I'll let others give their opinions as well, I'm happy to review this one if we do want to look at this piecemeal.
Signature · 2 lines
Paul Barker Konsulko Group
________________________________________
From: Paul Barker <[email protected]>
Sent: Thursday, July 2, 2020 10:28 PM
To: Fredrik Gustafsson
Cc: openembedded-core; tools-cfpbuild-internal
Subject: Re: [OE-core] Add package managers as a plugin
Quoted 10 lines · Fredrik Gustafsson
OE-core today has three different package managers, the well-known formats deb and rpm is supported as well as ipkg that is good for embedded devices.
When building and having a good cache hit, a significant amount of time is spent in the phase of generating a rootfs, which is really about the performance of the package manager. To save build time and also get a package manager that is suitanle for use on targets where flash memory is a concern, support for apk is suggested.
Quoted 6 lines
However, it might or might not be what's wanted for OE-core since it increases the test matrix. Therefore I will refactor the package management code to allow a layer to add a new package manager without editing the meta layer.
This refactor will be divided into multiple patch series to be easier to review, this is the second patch series.
Sorry to be a pain but I think this series suffers from the opposite problem to the previous version - it's only a partial move of some of the code. I'd much rather review a series which does the complete refactor of the existing code into separate modules without making any functional changes. I'll let others give their opinions as well, I'm happy to review this one if we do want to look at this piecemeal.
--
Paul Barker
Konsulko Group
No problems, as said before I will do my best to make this easier to review. I've sent a new patch series.
/Fredrik