Pull postinst-intercepts from BBPATH
cover Pull postinst-intercepts from BBPATH
From: Christopher Larson <[email protected]>
These are the bits to allow us to pull intercepts from BBPATH. This is kept as a separate class, as it's needed by both image construction and sdk construction, the latter to support meta-toolchain & similar recipes.
The following changes since commit 83c9405df5748744ef673ac8757bb89d7050ad8d:
alsa-tools: rewrite packaging (2018-06-15 11:18:15 +0100)
are available in the git repository at:
[email protected]:kergoth/openembedded-core.git bbpath-intercepts
for you to fetch changes up to 4bb748f4c9b177722a13a1f09dcd86b4595752ea:
populate_sdk_base.bbclass: inherit and use bbpath-intercepts (2018-06-16 02:05:39 +0500)
----------------------------------------------------------------
Christopher Larson (5):
oe.path: add which_wild function
oe.package_manager: support loading intercepts from multiple paths bbpath-intercepts.bbclass: add class
image.bbclass: inherit and use bbpath-intercepts populate_sdk_base.bbclass: inherit and use bbpath-intercepts
meta/classes/bbpath-intercepts.bbclass | 23 +++++++++++++++++++++++ meta/classes/image.bbclass | 3 ++- meta/classes/populate_sdk_base.bbclass | 3 ++- meta/lib/oe/package_manager.py | 19 +++++++++++++------ meta/lib/oe/path.py | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 8 deletions(-) create mode 100644 meta/classes/bbpath-intercepts.bbclass
Signature · 1 line
2.11.1
PATCH 1/5 oe.path: add which_wild function
From: Christopher Larson <[email protected]>
This is a function much like shutil.which or bb.utils.which, retaining shutil.which-like function semantics, bb.utils.which's support for returning available candidates for signatures, and most importantly, supports wildcards, returning only the first occurrance of each found pathname in the search path.
--- meta/lib/oe/path.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+)
meta/lib/oe/path.py +34 -0
@@ -259,3 +259,37 @@ def is_path_parent(possible_parent, *paths): if not path_abs.startswith(possible_parent_abs): return False return True + +def which_wild(pathname, path=None, mode=os.F_OK, *, reverse=False, candidates=False): + """Search a search path for pathname, supporting wildcards. + + Return all paths in the specific search path matching the wildcard pattern + in pathname, returning only the first encountered for each file. If + candidates is True, information on all potential candidate paths are + included. + """ + paths = (path or os.environ.get('PATH', os.defpath)).split(':') + if reverse: + paths.reverse() + + seen, files = set(), [] + for index, element in enumerate(paths): + if not os.path.isabs(element): + element = os.path.abspath(element) + + candidate = os.path.join(element, pathname) + globbed = glob.glob(candidate) + if globbed: + for found_path in sorted(globbed): + if not os.access(found_path, mode): + continue + rel = os.path.relpath(found_path, element) + if rel not in seen: + seen.add(rel) + if candidates: + files.append((found_path, [os.path.join(p, rel) for p in paths[:index+1]])) + else: + files.append(found_path) + + return files +
Signature · 1 line
2.11.1
PATCH 2/5 oe.package_manager: support loading intercepts from multiple paths
In reply to Christopher LarsonFrom: Christopher Larson <[email protected]>
- if POSTINST_INTERCEPTS is set, use the listed intercept files, or
- if POSTINST_INTERCEPTS_PATH is set, load from the listed paths, or
- if POSTINST_INTERCEPTS_DIR is set, load from it (for compatibility), or
- load from ${COREBASE}/meta/postinst-intercepts
--- meta/lib/oe/package_manager.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-)
meta/lib/oe/package_manager.py +13 -6
@@ -336,17 +336,24 @@ class PackageManager(object, metaclass=ABCMeta): def _initialize_intercepts(self): bb.note("Initializing intercept dir for %s" % self.target_rootfs) - postinst_intercepts_dir = self.d.getVar("POSTINST_INTERCEPTS_DIR") - if not postinst_intercepts_dir: - postinst_intercepts_dir = self.d.expand("${COREBASE}/scripts/postinst-intercepts") # As there might be more than one instance of PackageManager operating at the same time # we need to isolate the intercept_scripts directories from each other, # hence the ugly hash digest in dir name. - self.intercepts_dir = os.path.join(self.d.getVar('WORKDIR'), - "intercept_scripts-%s" %(hashlib.sha256(self.target_rootfs.encode()).hexdigest()) ) + self.intercepts_dir = os.path.join(self.d.getVar('WORKDIR'), "intercept_scripts-%s" % + (hashlib.sha256(self.target_rootfs.encode()).hexdigest())) + postinst_intercepts = (self.d.getVar("POSTINST_INTERCEPTS") or "").split() + if not postinst_intercepts: + postinst_intercepts_path = self.d.getVar("POSTINST_INTERCEPTS_PATH") + if not postinst_intercepts_path: + postinst_intercepts_path = self.d.getVar("POSTINST_INTERCEPTS_DIR") or self.d.expand("${COREBASE}/scripts/postinst-intercepts") + postinst_intercepts = oe.path.which_wild('*', postinst_intercepts_path) + + bb.debug(1, 'Collected intercepts:\n%s' % ''.join(' %s\n' % i for i in postinst_intercepts)) bb.utils.remove(self.intercepts_dir, True) - shutil.copytree(postinst_intercepts_dir, self.intercepts_dir) + bb.utils.mkdirhier(self.intercepts_dir) + for intercept in postinst_intercepts: + bb.utils.copyfile(intercept, os.path.join(self.intercepts_dir, os.path.basename(intercept))) @abstractmethod def _handle_intercept_failure(self, failed_script):
Signature · 1 line
2.11.1
PATCH 3/5 bbpath-intercepts.bbclass: add class
In reply to Christopher LarsonFrom: Christopher Larson <[email protected]>
This class sets POSTINST_INTERCEPTS and POSTINST_INTERCEPTS_CHECKSUMS, to allow us to pull intercepts from BBPATH. This is kept as a separate class, as it's needed by both image construction and sdk construction, the latter to support meta-toolchain & similar recipes.
--- meta/classes/bbpath-intercepts.bbclass | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 meta/classes/bbpath-intercepts.bbclass
meta/classes/bbpath-intercepts.bbclass new file +23 -0
@@ -0,0 +1,23 @@ +# Gather existing and candidate postinst intercepts from BBPATH +POSTINST_INTERCEPTS_DIR ?= "${COREBASE}/scripts/postinst-intercepts" +POSTINST_INTERCEPTS_PATHS ?= "${@':'.join('%s/postinst-intercepts' % p for p in '${BBPATH}'.split(':'))}:${POSTINST_INTERCEPTS_DIR}" + +python find_intercepts() { + intercepts = {} + search_paths = [] + paths = d.getVar('POSTINST_INTERCEPTS_PATHS').split(':') + overrides = (':' + d.getVar('FILESOVERRIDES')).split(':') + [''] + search_paths = [os.path.join(p, op) for p in paths for op in overrides] + searched = oe.path.which_wild('*', ':'.join(search_paths), candidates=True) + files, chksums = [], [] + for pathname, candidates in searched: + if os.path.isfile(pathname): + files.append(pathname) + chksums.append('%s:True' % pathname) + chksums.extend('%s:False' % c for c in candidates[:-1]) + + d.setVar('POSTINST_INTERCEPT_CHECKSUMS', ' '.join(chksums)) + d.setVar('POSTINST_INTERCEPTS', ' '.join(files)) +} +find_intercepts[eventmask] += "bb.event.RecipePreFinalise" +addhandler find_intercepts
Signature · 1 line
2.11.1
Re: [PATCH 3/5] bbpath-intercepts.bbclass: add class
Quoted 10 lines · Christopher Larson
From: Christopher Larson <[email protected]>
This class sets
POSTINST_INTERCEPTSandPOSTINST_INTERCEPTS_CHECKSUMS,
to allow us to pull intercepts from BBPATH. This is kept as a separate
class, as it's needed by both image construction and sdk construction,
the latter to support meta-toolchain & similar recipes.
Quoted 5 lines
Signed-off-by Christopher Larson <[email protected]>--- meta/classes/bbpath-intercepts.bbclass | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 meta/classes/bbpath-intercepts.bbclass
We're struggling a bit with classes and namespaces. I'm wondering if this might be better as "image-postinst-intercepts.bbclass" or similar. That would mean the variable namespace matches the class and that the image classes are more 'together'.
Your next patch will need rebasing against the changes that just merged into master in image.bbclass.
I have also wondered about pushing more code to lib/oe but I can see why this makes sense as a bbclass.
Cheers,
Richard
Re: [PATCH 3/5] bbpath-intercepts.bbclass: add class
Quoted 22 lines · Richard Purdie
Quoted 10 lines · Christopher Larson
From: Christopher Larson <[email protected]>
This class sets
POSTINST_INTERCEPTSandPOSTINST_INTERCEPTS_CHECKSUMS,
to allow us to pull intercepts from BBPATH. This is kept as a separate
class, as it's needed by both image construction and sdk construction,
the latter to support meta-toolchain & similar recipes.Quoted 5 lines
Signed-off-by Christopher Larson <[email protected]>--- meta/classes/bbpath-intercepts.bbclass | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 meta/classes/bbpath-intercepts.bbclassWe're struggling a bit with classes and namespaces. I'm wondering if this might be better as "image-postinst-intercepts.bbclass" or similar. That would mean the variable namespace matches the class and that the image classes are more 'together'.
Quoted 6 lines
Your next patch will need rebasing against the changes that just merged into master in image.bbclass.
I have also wondered about pushing more code to lib/oe but I can see why this makes sense as a bbclass.
Yeah, I think in this case this works as a class, but you're right, we should probably shift more to the python package. Namespacing is probably a good idea, we've talked about doing that more in the past. In this case it's technically not image-only, though an sdk uses the same underlying functionality, so it's debatable. I wonder if we should namespace into a subdirectory to also make it easier to make sense of classes/ at a command-line, i.e. inherit image/postinst-intercepts.bbclass? Hmmm.
Signature · 4 lines
Christopher Larson kergoth at gmail dot com Founder - BitBake, OpenEmbedded, OpenZaurus Senior Software Engineer, Mentor Graphics
Re: [PATCH 3/5] bbpath-intercepts.bbclass: add class
Quoted 34 lines · Christopher Larson
Quoted 22 lines · Richard Purdie
Quoted 10 lines · Christopher Larson
From: Christopher Larson <[email protected]>
This class sets
POSTINST_INTERCEPTSandPOSTINST_INTERCEPTS_CHECKSUMS,
to allow us to pull intercepts from BBPATH. This is kept as a separate
class, as it's needed by both image construction and sdk construction,
the latter to support meta-toolchain & similar recipes.Quoted 5 lines
Signed-off-by Christopher Larson <[email protected]>--- meta/classes/bbpath-intercepts.bbclass | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 meta/classes/bbpath-intercepts.bbclassWe're struggling a bit with classes and namespaces. I'm wondering if this might be better as "image-postinst-intercepts.bbclass" or similar. That would mean the variable namespace matches the class and that the image classes are more 'together'.
Quoted 6 lines
Your next patch will need rebasing against the changes that just merged into master in image.bbclass.
I have also wondered about pushing more code to lib/oe but I can see why this makes sense as a bbclass.
Quoted 8 lines
Yeah, I think in this case this works as a class, but you're right, we should probably shift more to the python package. Namespacing is probably a good idea, we've talked about doing that more in the past. In this case it's technically not image-only, though an sdk uses the same underlying functionality, so it's debatable. I wonder if we should namespace into a subdirectory to also make it easier to make sense of classes/ at a command-line, i.e. inherit image/postinst-intercepts.bbclass? Hmmm.
I'll submit a v2 rebased with the class renamed as you suggested for now, later today. Thanks.
Signature · 4 lines
Christopher Larson kergoth at gmail dot com Founder - BitBake, OpenEmbedded, OpenZaurus Senior Software Engineer, Mentor Graphics
Re: [PATCH 3/5] bbpath-intercepts.bbclass: add class
In reply to Christopher LarsonQuoted 3 lines · Christopher Larson
Yeah, I think in this case this works as a class, but you're right, we should probably shift more to the python package. Namespacing is probably a good idea, we've talked about doing that more in the past.
Whilst changing what we have today is hard, we can at least do better when we add new classes. I've been trying to, not sure how well its going but we can try!
Quoted 5 lines
In this case it's technically not image-only, though an sdk uses the same underlying functionality, so it's debatable. I wonder if we should namespace into a subdirectory to also make it easier to make sense of classes/ at a command-line, i.e. inherit image/postinst- intercepts.bbclass? Hmmm.
I'd thought about image verses sdk and decided its roughly "image" and clearer this way that any other I could come up with.
There is an open bug about "global" classes verses "recipe" classes too and how hard it is to tell the difference currently. I'd probably favour splitting things up a bit. I'd not considered subdirs, its worth thinking about...
Cheers,
Richard
PATCH 4/5 image.bbclass: inherit and use bbpath-intercepts
In reply to Christopher LarsonFrom: Christopher Larson <[email protected]>
--- meta/classes/image.bbclass | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
meta/classes/image.bbclass +2 -1
@@ -191,7 +191,7 @@ LINGUAS_INSTALL ?= "${@" ".join(map(lambda s: "locale-base-%s" % s, d.getVar('IM # aren't yet available. PSEUDO_PASSWD = "${IMAGE_ROOTFS}:${STAGING_DIR_NATIVE}" -inherit rootfs-postcommands +inherit rootfs-postcommands bbpath-intercepts PACKAGE_EXCLUDE ??= "" PACKAGE_EXCLUDE[type] = "list"
@@ -262,6 +262,7 @@ fakeroot python do_rootfs () { do_rootfs[dirs] = "${TOPDIR}" do_rootfs[cleandirs] += "${S} ${IMGDEPLOYDIR}" do_rootfs[umask] = "022" +do_rootfs[file-checksums] += "${POSTINST_INTERCEPT_CHECKSUMS}" addtask rootfs after do_prepare_recipe_sysroot fakeroot python do_image () {
Signature · 1 line
2.11.1
PATCH 5/5 populate_sdk_base.bbclass: inherit and use bbpath-intercepts
In reply to Christopher LarsonFrom: Christopher Larson <[email protected]>
--- meta/classes/populate_sdk_base.bbclass | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
meta/classes/populate_sdk_base.bbclass +2 -1
@@ -1,4 +1,4 @@ -inherit meta +inherit meta bbpath-intercepts # Wildcards specifying complementary packages to install for every package that has been explicitly # installed into the rootfs
@@ -306,4 +306,5 @@ do_populate_sdk[dirs] = "${PKGDATA_DIR} ${TOPDIR}" do_populate_sdk[depends] += "${@' '.join([x + ':do_populate_sysroot' for x in d.getVar('SDK_DEPENDS').split()])} ${@d.getVarFlag('do_rootfs', 'depends', False)}" do_populate_sdk[rdepends] = "${@' '.join([x + ':do_package_write_${IMAGE_PKGTYPE} ' + x + ':do_packagedata' for x in d.getVar('SDK_RDEPENDS').split()])}" do_populate_sdk[recrdeptask] += "do_packagedata do_package_write_rpm do_package_write_ipk do_package_write_deb" +do_populate_sdk[file-checksums] += "${POSTINST_INTERCEPT_CHECKSUMS}" addtask populate_sdk
Signature · 1 line
2.11.1