Skip to contentSkip to conversation

Pull postinst-intercepts from BBPATH

PATCH Superseded Christopher Larson 2 participants 10 messages Jun 21, 2018 – Jun 25, 2018

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover Pull postinst-intercepts from BBPATH

From: Christopher Larson <[email protected]>

These are the bits to allow us to pull intercepts from BBPATH. This is kept as a separate class, as it's needed by both image construction and sdk construction, the latter to support meta-toolchain & similar recipes.

The following changes since commit 83c9405df5748744ef673ac8757bb89d7050ad8d:

alsa-tools: rewrite packaging (2018-06-15 11:18:15 +0100)

are available in the git repository at:

[email protected]:kergoth/openembedded-core.git bbpath-intercepts

for you to fetch changes up to 4bb748f4c9b177722a13a1f09dcd86b4595752ea:

populate_sdk_base.bbclass: inherit and use bbpath-intercepts (2018-06-16 02:05:39 +0500)

----------------------------------------------------------------
Christopher Larson (5):
oe.path: add which_wild function
oe.package_manager: support loading intercepts from multiple paths bbpath-intercepts.bbclass: add class
image.bbclass: inherit and use bbpath-intercepts populate_sdk_base.bbclass: inherit and use bbpath-intercepts

 meta/classes/bbpath-intercepts.bbclass | 23 +++++++++++++++++++++++
 meta/classes/image.bbclass             |  3 ++-
 meta/classes/populate_sdk_base.bbclass |  3 ++-
 meta/lib/oe/package_manager.py         | 19 +++++++++++++------
 meta/lib/oe/path.py                    | 34 ++++++++++++++++++++++++++++++++++
 5 files changed, 74 insertions(+), 8 deletions(-)
 create mode 100644 meta/classes/bbpath-intercepts.bbclass
Signature · 1 line
2.11.1
#1 lore raw

PATCH 1/5 oe.path: add which_wild function

From: Christopher Larson <[email protected]>

This is a function much like shutil.which or bb.utils.which, retaining shutil.which-like function semantics, bb.utils.which's support for returning available candidates for signatures, and most importantly, supports wildcards, returning only the first occurrance of each found pathname in the search path.

---
 meta/lib/oe/path.py | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
meta/lib/oe/path.py +34 -0
index 76c58fa7601..be02218c31d 100644
@@ -259,3 +259,37 @@ def is_path_parent(possible_parent, *paths):
         if not path_abs.startswith(possible_parent_abs):
             return False
     return True
+
+def which_wild(pathname, path=None, mode=os.F_OK, *, reverse=False, candidates=False):
+    """Search a search path for pathname, supporting wildcards.
+
+    Return all paths in the specific search path matching the wildcard pattern
+    in pathname, returning only the first encountered for each file. If
+    candidates is True, information on all potential candidate paths are
+    included.
+    """
+    paths = (path or os.environ.get('PATH', os.defpath)).split(':')
+    if reverse:
+        paths.reverse()
+
+    seen, files = set(), []
+    for index, element in enumerate(paths):
+        if not os.path.isabs(element):
+            element = os.path.abspath(element)
+
+        candidate = os.path.join(element, pathname)
+        globbed = glob.glob(candidate)
+        if globbed:
+            for found_path in sorted(globbed):
+                if not os.access(found_path, mode):
+                    continue
+                rel = os.path.relpath(found_path, element)
+                if rel not in seen:
+                    seen.add(rel)
+                    if candidates:
+                        files.append((found_path, [os.path.join(p, rel) for p in paths[:index+1]]))
+                    else:
+                        files.append(found_path)
+
+    return files
+
Signature · 1 line
2.11.1
#2 lore raw

PATCH 2/5 oe.package_manager: support loading intercepts from multiple paths

In reply to Christopher Larson

From: Christopher Larson <[email protected]>

- if POSTINST_INTERCEPTS is set, use the listed intercept files, or
- if POSTINST_INTERCEPTS_PATH is set, load from the listed paths, or
- if POSTINST_INTERCEPTS_DIR is set, load from it (for compatibility), or
- load from ${COREBASE}/meta/postinst-intercepts

---
 meta/lib/oe/package_manager.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)
meta/lib/oe/package_manager.py +13 -6
index 5ac729455e9..7f9cbba7084 100644
@@ -336,17 +336,24 @@ class PackageManager(object, metaclass=ABCMeta):
 
     def _initialize_intercepts(self):
         bb.note("Initializing intercept dir for %s" % self.target_rootfs)
-        postinst_intercepts_dir = self.d.getVar("POSTINST_INTERCEPTS_DIR")
-        if not postinst_intercepts_dir:
-            postinst_intercepts_dir = self.d.expand("${COREBASE}/scripts/postinst-intercepts")
         # As there might be more than one instance of PackageManager operating at the same time
         # we need to isolate the intercept_scripts directories from each other,
         # hence the ugly hash digest in dir name.
-        self.intercepts_dir = os.path.join(self.d.getVar('WORKDIR'),
-                                      "intercept_scripts-%s" %(hashlib.sha256(self.target_rootfs.encode()).hexdigest()) )
+        self.intercepts_dir = os.path.join(self.d.getVar('WORKDIR'), "intercept_scripts-%s" %
+                                           (hashlib.sha256(self.target_rootfs.encode()).hexdigest()))
 
+        postinst_intercepts = (self.d.getVar("POSTINST_INTERCEPTS") or "").split()
+        if not postinst_intercepts:
+            postinst_intercepts_path = self.d.getVar("POSTINST_INTERCEPTS_PATH")
+            if not postinst_intercepts_path:
+                postinst_intercepts_path = self.d.getVar("POSTINST_INTERCEPTS_DIR") or self.d.expand("${COREBASE}/scripts/postinst-intercepts")
+            postinst_intercepts = oe.path.which_wild('*', postinst_intercepts_path)
+
+        bb.debug(1, 'Collected intercepts:\n%s' % ''.join('  %s\n' % i for i in postinst_intercepts))
         bb.utils.remove(self.intercepts_dir, True)
-        shutil.copytree(postinst_intercepts_dir, self.intercepts_dir)
+        bb.utils.mkdirhier(self.intercepts_dir)
+        for intercept in postinst_intercepts:
+            bb.utils.copyfile(intercept, os.path.join(self.intercepts_dir, os.path.basename(intercept)))
 
     @abstractmethod
     def _handle_intercept_failure(self, failed_script):
Signature · 1 line
2.11.1
#3 lore raw

PATCH 3/5 bbpath-intercepts.bbclass: add class

In reply to Christopher Larson

From: Christopher Larson <[email protected]>

This class sets POSTINST_INTERCEPTS and POSTINST_INTERCEPTS_CHECKSUMS, to allow us to pull intercepts from BBPATH. This is kept as a separate class, as it's needed by both image construction and sdk construction, the latter to support meta-toolchain & similar recipes.

---
 meta/classes/bbpath-intercepts.bbclass | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 meta/classes/bbpath-intercepts.bbclass
meta/classes/bbpath-intercepts.bbclass new file +23 -0
new file mode 100644
index 00000000000..ed30bbd98d9
@@ -0,0 +1,23 @@
+# Gather existing and candidate postinst intercepts from BBPATH
+POSTINST_INTERCEPTS_DIR ?= "${COREBASE}/scripts/postinst-intercepts"
+POSTINST_INTERCEPTS_PATHS ?= "${@':'.join('%s/postinst-intercepts' % p for p in '${BBPATH}'.split(':'))}:${POSTINST_INTERCEPTS_DIR}"
+
+python find_intercepts() {
+    intercepts = {}
+    search_paths = []
+    paths = d.getVar('POSTINST_INTERCEPTS_PATHS').split(':')
+    overrides = (':' + d.getVar('FILESOVERRIDES')).split(':') + ['']
+    search_paths = [os.path.join(p, op) for p in paths for op in overrides]
+    searched = oe.path.which_wild('*', ':'.join(search_paths), candidates=True)
+    files, chksums = [], []
+    for pathname, candidates in searched:
+        if os.path.isfile(pathname):
+            files.append(pathname)
+            chksums.append('%s:True' % pathname)
+            chksums.extend('%s:False' % c for c in candidates[:-1])
+
+    d.setVar('POSTINST_INTERCEPT_CHECKSUMS', ' '.join(chksums))
+    d.setVar('POSTINST_INTERCEPTS', ' '.join(files))
+}
+find_intercepts[eventmask] += "bb.event.RecipePreFinalise"
+addhandler find_intercepts
Signature · 1 line
2.11.1
#4 lore raw

Re: [PATCH 3/5] bbpath-intercepts.bbclass: add class

Quoted 10 lines · Christopher Larson

From: Christopher Larson <[email protected]>

This class sets POSTINST_INTERCEPTS and POSTINST_INTERCEPTS_CHECKSUMS,
to allow us to pull intercepts from BBPATH. This is kept as a separate
class, as it's needed by both image construction and sdk construction,
the latter to support meta-toolchain & similar recipes.

Quoted 5 lines
---
 meta/classes/bbpath-intercepts.bbclass | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 meta/classes/bbpath-intercepts.bbclass

We're struggling a bit with classes and namespaces. I'm wondering if this might be better as "image-postinst-intercepts.bbclass" or similar. That would mean the variable namespace matches the class and that the image classes are more 'together'.

Your next patch will need rebasing against the changes that just merged into master in image.bbclass.

I have also wondered about pushing more code to lib/oe but I can see why this makes sense as a bbclass.

Cheers,

Richard

#5 lore raw

Re: [PATCH 3/5] bbpath-intercepts.bbclass: add class

Quoted 22 lines · Richard Purdie
Quoted 10 lines · Christopher Larson

From: Christopher Larson <[email protected]>

This class sets POSTINST_INTERCEPTS and POSTINST_INTERCEPTS_CHECKSUMS,
to allow us to pull intercepts from BBPATH. This is kept as a separate
class, as it's needed by both image construction and sdk construction,
the latter to support meta-toolchain & similar recipes.

Quoted 5 lines
---
 meta/classes/bbpath-intercepts.bbclass | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 meta/classes/bbpath-intercepts.bbclass

We're struggling a bit with classes and namespaces. I'm wondering if this might be better as "image-postinst-intercepts.bbclass" or similar. That would mean the variable namespace matches the class and that the image classes are more 'together'.

Quoted 6 lines

Your next patch will need rebasing against the changes that just merged into master in image.bbclass.

I have also wondered about pushing more code to lib/oe but I can see why this makes sense as a bbclass.

Yeah, I think in this case this works as a class, but you're right, we should probably shift more to the python package. Namespacing is probably a good idea, we've talked about doing that more in the past. In this case it's technically not image-only, though an sdk uses the same underlying functionality, so it's debatable. I wonder if we should namespace into a subdirectory to also make it easier to make sense of classes/ at a command-line, i.e. inherit image/postinst-intercepts.bbclass? Hmmm.

Signature · 4 lines
Christopher Larson
kergoth at gmail dot com
Founder - BitBake, OpenEmbedded, OpenZaurus
Senior Software Engineer, Mentor Graphics
#6 lore raw

Re: [PATCH 3/5] bbpath-intercepts.bbclass: add class

Quoted 34 lines · Christopher Larson
Quoted 22 lines · Richard Purdie
Quoted 10 lines · Christopher Larson

From: Christopher Larson <[email protected]>

This class sets POSTINST_INTERCEPTS and POSTINST_INTERCEPTS_CHECKSUMS,
to allow us to pull intercepts from BBPATH. This is kept as a separate
class, as it's needed by both image construction and sdk construction,
the latter to support meta-toolchain & similar recipes.

Quoted 5 lines
---
 meta/classes/bbpath-intercepts.bbclass | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 meta/classes/bbpath-intercepts.bbclass

We're struggling a bit with classes and namespaces. I'm wondering if this might be better as "image-postinst-intercepts.bbclass" or similar. That would mean the variable namespace matches the class and that the image classes are more 'together'.

Quoted 6 lines

Your next patch will need rebasing against the changes that just merged into master in image.bbclass.

I have also wondered about pushing more code to lib/oe but I can see why this makes sense as a bbclass.

Quoted 8 lines

Yeah, I think in this case this works as a class, but you're right, we should probably shift more to the python package. Namespacing is probably a good idea, we've talked about doing that more in the past. In this case it's technically not image-only, though an sdk uses the same underlying functionality, so it's debatable. I wonder if we should namespace into a subdirectory to also make it easier to make sense of classes/ at a command-line, i.e. inherit image/postinst-intercepts.bbclass? Hmmm.

I'll submit a v2 rebased with the class renamed as you suggested for now, later today. Thanks.

Signature · 4 lines
Christopher Larson
kergoth at gmail dot com
Founder - BitBake, OpenEmbedded, OpenZaurus
Senior Software Engineer, Mentor Graphics
#7 lore raw

Re: [PATCH 3/5] bbpath-intercepts.bbclass: add class

In reply to Christopher Larson
Quoted 3 lines · Christopher Larson

Yeah, I think in this case this works as a class, but you're right, we should probably shift more to the python package. Namespacing is probably a good idea, we've talked about doing that more in the past.

Whilst changing what we have today is hard, we can at least do better when we add new classes. I've been trying to, not sure how well its going but we can try!

Quoted 5 lines

In this case it's technically not image-only, though an sdk uses the same underlying functionality, so it's debatable. I wonder if we should namespace into a subdirectory to also make it easier to make sense of classes/ at a command-line, i.e. inherit image/postinst- intercepts.bbclass? Hmmm.

I'd thought about image verses sdk and decided its roughly "image" and clearer this way that any other I could come up with.

There is an open bug about "global" classes verses "recipe" classes too and how hard it is to tell the difference currently. I'd probably favour splitting things up a bit. I'd not considered subdirs, its worth thinking about...

Cheers,

Richard

#8 lore raw

PATCH 4/5 image.bbclass: inherit and use bbpath-intercepts

In reply to Christopher Larson

From: Christopher Larson <[email protected]>

---
 meta/classes/image.bbclass | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
meta/classes/image.bbclass +2 -1
index 2247b305da8..29bd716f8d3 100644
@@ -191,7 +191,7 @@ LINGUAS_INSTALL ?= "${@" ".join(map(lambda s: "locale-base-%s" % s, d.getVar('IM
 # aren't yet available.
 PSEUDO_PASSWD = "${IMAGE_ROOTFS}:${STAGING_DIR_NATIVE}"
 
-inherit rootfs-postcommands
+inherit rootfs-postcommands bbpath-intercepts
 
 PACKAGE_EXCLUDE ??= ""
 PACKAGE_EXCLUDE[type] = "list"
@@ -262,6 +262,7 @@ fakeroot python do_rootfs () {
 do_rootfs[dirs] = "${TOPDIR}"
 do_rootfs[cleandirs] += "${S} ${IMGDEPLOYDIR}"
 do_rootfs[umask] = "022"
+do_rootfs[file-checksums] += "${POSTINST_INTERCEPT_CHECKSUMS}"
 addtask rootfs after do_prepare_recipe_sysroot
 
 fakeroot python do_image () {
Signature · 1 line
2.11.1
#9 lore raw

PATCH 5/5 populate_sdk_base.bbclass: inherit and use bbpath-intercepts

In reply to Christopher Larson

From: Christopher Larson <[email protected]>

---
 meta/classes/populate_sdk_base.bbclass | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
meta/classes/populate_sdk_base.bbclass +2 -1
index 3da350747e2..8b576e1bfde 100644
@@ -1,4 +1,4 @@
-inherit meta
+inherit meta bbpath-intercepts
 
 # Wildcards specifying complementary packages to install for every package that has been explicitly
 # installed into the rootfs
@@ -306,4 +306,5 @@ do_populate_sdk[dirs] = "${PKGDATA_DIR} ${TOPDIR}"
 do_populate_sdk[depends] += "${@' '.join([x + ':do_populate_sysroot' for x in d.getVar('SDK_DEPENDS').split()])}  ${@d.getVarFlag('do_rootfs', 'depends', False)}"
 do_populate_sdk[rdepends] = "${@' '.join([x + ':do_package_write_${IMAGE_PKGTYPE} ' + x + ':do_packagedata' for x in d.getVar('SDK_RDEPENDS').split()])}"
 do_populate_sdk[recrdeptask] += "do_packagedata do_package_write_rpm do_package_write_ipk do_package_write_deb"
+do_populate_sdk[file-checksums] += "${POSTINST_INTERCEPT_CHECKSUMS}"
 addtask populate_sdk
Signature · 1 line
2.11.1