buildhistory.bbclass: remove out-dated information on request

PATCH Superseded Patrick Ohly 3 participants 5 messages Feb 8, 2016 – Feb 12, 2016

lore

Share & tools

lore

Patrick Ohly

author

2016-02-08 06:57 UTC #0 lore raw

PATCH 1/1 buildhistory.bbclass: remove out-dated information on request

buildhistory.bbclass by design is incremental: each build adds or updates information. Information is never removed.

Sometimes it can be useful to reduce the information only to those recipes that were build during a specific bitbake invocation, for example when the invocation does a full world build.

This is now possible by invoking bitbake with:
BB_ENV_EXTRAWHITE=BUILDHISTORY_REMOVE_OLD BUILDHISTORY_REMOVE_OLD=1 bitbake

In this mode, buildhistory.bbclass first moves all existing information into a temporary directory called "old" inside the build history directory. There the information is used for the "version going backwards QA check". Then when the build is complete and before (potentially) committing to git, the temporary directory gets deleted.

Because information that has not changed during the build will be reconstructed, a git log will then only show real updates, additions and removals.

Signed-off-by Patrick Ohly <[email protected]>

---
 meta/classes/buildhistory.bbclass | 43 +++++++++++++++++++++++++++++++--------
 1 file changed, 34 insertions(+), 9 deletions(-)

meta/classes/buildhistory.bbclass +34 -9

index 3c4647a..1fadd2e 100644
@@ -11,6 +11,12 @@ BUILDHISTORY_FEATURES ?= "image package sdk"
 BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory"
 BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
 BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
+# Setting this to non-empty will remove the old content of the buildhistory as part of
+# the current bitbake invocation and replace it with information about what was built
+# during the build.
+BUILDHISTORY_REMOVE_OLD ?= ""
+BUILDHISTORY_OLD_DIR = "${BUILDHISTORY_DIR}/${@ "old" if "${BUILDHISTORY_REMOVE_OLD}" else ""}"
+BUILDHISTORY_OLD_DIR_PACKAGE = "${BUILDHISTORY_OLD_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
 BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}${SDK_EXT}/${IMAGE_BASENAME}"
 BUILDHISTORY_IMAGE_FILES ?= "/etc/passwd /etc/group"
 BUILDHISTORY_SDK_FILES ?= "conf/local.conf conf/bblayers.conf conf/auto.conf conf/locked-sigs.inc conf/devtool.conf"

@@ -49,6 +55,7 @@ python buildhistory_emit_pkghistory() {
     import errno
 
     pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True)
+    oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE', True)
 
     class RecipeInfo:
         def __init__(self, name):

@@ -139,7 +146,7 @@ python buildhistory_emit_pkghistory() {
 
     def getlastpkgversion(pkg):
         try:
-            histfile = os.path.join(pkghistdir, pkg, "latest")
+            histfile = os.path.join(oldpkghistdir, pkg, "latest")
             return readPackageInfo(pkg, histfile)
         except EnvironmentError:
             return None

@@ -722,17 +729,35 @@ END
 
 python buildhistory_eventhandler() {
     if e.data.getVar('BUILDHISTORY_FEATURES', True).strip():
-        if e.data.getVar("BUILDHISTORY_COMMIT", True) == "1":
-            bb.note("Writing buildhistory")
-            localdata = bb.data.createCopy(e.data)
-            localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures))
-            interrupted = getattr(e, '_interrupted', 0)
-            localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted))
-            bb.build.exec_func("buildhistory_commit", localdata)
+        remove = e.data.getVar("BUILDHISTORY_REMOVE_OLD", True)
+        olddir = e.data.getVar("BUILDHISTORY_OLD_DIR", True)
+        if isinstance(e, bb.event.BuildStarted):
+            if remove:
+                import shutil
+                # Clean up after potentially interrupted build.
+                if os.path.isdir(olddir):
+                    shutil.rmtree(olddir)
+                rootdir = e.data.getVar("BUILDHISTORY_DIR", True)
+                entries = [ x for x in os.listdir(rootdir) if not x.startswith('.') ]
+                bb.utils.mkdirhier(olddir)
+                for entry in entries:
+                    os.rename(os.path.join(rootdir, entry),
+                              os.path.join(olddir, entry))
+        elif isinstance(e, bb.event.BuildCompleted):
+            if remove:
+                import shutil
+                shutil.rmtree(olddir)
+            if e.data.getVar("BUILDHISTORY_COMMIT", True) == "1":
+                bb.note("Writing buildhistory")
+                localdata = bb.data.createCopy(e.data)
+                localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures))
+                interrupted = getattr(e, '_interrupted', 0)
+                localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted))
+                bb.build.exec_func("buildhistory_commit", localdata)
 }
 
 addhandler buildhistory_eventhandler
-buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted"
+buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted bb.event.BuildStarted"
 
 
 # FIXME this ought to be moved into the fetcher

Signature · 1 line

2.1.4

Richard Purdie

2016-02-08 08:52 UTC #1 lore raw

Quoted 18 lines · Patrick Ohly

buildhistory.bbclass by design is incremental: each build adds or updates information. Information is never removed.
Sometimes it can be useful to reduce the information only to those recipes that were build during a specific bitbake invocation, for example when the invocation does a full world build.
This is now possible by invoking bitbake with:
BB_ENV_EXTRAWHITE=BUILDHISTORY_REMOVE_OLD
BUILDHISTORY_REMOVE_OLD=1 bitbake
In this mode, buildhistory.bbclass first moves all existing information into a temporary directory called "old" inside the build history directory. There the information is used for the "version going backwards QA check". Then when the build is complete and before (potentially) committing to git, the temporary directory gets deleted.

Quoted 5 lines

Because information that has not changed during the build will be reconstructed, a git log will then only show real updates, additions and removals.
Signed-off-by Patrick Ohly <[email protected]>

The implicit assumption here is that you'd only every build large targets like "world". A "bitbake bash" isn't really going to do what you'd expect/want.

I think we need to make that clearer in the comments and that this is only useful on automated infrastructure.

Cheers,

Richard

Patrick Ohly

author

2016-02-08 12:35 UTC #2 lore raw

Quoted 23 lines · Richard Purdie

Quoted 22 lines · Patrick Ohly
buildhistory.bbclass by design is incremental: each build adds or updates information. Information is never removed.
Sometimes it can be useful to reduce the information only to those recipes that were build during a specific bitbake invocation, for example when the invocation does a full world build.
This is now possible by invoking bitbake with:
BB_ENV_EXTRAWHITE=BUILDHISTORY_REMOVE_OLD
BUILDHISTORY_REMOVE_OLD=1 bitbake
In this mode, buildhistory.bbclass first moves all existing information into a temporary directory called "old" inside the build history directory. There the information is used for the "version going backwards QA check". Then when the build is complete and before (potentially) committing to git, the temporary directory gets deleted.
Because information that has not changed during the build will be reconstructed, a git log will then only show real updates, additions and removals.

Quoted 5 lines

Quoted 1 line
Signed-off-by Patrick Ohly <[email protected]>
The implicit assumption here is that you'd only every build large targets like "world". A "bitbake bash" isn't really going to do what you'd expect/want.

Yes, I know. I think the "reduce the information only to those recipes that were built during a specific bitbake invocation" is a bit clearer about that than the last paragraph, which assumes a full build.

Quoted 2 lines

I think we need to make that clearer in the comments and that this is only useful on automated infrastructure.

I wouldn't say that. One could do "BUILDHISTORY_REMOVE_OLD=1 bitbake <something> && bitbake <something else> && bitbake <even more>", and that would work fine also when doing manual builds. The drawback in that case is that the version-going-backwards check does not work for <something else> and <even more>.

Perhaps it would be more appropriate to replace BUILDHISTORY_REMOVE_OLD with "BUILDHISTORY_RESET"?

Signature · 6 lines

Best Regards, Patrick Ohly

The content of this message is my personal opinion only and although
I am an employee of Intel, the statements I make here in no way
represent Intel's position on the issue, nor am I authorized to speak
on behalf of Intel on this matter.

Paul Eggleton

2016-02-10 12:31 UTC #3 lore raw

Quoted 5 lines · On Mon, 08 Feb 2016 13:35:20 Patrick Ohly wrote:

Quoted 4 lines · Richard Purdie
Quoted 3 lines · Patrick Ohly
buildhistory.bbclass by design is incremental: each build adds or updates information. Information is never removed.

Quoted 8 lines

Quoted 8 lines
Quoted 8 lines
Sometimes it can be useful to reduce the information only to those recipes that were build during a specific bitbake invocation, for example when the invocation does a full world build.
This is now possible by invoking bitbake with:
BB_ENV_EXTRAWHITE=BUILDHISTORY_REMOVE_OLD
BUILDHISTORY_REMOVE_OLD=1 bitbake

I think you'd be more likely to just set this in the auto.conf created by the CI infrastructure, but of course there's nothing to stop you doing it this way.

Quoted 24 lines

Quoted 11 lines
Quoted 11 lines
In this mode, buildhistory.bbclass first moves all existing information into a temporary directory called "old" inside the build history directory. There the information is used for the "version going backwards QA check". Then when the build is complete and before (potentially) committing to git, the temporary directory gets deleted.
Because information that has not changed during the build will be reconstructed, a git log will then only show real updates, additions and removals.
Quoted 5 lines
Quoted 1 line
Signed-off-by Patrick Ohly <[email protected]>
The implicit assumption here is that you'd only every build large targets like "world". A "bitbake bash" isn't really going to do what you'd expect/want.
Yes, I know. I think the "reduce the information only to those recipes that were built during a specific bitbake invocation" is a bit clearer about that than the last paragraph, which assumes a full build.
Quoted 2 lines
I think we need to make that clearer in the comments and that this is only useful on automated infrastructure.

Quoted 8 lines

I wouldn't say that. One could do "BUILDHISTORY_REMOVE_OLD=1 bitbake <something> && bitbake <something else> && bitbake <even more>", and that would work fine also when doing manual builds. The drawback in that case is that the version-going-backwards check does not work for <something else> and <even more>.
Perhaps it would be more appropriate to replace BUILDHISTORY_REMOVE_OLD with "BUILDHISTORY_RESET"?

FWIW, I'm reasonably happy with this if we add the suggested comments and rename the variable to BUILDHISTORY_RESET.

Cheers,
Paul

Signature · 2 lines

Paul Eggleton
Intel Open Source Technology Centre

Patrick Ohly

author

2016-02-12 08:37 UTC #4 lore raw

PATCH v2 1/1 buildhistory.bbclass: remove out-dated information on request

buildhistory.bbclass by design is incremental: each build adds or updates information. Information is never removed.

Sometimes it can be useful to reduce the information only to those recipes that were build during a specific bitbake invocation, for example when the invocation does a full world build.

This is now possible by setting BUILDHISTORY_RESET as explained in the modified class. The comment on the variable also mentions the caveats associated with using this mode.

In this mode, buildhistory.bbclass first moves all existing information into a temporary directory called "old" inside the build history directory. There the information is used for the "version going backwards" QA check. Then when the build is complete and before (potentially) committing to git, the temporary directory gets deleted.

Because information that has not changed during the build will be reconstructed during full world rebuilds, a git log will then only show real updates, additions and removals.

Signed-off-by Patrick Ohly <[email protected]>

---
 meta/classes/buildhistory.bbclass | 60 +++++++++++++++++++++++++++++++++------
 1 file changed, 51 insertions(+), 9 deletions(-)

meta/classes/buildhistory.bbclass +51 -9

index 3c4647a..b18fb84 100644
@@ -11,6 +11,29 @@ BUILDHISTORY_FEATURES ?= "image package sdk"
 BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory"
 BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
 BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
+
+# Setting this to non-empty will remove the old content of the buildhistory as part of
+# the current bitbake invocation and replace it with information about what was built
+# during the build.
+#
+# This is meant to be used in continuous integration (CI) systems when invoking bitbake
+# for full world builds. The effect in that case is that information about packages
+# that no longer get build also gets removed from the buildhistory, which is not
+# the case otherwise.
+#
+# The advantage over manually cleaning the buildhistory outside of bitbake is that
+# the "version-going-backwards" check still works. When relying on that, be careful
+# about failed world builds: they will lead to incomplete information in the
+# buildhistory because information about packages that could not be built will
+# also get removed. A CI system should handle that by discarding the buildhistory
+# of failed builds.
+#
+# The expected usage is via auto.conf, but passing via the command line also works
+# with: BB_ENV_EXTRAWHITE=BUILDHISTORY_RESET BUILDHISTORY_RESET=1
+BUILDHISTORY_RESET ?= ""
+
+BUILDHISTORY_OLD_DIR = "${BUILDHISTORY_DIR}/${@ "old" if "${BUILDHISTORY_RESET}" else ""}"
+BUILDHISTORY_OLD_DIR_PACKAGE = "${BUILDHISTORY_OLD_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
 BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}${SDK_EXT}/${IMAGE_BASENAME}"
 BUILDHISTORY_IMAGE_FILES ?= "/etc/passwd /etc/group"
 BUILDHISTORY_SDK_FILES ?= "conf/local.conf conf/bblayers.conf conf/auto.conf conf/locked-sigs.inc conf/devtool.conf"

@@ -49,6 +72,7 @@ python buildhistory_emit_pkghistory() {
     import errno
 
     pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True)
+    oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE', True)
 
     class RecipeInfo:
         def __init__(self, name):

@@ -139,7 +163,7 @@ python buildhistory_emit_pkghistory() {
 
     def getlastpkgversion(pkg):
         try:
-            histfile = os.path.join(pkghistdir, pkg, "latest")
+            histfile = os.path.join(oldpkghistdir, pkg, "latest")
             return readPackageInfo(pkg, histfile)
         except EnvironmentError:
             return None

@@ -722,17 +746,35 @@ END
 
 python buildhistory_eventhandler() {
     if e.data.getVar('BUILDHISTORY_FEATURES', True).strip():
-        if e.data.getVar("BUILDHISTORY_COMMIT", True) == "1":
-            bb.note("Writing buildhistory")
-            localdata = bb.data.createCopy(e.data)
-            localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures))
-            interrupted = getattr(e, '_interrupted', 0)
-            localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted))
-            bb.build.exec_func("buildhistory_commit", localdata)
+        reset = e.data.getVar("BUILDHISTORY_RESET", True)
+        olddir = e.data.getVar("BUILDHISTORY_OLD_DIR", True)
+        if isinstance(e, bb.event.BuildStarted):
+            if reset:
+                import shutil
+                # Clean up after potentially interrupted build.
+                if os.path.isdir(olddir):
+                    shutil.rmtree(olddir)
+                rootdir = e.data.getVar("BUILDHISTORY_DIR", True)
+                entries = [ x for x in os.listdir(rootdir) if not x.startswith('.') ]
+                bb.utils.mkdirhier(olddir)
+                for entry in entries:
+                    os.rename(os.path.join(rootdir, entry),
+                              os.path.join(olddir, entry))
+        elif isinstance(e, bb.event.BuildCompleted):
+            if reset:
+                import shutil
+                shutil.rmtree(olddir)
+            if e.data.getVar("BUILDHISTORY_COMMIT", True) == "1":
+                bb.note("Writing buildhistory")
+                localdata = bb.data.createCopy(e.data)
+                localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures))
+                interrupted = getattr(e, '_interrupted', 0)
+                localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted))
+                bb.build.exec_func("buildhistory_commit", localdata)
 }
 
 addhandler buildhistory_eventhandler
-buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted"
+buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted bb.event.BuildStarted"
 
 
 # FIXME this ought to be moved into the fetcher

Signature · 1 line

2.1.4