Skip to contentSkip to conversation

buildhistory: simplify buildhistory_list_files()

RFC PATCH Superseded Jacob Kroon 5 participants 12 messages Jan 6, 2019 – Jan 9, 2019

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/2 buildhistory: simplify buildhistory_list_files()

Avoid duplicating shell code for the two cases, fakeroot/non-fakeroot.

---
 meta/classes/buildhistory.bbclass | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
meta/classes/buildhistory.bbclass +6 -4
index d1f3e6aa82..33eb1b00f6 100644
@@ -519,12 +519,14 @@ buildhistory_get_sdk_installed_target() {
 
 buildhistory_list_files() {
 	# List the files in the specified directory, but exclude date/time etc.
-	# This awk script is somewhat messy, but handles where the size is not printed for device files under pseudo
+	# This is somewhat messy, but handles where the size is not printed for device files under pseudo
+	( cd $1
+	find_cmd='find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n"'
 	if [ "$3" = "fakeroot" ] ; then
-		( cd $1 && ${FAKEROOTENV} ${FAKEROOTCMD} find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n" | sort -k5 | sed 's/ * -> $//' > $2 )
+		eval ${FAKEROOTENV} ${FAKEROOTCMD} $find_cmd
 	else
-		( cd $1 && find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n" | sort -k5 | sed 's/ * -> $//' > $2 )
-	fi
+		eval $find_cmd
+	fi | sort -k5 | sed 's/ * -> $//' > $2 )
 }
 
 buildhistory_list_pkg_files() {
Signature · 1 line
2.11.0
#1 lore raw

RFC PATCH 2/2 buildhistory: support generating md5sum of files

Introduce 'md5' in BUILDHISTORY_FEATURES and enable it by default when doing reproducible builds.

When enabled this will additionally create:

files-in-package-md5.txt
files-in-image-md5.txt
files-in-sdk-md5.txt

containing the md5 checksums of regular files.

---
 meta/classes/buildhistory.bbclass | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
meta/classes/buildhistory.bbclass +8 -2
index 33eb1b00f6..00f0701dec 100644
@@ -7,7 +7,8 @@
 # Copyright (C) 2007-2011 Koen Kooi <[email protected]>
 #
 
-BUILDHISTORY_FEATURES ?= "image package sdk"
+BUILDHISTORY_FEATURES ?= "image package sdk \
+  ${@ "md5" if bb.utils.to_boolean(d.getVar('BUILD_REPRODUCIBLE_BINARIES')) else ""}"
 BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory"
 BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
 BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
@@ -526,7 +527,12 @@ buildhistory_list_files() {
 		eval ${FAKEROOTENV} ${FAKEROOTCMD} $find_cmd
 	else
 		eval $find_cmd
-	fi | sort -k5 | sed 's/ * -> $//' > $2 )
+	fi | sort -k5 | sed 's/ * -> $//' > $2
+	if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'md5', '1', '0', d)}" = "1" ] ; then
+		md5filename=$(echo $2 | sed 's/\.txt$/-md5.txt/')
+		find -type f | xargs -I{} -n1 md5sum {} | sort -k2 > $md5filename
+		[ -s $md5filename ] || rm $md5filename # remove result if empty
+	fi )
 }
 
 buildhistory_list_pkg_files() {
Signature · 1 line
2.11.0
#2 lore raw

Re: [RFC][PATCH 2/2] buildhistory: support generating md5sum of files

Hi,

Quoted 44 lines · Jacob Kroon

Introduce 'md5' in BUILDHISTORY_FEATURES and enable it by default when doing reproducible builds.

When enabled this will additionally create:

files-in-package-md5.txt
files-in-image-md5.txt
files-in-sdk-md5.txt

containing the md5 checksums of regular files.

---
 meta/classes/buildhistory.bbclass | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
meta/classes/buildhistory.bbclass +2 -1
index 33eb1b00f6..00f0701dec 100644
@@ -7,7 +7,8 @@
 # Copyright (C) 2007-2011 Koen Kooi <[email protected]>
 #
 
-BUILDHISTORY_FEATURES ?= "image package sdk"
+BUILDHISTORY_FEATURES ?= "image package sdk \
+  ${@ "md5" if

bb.utils.to_boolean(d.getVar('BUILD_REPRODUCIBLE_BINARIES')) else ""}"
BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory" BUILDHISTORY_DIR_IMAGE =
"${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
BUILDHISTORY_DIR_PACKAGE =
"${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"

meta/classes/buildhistory.bbclass +2 -1
@@ -526,7 +527,12 @@ buildhistory_list_files() {
 		eval ${FAKEROOTENV} ${FAKEROOTCMD} $find_cmd
 	else
 		eval $find_cmd
-	fi | sort -k5 | sed 's/ * -> $//' > $2 )
+	fi | sort -k5 | sed 's/ * -> $//' > $2
+	if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'md5', '1', '0',

d)}" = "1" ] ; then

meta/classes/buildhistory.bbclass +2 -0
+		md5filename=$(echo $2 | sed 's/\.txt$/-md5.txt/')
+		find -type f | xargs -I{} -n1 md5sum {} | sort -k2 >
Quoted 1 line

$md5filename

Why don't you
find . -type f -exec md5sum {} + | sort -sk2 > $md5filename
?
It'll be quite a bit faster because way fewer processes will be spawned.

Am I missing something?

I don't know what the intended use-case of the md5 files is, but could sha256 or similar maybe be more appropriate?

Cheers,
Andre'

Quoted 9 lines
meta/classes/buildhistory.bbclass +1 -0
+		[ -s $md5filename ] || rm $md5filename # remove result if

empty

meta/classes/buildhistory.bbclass +1 -0
+	fi )
 }
 
 buildhistory_list_pkg_files() {
Signature · 1 line
2.11.0
#3 lore raw

Re: [RFC][PATCH 2/2] buildhistory: support generating md5sum of files

Hi André,

Quoted 55 lines · André Draszik

Hi,

Quoted 44 lines · Jacob Kroon

Introduce 'md5' in BUILDHISTORY_FEATURES and enable it by default when doing reproducible builds.

When enabled this will additionally create:

files-in-package-md5.txt
files-in-image-md5.txt
files-in-sdk-md5.txt

containing the md5 checksums of regular files.

---
 meta/classes/buildhistory.bbclass | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
meta/classes/buildhistory.bbclass +2 -1
index 33eb1b00f6..00f0701dec 100644
@@ -7,7 +7,8 @@
 # Copyright (C) 2007-2011 Koen Kooi <[email protected]>
 #
 
-BUILDHISTORY_FEATURES ?= "image package sdk"
+BUILDHISTORY_FEATURES ?= "image package sdk \
+  ${@ "md5" if

bb.utils.to_boolean(d.getVar('BUILD_REPRODUCIBLE_BINARIES')) else ""}"
BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory" BUILDHISTORY_DIR_IMAGE =
"${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
BUILDHISTORY_DIR_PACKAGE =
"${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"

meta/classes/buildhistory.bbclass +2 -1
@@ -526,7 +527,12 @@ buildhistory_list_files() {
              eval ${FAKEROOTENV} ${FAKEROOTCMD} $find_cmd
      else
              eval $find_cmd
-     fi | sort -k5 | sed 's/ * -> $//' > $2 )
+     fi | sort -k5 | sed 's/ * -> $//' > $2
+     if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'md5', '1', '0',

d)}" = "1" ] ; then

meta/classes/buildhistory.bbclass +2 -0
+             md5filename=$(echo $2 | sed 's/\.txt$/-md5.txt/')
+             find -type f | xargs -I{} -n1 md5sum {} | sort -k2 >
Quoted 1 line

$md5filename

Why don't you
find . -type f -exec md5sum {} + | sort -sk2 > $md5filename
?
It'll be quite a bit faster because way fewer processes will be spawned.

Quoted 1 line

Am I missing something?

You're right, I will update the patch. I'm assuming I don't need the stable sort, -s,
since the filenames should all be unique.

Quoted 2 lines

I don't know what the intended use-case of the md5 files is, but could sha256 or similar maybe be more appropriate?

I thought it would be a good idea to store some sort of checksum of files in the buildhistory when doing reproducible builds, so that it is easier to detect when a rebuild produces changed files, but perhaps there is some way to do this already that I am missing ?

But I have no real motivation for choosing md5, other than that I assumed it would be less
cpu intensive than sha256, and the fact I'm not too worried about collisions.

Thanks for the feedback,
Jacob

Quoted 19 lines

Cheers,
Andre'

Quoted 9 lines

+ [ -s $md5filename ] || rm $md5filename # remove result if
empty
+ fi )
}

buildhistory_list_pkg_files() {
--
2.11.0

--
_______________________________________________
Openembedded-core mailing list
[email protected]
http://lists.openembedded.org/mailman/listinfo/openembedded-core

#4 lore raw

Re: [RFC][PATCH 2/2] buildhistory: support generating md5sum of files

Quoted 13 lines · Jacob Kroon
Quoted 3 lines · André Draszik

I don't know what the intended use-case of the md5 files is, but could
sha256 or similar maybe be more appropriate?

I thought it would be a good idea to store some sort of checksum of files in the
buildhistory when doing reproducible builds, so that it is easier to detect
when a rebuild produces changed files, but perhaps there is some way to do
this already that I am missing ?

Quoted 4 lines

But I have no real motivation for choosing md5, other than that I assumed it would be less
cpu intensive than sha256, and the fact I'm not too worried about collisions.

I used to lean that way as well however there is a fair bit of negativity associated with *any* use of md5 so I'm tempted to say use sha256 as we don't know how these checksums may be used in the future.

I'd note there is some overlap here with the sstate hash equivalency work where we build a hash representing a tasks complete output. Can you reuse any of the functions there?

Cheers,

Richard

#5 lore raw

Re: [RFC][PATCH 2/2] buildhistory: support generating md5sum of files

Hi Richard,

Quoted 15 lines · Richard Purdie
Quoted 13 lines · Jacob Kroon
Quoted 3 lines · André Draszik

I don't know what the intended use-case of the md5 files is, but could
sha256 or similar maybe be more appropriate?

I thought it would be a good idea to store some sort of checksum of files in the
buildhistory when doing reproducible builds, so that it is easier to detect
when a rebuild produces changed files, but perhaps there is some way to do
this already that I am missing ?

Quoted 8 lines
Quoted 4 lines

But I have no real motivation for choosing md5, other than that I assumed it would be less
cpu intensive than sha256, and the fact I'm not too worried about collisions.

I used to lean that way as well however there is a fair bit of negativity associated with *any* use of md5 so I'm tempted to say use sha256 as we don't know how these checksums may be used in the future.

Ok.

Quoted 3 lines

I'd note there is some overlap here with the sstate hash equivalency work where we build a hash representing a tasks complete output. Can you reuse any of the functions there?

I'll look into it, I realize it would of course be nice if we only needed to calculate the checksums once.

Thanks for the feedback,
Jacob

Quoted 4 lines

Cheers,

Richard

#6 lore raw

Re: [RFC][PATCH 2/2] buildhistory: support generating md5sum of files

In reply to Jacob Kroon
Quoted 32 lines · Jacob Kroon

Introduce 'md5' in BUILDHISTORY_FEATURES and enable it by default when doing reproducible builds.

When enabled this will additionally create:

files-in-package-md5.txt
files-in-image-md5.txt
files-in-sdk-md5.txt

containing the md5 checksums of regular files.

---
 meta/classes/buildhistory.bbclass | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
meta/classes/buildhistory.bbclass +2 -1
index 33eb1b00f6..00f0701dec 100644
@@ -7,7 +7,8 @@
 # Copyright (C) 2007-2011 Koen Kooi <[email protected]>
 #
 
-BUILDHISTORY_FEATURES ?= "image package sdk"
+BUILDHISTORY_FEATURES ?= "image package sdk \
+  ${@ "md5" if bb.utils.to_boolean(d.getVar('BUILD_REPRODUCIBLE_BINARIES')) else ""}"
 BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory"
 BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
 BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"

meta/classes/buildhistory.bbclass buildhistory_list_files() { · line 534

Quoted 9 lines
meta/classes/buildhistory.bbclass +5 -1
@@ -526,7 +527,12 @@ buildhistory_list_files() {
                eval ${FAKEROOTENV} ${FAKEROOTCMD} $find_cmd
        else
                eval $find_cmd
-       fi | sort -k5 | sed 's/ * -> $//' > $2 )
+       fi | sort -k5 | sed 's/ * -> $//' > $2
+       if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'md5', '1', '0', d)}" = "1" ] ; then
+               md5filename=$(echo $2 | sed 's/\.txt$/-md5.txt/')
+               find -type f | xargs -I{} -n1 md5sum {} | sort -k2 > $md5filename
+               [ -s $md5filename ] || rm $md5filename # remove result if empty

I added this remove because I thought it didn't make sense to keep empty files around, but I now realize that the "files-in-package.txt" file is kept around, even if empty. Is there a preference on what to do here ?

Quoted 7 lines
meta/classes/buildhistory.bbclass +1 -1
+       fi )
 }
 
 buildhistory_list_pkg_files() {
--

2.11.0

#7 lore raw

Re: [RFC][PATCH 2/2] buildhistory: support generating md5sum of files

Quoted 39 lines · Jacob Kroon
Quoted 38 lines · Jacob Kroon

Introduce 'md5' in BUILDHISTORY_FEATURES and enable it by default when doing reproducible builds.

When enabled this will additionally create:

files-in-package-md5.txt
files-in-image-md5.txt
files-in-sdk-md5.txt

containing the md5 checksums of regular files.

---
 meta/classes/buildhistory.bbclass | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
meta/classes/buildhistory.bbclass +4 -2
index 33eb1b00f6..00f0701dec 100644
@@ -7,7 +7,8 @@
 # Copyright (C) 2007-2011 Koen Kooi <[email protected]>
 #
 
-BUILDHISTORY_FEATURES ?= "image package sdk"
+BUILDHISTORY_FEATURES ?= "image package sdk \
+  ${@ "md5" if bb.utils.to_boolean(d.getVar('BUILD_REPRODUCIBLE_BINARIES')) else ""}"
 BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory"
 BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
 BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
@@ -526,7 +527,12 @@ buildhistory_list_files() {
                eval ${FAKEROOTENV} ${FAKEROOTCMD} $find_cmd
        else
                eval $find_cmd
-       fi | sort -k5 | sed 's/ * -> $//' > $2 )
+       fi | sort -k5 | sed 's/ * -> $//' > $2
+       if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'md5', '1', '0', d)}" = "1" ] ; then
Quoted 8 lines

meta/classes/buildhistory.bbclass buildhistory_list_files() { · line 534

Quoted 3 lines
meta/classes/buildhistory.bbclass +3 -0
+               md5filename=$(echo $2 | sed 's/\.txt$/-md5.txt/')
+               find -type f | xargs -I{} -n1 md5sum {} | sort -k2 > $md5filename
+               [ -s $md5filename ] || rm $md5filename # remove result if empty

I added this remove because I thought it didn't make sense to keep empty files around, but I now realize that the "files-in-package.txt" file is kept around, even if empty. Is there a preference on what to do here ?

FWIW, I'm wiping the all buildhistory data with external scripts before doing a clean build. Basically a "git rm -rf *" in buildhistory directory. Otherwise stale data about images, packages etc which are no longer built remain in buildhistory.

-Mikko

#8 lore raw

Re: [RFC][PATCH 2/2] buildhistory: support generating md5sum of files

Quoted 50 lines · On Tue, Jan 8, 2019 at 11:32 AM <[email protected]> wrote:
Quoted 39 lines · Jacob Kroon
Quoted 38 lines · Jacob Kroon

Introduce 'md5' in BUILDHISTORY_FEATURES and enable it by default when doing reproducible builds.

When enabled this will additionally create:

files-in-package-md5.txt
files-in-image-md5.txt
files-in-sdk-md5.txt

containing the md5 checksums of regular files.

---
 meta/classes/buildhistory.bbclass | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
meta/classes/buildhistory.bbclass +4 -2
index 33eb1b00f6..00f0701dec 100644
@@ -7,7 +7,8 @@
 # Copyright (C) 2007-2011 Koen Kooi <[email protected]>
 #
 
-BUILDHISTORY_FEATURES ?= "image package sdk"
+BUILDHISTORY_FEATURES ?= "image package sdk \
+  ${@ "md5" if bb.utils.to_boolean(d.getVar('BUILD_REPRODUCIBLE_BINARIES')) else ""}"
 BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory"
 BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
 BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
@@ -526,7 +527,12 @@ buildhistory_list_files() {
                eval ${FAKEROOTENV} ${FAKEROOTCMD} $find_cmd
        else
                eval $find_cmd
-       fi | sort -k5 | sed 's/ * -> $//' > $2 )
+       fi | sort -k5 | sed 's/ * -> $//' > $2
+       if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'md5', '1', '0', d)}" = "1" ] ; then
Quoted 8 lines

meta/classes/buildhistory.bbclass buildhistory_list_files() { · line 534

Quoted 3 lines
meta/classes/buildhistory.bbclass +3 -0
+               md5filename=$(echo $2 | sed 's/\.txt$/-md5.txt/')
+               find -type f | xargs -I{} -n1 md5sum {} | sort -k2 > $md5filename
+               [ -s $md5filename ] || rm $md5filename # remove result if empty

I added this remove because I thought it didn't make sense to keep empty files around, but I now realize that the "files-in-package.txt" file is kept around, even if empty. Is there a preference on what to do here ?

Quoted 4 lines

FWIW, I'm wiping the all buildhistory data with external scripts before doing a clean build. Basically a "git rm -rf *" in buildhistory directory. Otherwise stale data about images, packages etc which are no longer built remain in buildhistory.

I think one can also do:

BB_ENV_EXTRAWHITE=BUILDHISTORY_RESET BUILDHISTORY_RESET=1 bitbake <build>

(see buildhistory.bbclass)

Quoted 1 line

-Mikko

#9 lore raw

Re: [RFC][PATCH 2/2] buildhistory: support generating md5sum of files

Quoted 56 lines · Jacob Kroon
Quoted 50 lines · On Tue, Jan 8, 2019 at 11:32 AM <[email protected]> wrote:
Quoted 39 lines · Jacob Kroon
Quoted 38 lines · Jacob Kroon

Introduce 'md5' in BUILDHISTORY_FEATURES and enable it by default when doing reproducible builds.

When enabled this will additionally create:

files-in-package-md5.txt
files-in-image-md5.txt
files-in-sdk-md5.txt

containing the md5 checksums of regular files.

---
 meta/classes/buildhistory.bbclass | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
meta/classes/buildhistory.bbclass +4 -2
index 33eb1b00f6..00f0701dec 100644
@@ -7,7 +7,8 @@
 # Copyright (C) 2007-2011 Koen Kooi <[email protected]>
 #
 
-BUILDHISTORY_FEATURES ?= "image package sdk"
+BUILDHISTORY_FEATURES ?= "image package sdk \
+  ${@ "md5" if bb.utils.to_boolean(d.getVar('BUILD_REPRODUCIBLE_BINARIES')) else ""}"
 BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory"
 BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
 BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
@@ -526,7 +527,12 @@ buildhistory_list_files() {
                eval ${FAKEROOTENV} ${FAKEROOTCMD} $find_cmd
        else
                eval $find_cmd
-       fi | sort -k5 | sed 's/ * -> $//' > $2 )
+       fi | sort -k5 | sed 's/ * -> $//' > $2
+       if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'md5', '1', '0', d)}" = "1" ] ; then
Quoted 8 lines

meta/classes/buildhistory.bbclass buildhistory_list_files() { · line 534

Quoted 3 lines
meta/classes/buildhistory.bbclass +3 -0
+               md5filename=$(echo $2 | sed 's/\.txt$/-md5.txt/')
+               find -type f | xargs -I{} -n1 md5sum {} | sort -k2 > $md5filename
+               [ -s $md5filename ] || rm $md5filename # remove result if empty

I added this remove because I thought it didn't make sense to keep empty files around, but I now realize that the "files-in-package.txt" file is kept around, even if empty. Is there a preference on what to do here ?

Quoted 4 lines

FWIW, I'm wiping the all buildhistory data with external scripts before doing a clean build. Basically a "git rm -rf *" in buildhistory directory. Otherwise stale data about images, packages etc which are no longer built remain in buildhistory.

Quoted 5 lines

I think one can also do:

BB_ENV_EXTRAWHITE=BUILDHISTORY_RESET BUILDHISTORY_RESET=1 bitbake <build>

(see buildhistory.bbclass)

Thanks for the hint!

-Mikko

#10 lore raw

Re: [RFC][PATCH 2/2] buildhistory: support generating md5sum of files

In reply to Jacob Kroon
Quoted 30 lines

-----Original Message-----
From: [email protected] <openembedded- [email protected]> On Behalf Of Jacob Kroon
Sent: den 6 januari 2019 19:14
To: [email protected]
Subject: [OE-core] [RFC][PATCH 2/2] buildhistory: support generating md5sum of files

Introduce 'md5' in BUILDHISTORY_FEATURES and enable it by default when doing reproducible builds.

When enabled this will additionally create:

files-in-package-md5.txt
files-in-image-md5.txt
files-in-sdk-md5.txt

containing the md5 checksums of regular files.

---
 meta/classes/buildhistory.bbclass | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
meta/classes/buildhistory.bbclass +0 -0
index 33eb1b00f6..00f0701dec 100644

meta/classes/buildhistory.bbclass line 11

Quoted 6 lines
meta/classes/buildhistory.bbclass +2 -1
@@ -7,7 +7,8 @@
 # Copyright (C) 2007-2011 Koen Kooi <[email protected]>
 #
 
-BUILDHISTORY_FEATURES ?= "image package sdk"
+BUILDHISTORY_FEATURES ?= "image package sdk \
+  ${@ "md5" if bb.utils.to_boolean(d.getVar('BUILD_REPRODUCIBLE_BINARIES')) else ""}"

May I suggest using ' instead of " within the Python block to lessen confusion when quotes are used within another set of quotes. (Even if bitbake handles it fine, not all editor coloring modes do.)

Quoted 18 lines
meta/classes/buildhistory.bbclass +6 -2
 BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory"
 BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
 BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
@@ -526,7 +527,12 @@ buildhistory_list_files() {
 		eval ${FAKEROOTENV} ${FAKEROOTCMD} $find_cmd
 	else
 		eval $find_cmd
-	fi | sort -k5 | sed 's/ * -> $//' > $2 )
+	fi | sort -k5 | sed 's/ * -> $//' > $2
+	if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'md5', '1', '0', d)}" = "1" ] ; then
+		md5filename=$(echo $2 | sed 's/\.txt$/-md5.txt/')
+		find -type f | xargs -I{} -n1 md5sum {} | sort -k2 > $md5filename
+		[ -s $md5filename ] || rm $md5filename # remove result if empty
+	fi )
 }
 
 buildhistory_list_pkg_files() {
--
Quoted 1 line

2.11.0

Don't forget to update the documentation for BUILDHISTORY_FEATURES if this is added.

//Peter

#11 lore raw

Re: [RFC][PATCH 2/2] buildhistory: support generating md5sum of files

Quoted 34 lines · Peter Kjellerstedt
Quoted 34 lines

-----Original Message-----
From: [email protected] <openembedded- [email protected]> On Behalf Of Jacob Kroon
Sent: den 6 januari 2019 19:14
To: [email protected]
Subject: [OE-core] [RFC][PATCH 2/2] buildhistory: support generating md5sum of files

Introduce 'md5' in BUILDHISTORY_FEATURES and enable it by default when doing reproducible builds.

When enabled this will additionally create:

files-in-package-md5.txt
files-in-image-md5.txt
files-in-sdk-md5.txt

containing the md5 checksums of regular files.

---
 meta/classes/buildhistory.bbclass | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
meta/classes/buildhistory.bbclass +0 -0
index 33eb1b00f6..00f0701dec 100644
@@ -7,7 +7,8 @@
 # Copyright (C) 2007-2011 Koen Kooi <[email protected]>
 #
 
Quoted 8 lines

meta/classes/buildhistory.bbclass line 11

Quoted 3 lines
meta/classes/buildhistory.bbclass +2 -1
-BUILDHISTORY_FEATURES ?= "image package sdk"
+BUILDHISTORY_FEATURES ?= "image package sdk \
+  ${@ "md5" if bb.utils.to_boolean(d.getVar('BUILD_REPRODUCIBLE_BINARIES')) else ""}"

May I suggest using ' instead of " within the Python block to lessen confusion when quotes are used within another set of quotes. (Even if bitbake handles it fine, not all editor coloring modes do.)

Ok.

Quoted 16 lines
Quoted 16 lines
meta/classes/buildhistory.bbclass +6 -1
 BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory"
 BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
 BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
@@ -526,7 +527,12 @@ buildhistory_list_files() {
              eval ${FAKEROOTENV} ${FAKEROOTCMD} $find_cmd
      else
              eval $find_cmd
-     fi | sort -k5 | sed 's/ * -> $//' > $2 )
+     fi | sort -k5 | sed 's/ * -> $//' > $2
+     if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'md5', '1', '0', d)}" = "1" ] ; then
+             md5filename=$(echo $2 | sed 's/\.txt$/-md5.txt/')
+             find -type f | xargs -I{} -n1 md5sum {} | sort -k2 > $md5filename
+             [ -s $md5filename ] || rm $md5filename # remove result if empty
+     fi )
 }
 
Quoted 7 lines
Quoted 3 lines
meta/classes/buildhistory.bbclass +0 -1
 buildhistory_list_pkg_files() {
--

2.11.0

Don't forget to update the documentation for BUILDHISTORY_FEATURES if this is added.

Oops, yeah I forgot that.

I'ill send a V2 with all comments addressed.

It does seem like once the equivalence work is ready and OE hashes its outputs, much in buildhistory.bbclass can be replaced with already prepared information. But for the cases where it is not used, and considering how small the change is, maybe it makes sense to add it anyway.

Thanks for the feedback,
Jacob

Quoted 2 lines

//Peter