Skip to contentSkip to conversation

improve reproducibility of 5 recipes

PATCH New Hongxu Jia 1 participant 6 messages Feb 5, 2018

lore
Share & tools
lore

Hongxu Jia

author
#0 lore raw

v2 cover improve reproducibility of 5 recipes

Changed in V2:

- Rebase on top of targeted branch master a0988c3374

//Hongxu

The following changes since commit 385944254d0ef88bd6450a221a54cbcb40020b42:

tcmode-default.inc: drop preferred version of gzip-native (2018-01-31 17:01:20 +0000)

are available in the git repository at:

git://git.pokylinux.org/poky-contrib hongxu/reproduce
http://git.pokylinux.org/cgit.cgi/poky-contrib/log/?h=hongxu/reproduce

Hongxu Jia (5):
python-xcbgen: improve reproducibility openssl/nativesdk-openssl: improve reproducibility
python3-core: improve reproducibility python-core/python-dev: improve reproducibility
kbd-ptest: improve reproducibility

 .../openssl/0001-improve-reproducibility.patch     | 50 ++++++++++++++++++++++
 .../recipes-connectivity/openssl/openssl_1.0.2m.bb | 11 +++++
 meta/recipes-core/kbd/kbd_2.0.4.bb                 |  7 +++
 meta/recipes-devtools/python/python3_3.5.3.bb      | 13 +++++-
 meta/recipes-devtools/python/python_2.7.14.bb      |  7 ++-
 meta/recipes-graphics/xorg-proto/xcb-proto_1.12.bb |  2 +-
 6 files changed, 87 insertions(+), 3 deletions(-)
 create mode 100644 meta/recipes-connectivity/openssl/openssl/0001-improve-reproducibility.patch
Signature · 1 line
2.8.1
author
#1 lore raw

PATCH v2 1/5 python-xcbgen: improve reproducibility

Use relative path to generate .pyc files could remove build host references which leads to non-reproducible builds.

---
 meta/recipes-graphics/xorg-proto/xcb-proto_1.12.bb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
meta/recipes-graphics/xorg-proto/xcb-proto_1.12.bb +1 -1
index bc04bc4..be6b1a0 100644
@@ -36,5 +36,5 @@ do_install_append() {
     # Makefile's do_install creates .pyc files for python3, now also create
     # them for python2 so that they will be recorded by manifest, and can be
     # cleaned correctly.
-    python -m py_compile ${D}${libdir}/xcb-proto/xcbgen/*.py
+    (cd ${D}; python -m py_compile ./${libdir}/xcb-proto/xcbgen/*.py)
 }
Signature · 1 line
2.8.1
author
#2 lore raw

PATCH v2 2/5 openssl/nativesdk-openssl: improve reproducibility

In reply to Hongxu Jia

Remove build host references from the internally generated file buildinf.h. The references get compiled into executables, which leads to non-reproducible builds. The removed references (--sysroot, -fdebug-prefix-map) were only used as part of the `openssl version -f' which do not have side effect.

The fix apply for target/nativesdk, and does not affect native.

---
 .../openssl/0001-improve-reproducibility.patch     | 50 ++++++++++++++++++++++
 .../recipes-connectivity/openssl/openssl_1.0.2m.bb | 11 +++++
 2 files changed, 61 insertions(+)
 create mode 100644 meta/recipes-connectivity/openssl/openssl/0001-improve-reproducibility.patch
meta/recipes-connectivity/openssl/openssl/0001-improve-reproducibility.patch new file +50 -0
new file mode 100644
index 0000000..35edaf5
@@ -0,0 +1,50 @@
+From e4a70a64877c084cd90a84d0888c994cbb679dc4 Mon Sep 17 00:00:00 2001
+From: Hongxu Jia <[email protected]>
+Date: Thu, 11 Jan 2018 09:41:03 +0800
+Subject: [PATCH] crypto/Makefile: improve reproducibility
+
+Remove build host references from the internally
+generated file buildinf.h. The references get compiled into
+executables, which leads to non-reproducible builds.
+The removed references (--sysroot, -fdebug-prefix-map) were
+only used as part of the `openssl version -f' which do not have
+side effect.
+
+...
+$ openssl version -f
+compiler: cc -I. -I.. -I../include  -fPIC -DOPENSSL_PIC
+-DOPENSSL_THREADS -D_REENTRANT -DDSO_DLFCN -DHAVE_DLFCN_H -m64
+-DL_ENDIAN -g -O2 -fstack-protector-strong -Wformat
+-Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2
+-Wl,-Bsymbolic-functions -Wl,-z,relro -Wa,--noexecstack -Wall
+-DMD32_REG_T=int -DOPENSSL_IA32_SSE2 -DOPENSSL_BN_ASM_MONT
+-DOPENSSL_BN_ASM_MONT5 -DOPENSSL_BN_ASM_GF2m -DSHA1_ASM -DSHA256_ASM
+-DSHA512_ASM -DMD5_ASM -DAES_ASM -DVPAES_ASM -DBSAES_ASM -DWHIRLPOOL_ASM
+-DGHASH_ASM -DECP_NISTZ256_ASM
+...
+
+Upstream-Status: Inappropriate [oe-core specific]
+
+Signed-off-by: Hongxu Jia <[email protected]>
+---
+ crypto/Makefile | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/crypto/Makefile b/crypto/Makefile
+index 29c2dcf..4851026 100644
+--- a/crypto/Makefile
++++ b/crypto/Makefile
+@@ -54,8 +54,9 @@ top:
+ 
+ all: shared
+ 
++COMPILER = $(shell echo "$(CC) $(CFLAGS)" | sed -e "s,$(TOOLCHAIN_OPTIONS),,g" -e "s,$(DEBUG_PREFIX_MAP),,g")
+ buildinf.h: ../Makefile
+-	$(PERL) $(TOP)/util/mkbuildinf.pl "$(CC) $(CFLAGS)" "$(PLATFORM)" >buildinf.h
++	$(PERL) $(TOP)/util/mkbuildinf.pl "$(COMPILER)" "$(PLATFORM)" >buildinf.h
+ 
+ x86cpuid.s:	x86cpuid.pl perlasm/x86asm.pl
+ 	$(PERL) x86cpuid.pl $(PERLASM_SCHEME) $(CFLAGS) $(PROCESSOR) > $@
+-- 
+1.8.3.1
+
meta/recipes-connectivity/openssl/openssl_1.0.2m.bb +11 -0
index 9270f52..4906aec 100644
@@ -43,10 +43,21 @@ SRC_URI += "file://find.pl;subdir=openssl-${PV}/util/ \
             file://0001-Fix-build-with-clang-using-external-assembler.patch \
             file://0001-openssl-force-soft-link-to-avoid-rare-race.patch  \
             file://0001-aes-armv4-bsaes-armv7-sha256-armv4-.pl-make-it-work-.patch \
+            ${SRC_URI_REPRODUCE} \
             "
 SRC_URI[md5sum] = "10e9e37f492094b9ef296f68f24a7666"
 SRC_URI[sha256sum] = "8c6ff15ec6b319b50788f42c7abc2890c08ba5a1cdcd3810eb9092deada37b0f"
 
+SRC_URI_REPRODUCE = "file://0001-improve-reproducibility.patch"
+SRC_URI_REPRODUCE_class-native = ""
+
+EXTRA_OEMAKE += "${EXTRA_OEMAKE_REPRODUCE}"
+EXTRA_OEMAKE_REPRODUCE_class-native = ""
+EXTRA_OEMAKE_REPRODUCE  = ' \
+                 TOOLCHAIN_OPTIONS="${TOOLCHAIN_OPTIONS}" \
+                 DEBUG_PREFIX_MAP="${DEBUG_PREFIX_MAP}" \
+'
+
 PACKAGES =+ "${PN}-engines"
 FILES_${PN}-engines = "${libdir}/ssl/engines/*.so ${libdir}/engines"
 
Signature · 1 line
2.8.1
author
#3 lore raw

PATCH v2 3/5 python3-core: improve reproducibility

In reply to Hongxu Jia

Since modify _sysconfigdata.py to improve reproducibility, its pyo files should be recompiled.

Remove remaining build host references of python3.5m-config

---
 meta/recipes-devtools/python/python3_3.5.3.bb | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)
meta/recipes-devtools/python/python3_3.5.3.bb +12 -1
index 3ae0db2..dfdaa92 100644
@@ -194,7 +194,18 @@ py_package_preprocess () {
 		-e 's:${BASE_WORKDIR}/${MULTIMACH_TARGET_SYS}::g' \
 		${PKGD}/${libdir}/python${PYTHON_MAJMIN}/config/Makefile \
 		${PKGD}/${libdir}/python${PYTHON_MAJMIN}/config-${PYTHON_MAJMIN}${PYTHON_ABI}/Makefile \
-		${PKGD}/${libdir}/python${PYTHON_MAJMIN}/_sysconfigdata.py
+		${PKGD}/${libdir}/python${PYTHON_MAJMIN}/_sysconfigdata.py \
+		${PKGD}/${bindir}/python${PYTHON_BINABI}-config
+
+	# Recompile _sysconfigdata after modifying it
+	cd ${PKGD}
+	${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} \
+	     -c "from py_compile import compile; compile('./${libdir}/python${PYTHON_MAJMIN}/_sysconfigdata.py')"
+	${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} \
+	     -c "from py_compile import compile; compile('./${libdir}/python${PYTHON_MAJMIN}/_sysconfigdata.py', optimize=1)"
+	${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN} \
+	     -c "from py_compile import compile; compile('./${libdir}/python${PYTHON_MAJMIN}/_sysconfigdata.py', optimize=2)"
+	cd -
 }
 
 # manual dependency additions
Signature · 1 line
2.8.1
author
#4 lore raw

PATCH v2 4/5 python-core/python-dev: improve reproducibility

In reply to Hongxu Jia

- Remove remaining build host references from packaged files. (sync with python3)

- Use relative path to recompile _sysconfigdata.py

---
 meta/recipes-devtools/python/python_2.7.14.bb | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)
meta/recipes-devtools/python/python_2.7.14.bb +6 -1
index 7d03bc8..0a09926 100644
@@ -141,9 +141,14 @@ py_package_preprocess () {
 
 	# Remove references to buildmachine paths in target Makefile and _sysconfigdata
 	sed -i -e 's:--sysroot=${STAGING_DIR_TARGET}::g' -e s:'--with-libtool-sysroot=${STAGING_DIR_TARGET}'::g \
+		-e 's|${DEBUG_PREFIX_MAP}||g' \
+		-e 's:${HOSTTOOLS_DIR}/::g' \
+		-e 's:${RECIPE_SYSROOT_NATIVE}::g' \
+		-e 's:${RECIPE_SYSROOT}::g' \
+		-e 's:${BASE_WORKDIR}/${MULTIMACH_TARGET_SYS}::g' \
 		${PKGD}/${libdir}/python${PYTHON_MAJMIN}/config/Makefile \
 		${PKGD}/${libdir}/python${PYTHON_MAJMIN}/_sysconfigdata.py
-    python -m py_compile ${PKGD}/${libdir}/python${PYTHON_MAJMIN}/_sysconfigdata.py
+	(cd ${PKGD}; python -m py_compile ./${libdir}/python${PYTHON_MAJMIN}/_sysconfigdata.py)
 }
 
 
Signature · 1 line
2.8.1
author
#5 lore raw

PATCH v2 5/5 kbd-ptest: improve reproducibility

In reply to Hongxu Jia

Remove remaining build host references of ptest's Makefile.

---
 meta/recipes-core/kbd/kbd_2.0.4.bb | 7 +++++++
 1 file changed, 7 insertions(+)
meta/recipes-core/kbd/kbd_2.0.4.bb +7 -0
index 423b47a..5937cb8 100644
@@ -29,6 +29,13 @@ do_compile_ptest() {
 do_install_ptest() {
     install -D ${B}/tests/Makefile ${D}${PTEST_PATH}/tests/Makefile
     sed -i -e '/Makefile:/,/^$/d' -e '/%: %.in/,/^$/d' \
+	-e 's:--sysroot=${STAGING_DIR_TARGET}::g' \
+	-e 's:${DEBUG_PREFIX_MAP}::g' \
+	-e 's:${HOSTTOOLS_DIR}/::g' \
+	-e 's:${RECIPE_SYSROOT_NATIVE}::g' \
+	-e 's:${RECIPE_SYSROOT}::g' \
+	-e 's:${S}/config/missing::g' \
+	-e 's:${WORKDIR}::g' \
 	-e '/libkeymap_.*_SOURCES =/d' -e '/$(EXEEXT):/,/^$/d' ${D}${PTEST_PATH}/tests/Makefile
 
     find ${B}/tests -executable -exec install {} ${D}${PTEST_PATH}/tests \;
Signature · 1 line
2.8.1