rust: correctly link rust-snapshot into build/stage0
PATCH v2 1/5 rust: correctly link rust-snapshot into build/stage0
From: Alexander Kanavin <[email protected]>
This does not seem to be used in regular builds, but is beneficial in rust selftest, where it allows dropping a custom patch that is unsuitable for upstream (and was rejected by them).
Also remove an obsolete comment that seems related to the code but describes something that was resolved long time ago.
I have confirmed that the rust selftest continues to pass with just this one commit on top of master (as the following changes do break the selftest).
--- .../rust/files/cargo-path.patch | 37 ------------------- meta/recipes-devtools/rust/rust-source.inc | 1 - meta/recipes-devtools/rust/rust_1.75.0.bb | 6 +-- 3 files changed, 2 insertions(+), 42 deletions(-) delete mode 100644 meta/recipes-devtools/rust/files/cargo-path.patch
meta/recipes-devtools/rust/files/cargo-path.patch deleted +0 -37
@@ -1,37 +0,0 @@ -Fix the cargo binary path error and ensure that it is fetched -during rustc bootstrap in rust oe-selftest. - -====================================================================== -ERROR: test_cargoflags (bootstrap_test.BuildBootstrap) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/home/build-st/tmp/work/cortexa57-poky-linux/rust/1.74.1/rustc-1.74.1-src/src/bootstrap/bootstrap_test.py", line 157, in test_cargoflags - args, _ = self.build_args(env={"CARGOFLAGS": "--timings"}) - File "/home/build-st/tmp/work/cortexa57-poky-linux/rust/1.74.1/rustc-1.74.1-src/src/bootstrap/bootstrap_test.py", line 154, in build_args - return build.build_bootstrap_cmd(env), env - File "/home/build-st/tmp/work/cortexa57-poky-linux/rust/1.74.1/rustc-1.74.1-src/src/bootstrap/bootstrap.py", line 960, in build_bootstrap_cmd - raise Exception("no cargo executable found at `{}`".format( -Exception: no cargo executable found at `/home/build-st/tmp/work/cortexa57-poky-linux/rust/1.74.1/rustc-1.74.1-src/build/x86_64-unknown-linux-gnu/stage0/bin/cargo` - -Upstream-Status: Submitted [https://github.com/rust-lang/rust/pull/120125] - -Signed-off-by: Yash Shinde <[email protected]> ---- -diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py ---- a/src/bootstrap/bootstrap.py -+++ b/src/bootstrap/bootstrap.py -@@ -954,9 +954,11 @@ - if "RUSTFLAGS_BOOTSTRAP" in env: - env["RUSTFLAGS"] += " " + env["RUSTFLAGS_BOOTSTRAP"] - -- env["PATH"] = os.path.join(self.bin_root(), "bin") + \ -- os.pathsep + env["PATH"] -- if not os.path.isfile(self.cargo()): -+ cargo_bin_path = os.path.join(self.bin_root(), "bin", "cargo") -+ if not os.path.isfile(cargo_bin_path): -+ cargo_bin_path = os.getenv("RUST_TARGET_PATH") + "rust-snapshot/bin/cargo" -+ env["PATH"] = os.path.dirname(cargo_bin_path) + os.pathsep + env["PATH"] -+ else: - raise Exception("no cargo executable found at `{}`".format( - self.cargo())) - args = [self.cargo(), "build", "--manifest-path",
meta/recipes-devtools/rust/rust-source.inc +0 -1
@@ -7,7 +7,6 @@ SRC_URI += "https://static.rust-lang.org/dist/rustc-${RUST_VERSION}-src.tar.xz;n file://rv32-missing-syscalls.patch;patchdir=${RUSTSRC} \ file://rv32-rustix-libc-backend.patch;patchdir=${RUSTSRC} \ file://rv32-cargo-rustix-0.38.19-fix.patch;patchdir=${RUSTSRC} \ - file://cargo-path.patch;patchdir=${RUSTSRC} \ file://custom-target-cfg.patch;patchdir=${RUSTSRC} \ file://rustc-bootstrap.patch;patchdir=${RUSTSRC} \ file://target-build-value.patch;patchdir=${RUSTSRC} \
meta/recipes-devtools/rust/rust_1.75.0.bb +2 -4
@@ -35,8 +35,6 @@ RUST_ALTERNATE_EXE_PATH_NATIVE = "${STAGING_LIBDIR_NATIVE}/llvm-rust/bin/llvm-co # own vendoring. CARGO_DISABLE_BITBAKE_VENDORING = "1" -# We can't use RUST_BUILD_SYS here because that may be "musl" if -# TCLIBC="musl". Snapshots are always -unknown-linux-gnu setup_cargo_environment () { # The first step is to build bootstrap and some early stage tools, # these are build for the same target as the snapshot, e.g.
@@ -54,8 +52,8 @@ do_rust_setup_snapshot () { # Some versions of rust (e.g. 1.18.0) tries to find cargo in stage0/bin/cargo # and fail without it there. - mkdir -p ${RUSTSRC}/build/${BUILD_SYS} - ln -sf ${WORKDIR}/rust-snapshot/ ${RUSTSRC}/build/${BUILD_SYS}/stage0 + mkdir -p ${RUSTSRC}/build/${RUST_BUILD_SYS} + ln -sf ${WORKDIR}/rust-snapshot/ ${RUSTSRC}/build/${RUST_BUILD_SYS}/stage0 # Need to use uninative's loader if enabled/present since the library paths # are used internally by rust and result in symbol mismatches if we don't
Signature · 1 line
2.39.2
PATCH v2 2/5 rust: add reproducibility patch to eliminate host leakage
From: Alexander Kanavin <[email protected]>
[YOCTO #15185]
--- ...te-host-information-into-compilation.patch | 51 +++++++++++++++++++ meta/recipes-devtools/rust/rust-source.inc | 1 + 2 files changed, 52 insertions(+) create mode 100644 meta/recipes-devtools/rust/files/0001-cargo-do-not-write-host-information-into-compilation.patch
meta/recipes-devtools/rust/files/0001-cargo-do-not-write-host-information-into-compilation.patch new file +51 -0
@@ -0,0 +1,51 @@ +From 065d7c263091118437465d714d8a29dbb6296921 Mon Sep 17 00:00:00 2001 +From: Alexander Kanavin <[email protected]> +Date: Mon, 13 May 2024 14:57:54 +0200 +Subject: [PATCH] cargo: do not write host information into compilation unit + hashes + +This breaks reproducibility in cross-builds where the cross-target +can be the same, but build hosts are different, as seen with +"rustc --version -v": +... +host: x86_64-unknown-linux-gnu + +vs. + +host: aarch64-unknown-linux-gnu + +This can possibly be improved by only hashing host info if the build +is a native one (e.g. there's no --target option passed to cargo +invocation) but I'm not sure how. + +Upstream-Status: Inappropriate [reported at https://github.com/rust-lang/cargo/issues/13922] +Signed-off-by: Alexander Kanavin <[email protected]> +--- + .../src/cargo/core/compiler/context/compilation_files.rs | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/tools/cargo/src/cargo/core/compiler/context/compilation_files.rs b/src/tools/cargo/src/cargo/core/compiler/context/compilation_files.rs +index d83dbf10c..b2ad8d9f3 100644 +--- a/src/tools/cargo/src/cargo/core/compiler/context/compilation_files.rs ++++ b/src/tools/cargo/src/cargo/core/compiler/context/compilation_files.rs +@@ -652,7 +652,7 @@ fn hash_rustc_version(bcx: &BuildContext<'_, '_>, hasher: &mut StableHasher) { + if vers.pre.is_empty() || bcx.config.cli_unstable().separate_nightlies { + // For stable, keep the artifacts separate. This helps if someone is + // testing multiple versions, to avoid recompiles. +- bcx.rustc().verbose_version.hash(hasher); ++ //bcx.rustc().verbose_version.hash(hasher); + return; + } + // On "nightly"/"beta"/"dev"/etc, keep each "channel" separate. Don't hash +@@ -665,7 +665,7 @@ fn hash_rustc_version(bcx: &BuildContext<'_, '_>, hasher: &mut StableHasher) { + // Keep "host" since some people switch hosts to implicitly change + // targets, (like gnu vs musl or gnu vs msvc). In the future, we may want + // to consider hashing `unit.kind.short_name()` instead. +- bcx.rustc().host.hash(hasher); ++ //bcx.rustc().host.hash(hasher); + // None of the other lines are important. Currently they are: + // binary: rustc <-- or "rustdoc" + // commit-hash: 38114ff16e7856f98b2b4be7ab4cd29b38bed59a +-- +2.39.2 +
meta/recipes-devtools/rust/rust-source.inc +1 -0
@@ -12,6 +12,7 @@ SRC_URI += "https://static.rust-lang.org/dist/rustc-${RUST_VERSION}-src.tar.xz;n file://target-build-value.patch;patchdir=${RUSTSRC} \ file://0001-Handle-vendored-sources-when-remapping-paths.patch;patchdir=${RUSTSRC} \ file://repro-issue-fix-with-v175.patch;patchdir=${RUSTSRC} \ + file://0001-cargo-do-not-write-host-information-into-compilation.patch;patchdir=${RUSTSRC} \ " SRC_URI[rust.sha256sum] = "4526f786d673e4859ff2afa0bab2ba13c918b796519a25c1acce06dba9542340"
Signature · 1 line
2.39.2
PATCH v2 3/5 rust: use rust-snapshot binaries only in rust-native
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
Otherwise, use rust-native and cargo-native binaries as that allows our native tweaks in them to be used for target/nativesdk rust - same as for everything else written in rust.
In particular, this allows building target rust with cargo-native that includes important reproducibility tweaks.
Unfortunately, this also breaks rust selftest, and that is partially addressed by the following commit.
[YOCTO #15185]
--- meta/recipes-devtools/rust/rust_1.75.0.bb | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-)
meta/recipes-devtools/rust/rust_1.75.0.bb +12 -2
@@ -11,6 +11,11 @@ DEPENDS += "file-native python3-native" DEPENDS:append:class-native = " rust-llvm-native" DEPENDS:append:class-nativesdk = " nativesdk-rust-llvm" +# native rust uses cargo/rustc from binary snapshots to bootstrap +# but everything else should use our native builds +DEPENDS:append:class-target = " cargo-native rust-native" +DEPENDS:append:class-nativesdk = " cargo-native rust-native" + DEPENDS += "rust-llvm (=${PV})" RDEPENDS:${PN}:append:class-target = " gcc g++ binutils"
@@ -68,6 +73,11 @@ addtask do_test_compile after do_configure do_rust_gen_targets do_rust_setup_snapshot[dirs] += "${WORKDIR}/rust-snapshot" do_rust_setup_snapshot[vardepsexclude] += "UNINATIVE_LOADER" +RUSTC_BOOTSTRAP = "${STAGING_BINDIR_NATIVE}/rustc" +CARGO_BOOTSTRAP = "${STAGING_BINDIR_NATIVE}/cargo" +RUSTC_BOOTSTRAP:class-native = "${WORKDIR}/rust-snapshot/bin/rustc" +CARGO_BOOTSTRAP:class-native = "${WORKDIR}/rust-snapshot/bin/cargo" + python do_configure() { import json import configparser
@@ -141,10 +151,10 @@ python do_configure() { config.set("build", "docs", e(False)) config.set("build", "tools", ["rust-demangler",]) - rustc = d.expand("${WORKDIR}/rust-snapshot/bin/rustc") + rustc = d.getVar('RUSTC_BOOTSTRAP') config.set("build", "rustc", e(rustc)) - cargo = d.expand("${WORKDIR}/rust-snapshot/bin/cargo") + cargo = d.getVar('CARGO_BOOTSTRAP') config.set("build", "cargo", e(cargo)) config.set("build", "vendor", e(True))
Signature · 1 line
2.39.2
PATCH v2 4/5 rust: build the default set of tools
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
Setting it explicitly replaces rust's default choice which is rustdoc (needed for example in selftests and otherwise expected to be present in typical rust installations):
https://github.com/rust-lang/rust/blob/master/config.example.toml#L320
This addresses some of the rust selftest failures but not all. Help is appreciate to restore the selftest.
Unfortunately, this also breaks rust reproducibility (or rather exposes that it was never properly fixed, as explained here:
https://lists.openembedded.org/g/openembedded-core/message/199288
)
--- meta/lib/oeqa/selftest/cases/reproducible.py | 2 ++ meta/recipes-devtools/rust/rust_1.75.0.bb | 1 - 2 files changed, 2 insertions(+), 1 deletion(-)
meta/lib/oeqa/selftest/cases/reproducible.py +2 -0
@@ -16,6 +16,8 @@ import os import datetime exclude_packages = [ + 'rust-rustdoc', + 'rust-dbg' ] def is_excluded(package):
meta/recipes-devtools/rust/rust_1.75.0.bb +0 -1
@@ -149,7 +149,6 @@ python do_configure() { config.add_section("build") config.set("build", "submodules", e(False)) config.set("build", "docs", e(False)) - config.set("build", "tools", ["rust-demangler",]) rustc = d.getVar('RUSTC_BOOTSTRAP') config.set("build", "rustc", e(rustc))
Signature · 1 line
2.39.2
PATCH v2 5/5 selftest/rust: correctly form the PATH environment variable
In reply to Alexander KanavinFrom: Alexander Kanavin <[email protected]>
There were two issues here:
- the path to native python was not included, so the test suite was running with host python, causing failures due to host python being given native libraries:
https://autobuilder.yoctoproject.org/typhoon/#/builders/148/builds/1711/steps/12/logs/stdio
(rust recipe uses native python throughout)
- tmp/hosttools was listead ahead of native-sysroot/usr/bin
--- meta/lib/oeqa/selftest/cases/rust.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
meta/lib/oeqa/selftest/cases/rust.py +2 -3
@@ -210,9 +210,8 @@ class RustSelfTestSystemEmulated(OESelftestTestCase, OEPTestResultTestCase): tmpdir = get_bb_var("TMPDIR", "rust") # Set path for target-poky-linux-gcc, RUST_TARGET_PATH and hosttools. - cmd = " export PATH=%s/recipe-sysroot-native/usr/bin:$PATH;" % rustlibpath - cmd = cmd + " export TARGET_VENDOR=\"-poky\";" - cmd = cmd + " export PATH=%s/recipe-sysroot-native/usr/bin/%s:%s/hosttools:$PATH;" % (rustlibpath, tcpath, tmpdir) + cmd = "export TARGET_VENDOR=\"-poky\";" + cmd = cmd + " export PATH=%s/recipe-sysroot-native/usr/bin/python3-native:%s/recipe-sysroot-native/usr/bin:%s/recipe-sysroot-native/usr/bin/%s:%s/hosttools:$PATH;" % (rustlibpath, rustlibpath, rustlibpath, tcpath, tmpdir) cmd = cmd + " export RUST_TARGET_PATH=%s/rust-targets;" % rustlibpath # Trigger testing. cmd = cmd + " export TEST_DEVICE_ADDR=\"%s:12345\";" % qemu.ip
Signature · 1 line
2.39.2