diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index 6ce4c6d62fa55..2965174b45bcc 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -105,7 +105,7 @@ build/ debuginfo/ ... - # Bootstrap host tools (which are always compiled with the stage0 compiler) + # Host tools (which are always compiled with the stage0 compiler) # are stored here. bootstrap-tools/ diff --git a/src/bootstrap/src/core/build_steps/check.rs b/src/bootstrap/src/core/build_steps/check.rs index 0497bae86a1c8..d4d3b7a7babe9 100644 --- a/src/bootstrap/src/core/build_steps/check.rs +++ b/src/bootstrap/src/core/build_steps/check.rs @@ -4,7 +4,10 @@ use crate::core::build_steps::compile::{ add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, std_crates_for_run_make, }; use crate::core::build_steps::tool; -use crate::core::build_steps::tool::{COMPILETEST_ALLOW_FEATURES, SourceType, prepare_tool_cargo}; +use crate::core::build_steps::tool::{ + COMPILETEST_ALLOW_FEATURES, SourceType, ToolTargetBuildMode, get_tool_target_compiler, + prepare_tool_cargo, +}; use crate::core::builder::{ self, Alias, Builder, Kind, RunConfig, ShouldRun, Step, StepMetadata, crate_description, }; @@ -247,8 +250,10 @@ fn prepare_compiler_for_check( mode: Mode, ) -> Compiler { let host = builder.host_target; + match mode { Mode::ToolBootstrap => builder.compiler(0, host), + Mode::ToolTarget => get_tool_target_compiler(builder, ToolTargetBuildMode::Build(target)), Mode::ToolStd => { // These tools require the local standard library to be checked let build_compiler = builder.compiler(builder.top_stage, host); diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 0587d21ecc28d..d1285340cd6ad 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -19,7 +19,7 @@ use serde_derive::Deserialize; use tracing::{instrument, span}; use crate::core::build_steps::gcc::{Gcc, add_cg_gcc_cargo_flags}; -use crate::core::build_steps::tool::SourceType; +use crate::core::build_steps::tool::{SourceType, copy_lld_artifacts}; use crate::core::build_steps::{dist, llvm}; use crate::core::builder; use crate::core::builder::{ @@ -2052,19 +2052,20 @@ impl Step for Assemble { } } - let maybe_install_llvm_bitcode_linker = |compiler| { + let maybe_install_llvm_bitcode_linker = || { if builder.config.llvm_bitcode_linker_enabled { trace!("llvm-bitcode-linker enabled, installing"); - let llvm_bitcode_linker = - builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker { - build_compiler: compiler, - target: target_compiler.host, - }); + let llvm_bitcode_linker = builder.ensure( + crate::core::build_steps::tool::LlvmBitcodeLinker::for_use_by_compiler( + builder, + target_compiler, + ), + ); // Copy the llvm-bitcode-linker to the self-contained binary directory let bindir_self_contained = builder - .sysroot(compiler) - .join(format!("lib/rustlib/{}/bin/self-contained", compiler.host)); + .sysroot(target_compiler) + .join(format!("lib/rustlib/{}/bin/self-contained", target_compiler.host)); let tool_exe = exe("llvm-bitcode-linker", target_compiler.host); t!(fs::create_dir_all(&bindir_self_contained)); @@ -2091,9 +2092,9 @@ impl Step for Assemble { builder.info(&format!("Creating a sysroot for stage{stage} compiler (use `rustup toolchain link 'name' build/host/stage{stage}`)", stage = target_compiler.stage)); } - let mut precompiled_compiler = target_compiler; - precompiled_compiler.forced_compiler(true); - maybe_install_llvm_bitcode_linker(precompiled_compiler); + // FIXME: this is incomplete, we do not copy a bunch of other stuff to the downloaded + // sysroot... + maybe_install_llvm_bitcode_linker(); return target_compiler; } @@ -2258,10 +2259,12 @@ impl Step for Assemble { copy_codegen_backends_to_sysroot(builder, build_compiler, target_compiler); if builder.config.lld_enabled { - builder.ensure(crate::core::build_steps::tool::LldWrapper { - build_compiler, - target_compiler, - }); + let lld_wrapper = + builder.ensure(crate::core::build_steps::tool::LldWrapper::for_use_by_compiler( + builder, + target_compiler, + )); + copy_lld_artifacts(builder, lld_wrapper, target_compiler); } if builder.config.llvm_enabled(target_compiler.host) && builder.config.llvm_tools_enabled { @@ -2286,15 +2289,14 @@ impl Step for Assemble { } // In addition to `rust-lld` also install `wasm-component-ld` when - // LLD is enabled. This is a relatively small binary that primarily - // delegates to the `rust-lld` binary for linking and then runs - // logic to create the final binary. This is used by the - // `wasm32-wasip2` target of Rust. + // is enabled. This is used by the `wasm32-wasip2` target of Rust. if builder.tool_enabled("wasm-component-ld") { - let wasm_component = builder.ensure(crate::core::build_steps::tool::WasmComponentLd { - compiler: build_compiler, - target: target_compiler.host, - }); + let wasm_component = builder.ensure( + crate::core::build_steps::tool::WasmComponentLd::for_use_by_compiler( + builder, + target_compiler, + ), + ); builder.copy_link( &wasm_component.tool_path, &libdir_bin.join(wasm_component.tool_path.file_name().unwrap()), @@ -2302,7 +2304,7 @@ impl Step for Assemble { ); } - maybe_install_llvm_bitcode_linker(target_compiler); + maybe_install_llvm_bitcode_linker(); // Ensure that `libLLVM.so` ends up in the newly build compiler directory, // so that it can be found when the newly built `rustc` is run. diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 8b2d65ace50aa..af17a69302b2c 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1575,7 +1575,10 @@ impl Step for Extended { compiler: builder.compiler(stage, target), backend: "cranelift".to_string(), }); - add_component!("llvm-bitcode-linker" => LlvmBitcodeLinker {compiler, target}); + add_component!("llvm-bitcode-linker" => LlvmBitcodeLinker { + target_compiler: compiler, + target + }); let etc = builder.src.join("src/etc/installer"); @@ -2343,7 +2346,7 @@ impl Step for LlvmTools { #[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] pub struct LlvmBitcodeLinker { - pub compiler: Compiler, + pub target_compiler: Compiler, pub target: TargetSelection, } @@ -2359,23 +2362,16 @@ impl Step for LlvmBitcodeLinker { fn make_run(run: RunConfig<'_>) { run.builder.ensure(LlvmBitcodeLinker { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.host_target, - run.target, - ), + target_compiler: run.builder.compiler(run.builder.top_stage, run.target), target: run.target, }); } fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; let target = self.target; - builder.ensure(compile::Rustc::new(compiler, target)); - - let llbc_linker = - builder.ensure(tool::LlvmBitcodeLinker { build_compiler: compiler, target }); + let llbc_linker = builder + .ensure(tool::LlvmBitcodeLinker::for_use_by_compiler(builder, self.target_compiler)); let self_contained_bin_dir = format!("lib/rustlib/{}/bin/self-contained", target.triple); diff --git a/src/bootstrap/src/core/build_steps/install.rs b/src/bootstrap/src/core/build_steps/install.rs index 4434d6658eb24..285624916d18d 100644 --- a/src/bootstrap/src/core/build_steps/install.rs +++ b/src/bootstrap/src/core/build_steps/install.rs @@ -287,7 +287,7 @@ install!((self, builder, _config), } }; LlvmBitcodeLinker, alias = "llvm-bitcode-linker", Self::should_build(_config), only_hosts: true, { - if let Some(tarball) = builder.ensure(dist::LlvmBitcodeLinker { compiler: self.compiler, target: self.target }) { + if let Some(tarball) = builder.ensure(dist::LlvmBitcodeLinker { target_compiler: self.compiler, target: self.target }) { install_sh(builder, "llvm-bitcode-linker", self.compiler.stage, Some(self.target), &tarball); } else { builder.info( diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 716bef3f38c49..4acc609626e80 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -2941,7 +2941,8 @@ impl Step for RemoteCopyLibs { builder.info(&format!("REMOTE copy libs to emulator ({target})")); - let remote_test_server = builder.ensure(tool::RemoteTestServer { compiler, target }); + let remote_test_server = + builder.ensure(tool::RemoteTestServer { build_compiler: compiler, target }); // Spawn the emulator and wait for it to come online let tool = builder.tool_exe(Tool::RemoteTestClient); diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 5de1b472d7940..df89eb1da20c2 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -41,7 +41,8 @@ pub enum ToolArtifactKind { #[derive(Debug, Clone, Hash, PartialEq, Eq)] struct ToolBuild { - compiler: Compiler, + /// Compiler that will build this tool. + build_compiler: Compiler, target: TargetSelection, tool: &'static str, path: &'static str, @@ -111,28 +112,28 @@ impl Step for ToolBuild { let mut tool = self.tool; let path = self.path; - let target_compiler = self.compiler; - self.compiler = if self.mode == Mode::ToolRustc { - get_tool_rustc_compiler(builder, self.compiler) + let target_compiler = self.build_compiler; + self.build_compiler = if self.mode == Mode::ToolRustc { + get_tool_rustc_compiler(builder, self.build_compiler) } else { - self.compiler + self.build_compiler }; match self.mode { Mode::ToolRustc => { // If compiler was forced, its artifacts should have been prepared earlier. - if !self.compiler.is_forced_compiler() { - builder.std(self.compiler, self.compiler.host); - builder.ensure(compile::Rustc::new(self.compiler, target)); + if !self.build_compiler.is_forced_compiler() { + builder.std(self.build_compiler, self.build_compiler.host); + builder.ensure(compile::Rustc::new(self.build_compiler, target)); } } Mode::ToolStd => { // If compiler was forced, its artifacts should have been prepared earlier. - if !self.compiler.is_forced_compiler() { - builder.std(self.compiler, target) + if !self.build_compiler.is_forced_compiler() { + builder.std(self.build_compiler, target); } } - Mode::ToolBootstrap => {} // uses downloaded stage0 compiler libs + Mode::ToolBootstrap | Mode::ToolTarget => {} // uses downloaded stage0 compiler libs _ => panic!("unexpected Mode for tool build"), } @@ -151,7 +152,7 @@ impl Step for ToolBuild { let mut cargo = prepare_tool_cargo( builder, - self.compiler, + self.build_compiler, self.mode, target, Kind::Build, @@ -173,7 +174,7 @@ impl Step for ToolBuild { // Rustc tools (miri, clippy, cargo, rustfmt, rust-analyzer) // could use the additional optimizations. - if self.mode == Mode::ToolRustc && is_lto_stage(&self.compiler) { + if self.mode == Mode::ToolRustc && is_lto_stage(&self.build_compiler) { let lto = match builder.config.rust_lto { RustcLto::Off => Some("off"), RustcLto::Thin => Some("thin"), @@ -195,8 +196,8 @@ impl Step for ToolBuild { Kind::Build, self.mode, self.tool, - self.compiler.stage, - &self.compiler.host, + self.build_compiler.stage + 1, + &self.build_compiler.host, &self.target, ); @@ -219,14 +220,14 @@ impl Step for ToolBuild { } let tool_path = match self.artifact_kind { ToolArtifactKind::Binary => { - copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool) + copy_link_tool_bin(builder, self.build_compiler, self.target, self.mode, tool) } ToolArtifactKind::Library => builder - .cargo_out(self.compiler, self.mode, self.target) + .cargo_out(self.build_compiler, self.mode, self.target) .join(format!("lib{tool}.rlib")), }; - ToolBuildResult { tool_path, build_compiler: self.compiler, target_compiler } + ToolBuildResult { tool_path, build_compiler: self.build_compiler, target_compiler } } } } @@ -364,6 +365,47 @@ pub(crate) fn get_tool_rustc_compiler( builder.compiler(target_compiler.stage.saturating_sub(1), builder.config.host_target) } +/// Determines how to build a `ToolTarget`, i.e. which compiler should be used to compile it. +/// The compiler stage is automatically auto-bumped if we need to cross-compile a stage 1 tool. +pub enum ToolTargetBuildMode { + /// Build the tool using rustc that corresponds to the selected CLI stage. + Build(TargetSelection), + /// Build the tool so that it can be attached to the sysroot of the passed compiler. + /// Since we always dist stage 2+, the compiler that builds the tool in this case has to be + /// stage 1+. + Dist(Compiler), +} + +/// Returns compiler that is able to compile a `ToolTarget` tool with the given `mode`. +pub(crate) fn get_tool_target_compiler( + builder: &Builder<'_>, + mode: ToolTargetBuildMode, +) -> Compiler { + let (target, build_compiler_stage) = match mode { + ToolTargetBuildMode::Build(target) => { + assert!(builder.top_stage > 0); + // If we want to build a stage N tool, we need to compile it with stage N-1 rustc + (target, builder.top_stage - 1) + } + ToolTargetBuildMode::Dist(target_compiler) => { + assert!(target_compiler.stage > 0); + // If we want to dist a stage N rustc, we want to attach stage N tool to it. + // And to build that tool, we need to compile it with stage N-1 rustc + (target_compiler.host, target_compiler.stage - 1) + } + }; + + let compiler = if builder.host_target == target { + builder.compiler(build_compiler_stage, builder.host_target) + } else { + // If we are cross-compiling a stage 1 tool, we cannot do that with a stage 0 compiler, + // so we auto-bump the tool's stage to 2. + builder.compiler(build_compiler_stage.max(1), builder.host_target) + }; + builder.std(compiler, target); + compiler +} + /// Links a built tool binary with the given `name` from the build directory to the /// tools directory. fn copy_link_tool_bin( @@ -450,7 +492,7 @@ macro_rules! bootstrap_tool { let compiletest_wants_stage0 = $tool_name == "compiletest" && builder.config.compiletest_use_stage0_libtest; builder.ensure(ToolBuild { - compiler: self.compiler, + build_compiler: self.compiler, target: self.target, tool: $tool_name, mode: if is_unstable && !compiletest_wants_stage0 { @@ -521,7 +563,6 @@ bootstrap_tool!( // rustdoc-gui-test has a crate dependency on compiletest, so it needs the same unstable features. RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = COMPILETEST_ALLOW_FEATURES; CoverageDump, "src/tools/coverage-dump", "coverage-dump"; - WasmComponentLd, "src/tools/wasm-component-ld", "wasm-component-ld", is_unstable_tool = true, allow_features = "min_specialization"; UnicodeTableGenerator, "src/tools/unicode-table-generator", "unicode-table-generator"; FeaturesStatusDump, "src/tools/features-status-dump", "features-status-dump"; OptimizedDist, "src/tools/opt-dist", "opt-dist", submodules = &["src/tools/rustc-perf"]; @@ -560,7 +601,7 @@ impl Step for RustcPerf { builder.require_submodule("src/tools/rustc-perf", None); let tool = ToolBuild { - compiler: self.compiler, + build_compiler: self.compiler, target: self.target, tool: "collector", mode: Mode::ToolBootstrap, @@ -576,7 +617,7 @@ impl Step for RustcPerf { let res = builder.ensure(tool.clone()); // We also need to symlink the `rustc-fake` binary to the corresponding directory, // because `collector` expects it in the same directory. - copy_link_tool_bin(builder, tool.compiler, tool.target, tool.mode, "rustc-fake"); + copy_link_tool_bin(builder, tool.build_compiler, tool.target, tool.mode, "rustc-fake"); res } @@ -620,7 +661,7 @@ impl Step for ErrorIndex { fn run(self, builder: &Builder<'_>) -> ToolBuildResult { builder.ensure(ToolBuild { - compiler: self.compiler, + build_compiler: self.compiler, target: self.compiler.host, tool: "error_index_generator", mode: Mode::ToolRustc, @@ -636,7 +677,7 @@ impl Step for ErrorIndex { #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct RemoteTestServer { - pub compiler: Compiler, + pub build_compiler: Compiler, pub target: TargetSelection, } @@ -649,17 +690,20 @@ impl Step for RemoteTestServer { fn make_run(run: RunConfig<'_>) { run.builder.ensure(RemoteTestServer { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.host_target), + build_compiler: get_tool_target_compiler( + run.builder, + ToolTargetBuildMode::Build(run.target), + ), target: run.target, }); } fn run(self, builder: &Builder<'_>) -> ToolBuildResult { builder.ensure(ToolBuild { - compiler: self.compiler, + build_compiler: self.build_compiler, target: self.target, tool: "remote-test-server", - mode: Mode::ToolStd, + mode: Mode::ToolTarget, path: "src/tools/remote-test-server", source_type: SourceType::InTree, extra_features: Vec::new(), @@ -668,6 +712,10 @@ impl Step for RemoteTestServer { artifact_kind: ToolArtifactKind::Binary, }) } + + fn metadata(&self) -> Option { + Some(StepMetadata::build("remote-test-server", self.target).built_by(self.build_compiler)) + } } #[derive(Debug, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] @@ -757,7 +805,7 @@ impl Step for Rustdoc { let ToolBuildResult { tool_path, build_compiler, target_compiler } = builder.ensure(ToolBuild { - compiler: target_compiler, + build_compiler: target_compiler, target, // Cargo adds a number of paths to the dylib search path on windows, which results in // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool" @@ -825,7 +873,7 @@ impl Step for Cargo { builder.build.require_submodule("src/tools/cargo", None); builder.ensure(ToolBuild { - compiler: self.compiler, + build_compiler: self.compiler, target: self.target, tool: "cargo", mode: Mode::ToolRustc, @@ -839,17 +887,50 @@ impl Step for Cargo { } } +/// Represents a built LldWrapper, the `lld-wrapper` tool itself, and a directory +/// containing a build of LLD. +#[derive(Clone)] +pub struct BuiltLldWrapper { + tool: ToolBuildResult, + lld_dir: PathBuf, +} + #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct LldWrapper { pub build_compiler: Compiler, - pub target_compiler: Compiler, + pub target: TargetSelection, +} + +impl LldWrapper { + /// Returns `LldWrapper` that should be **used** by the passed compiler. + pub fn for_use_by_compiler(builder: &Builder<'_>, target_compiler: Compiler) -> Self { + Self { + build_compiler: get_tool_target_compiler( + builder, + ToolTargetBuildMode::Dist(target_compiler), + ), + target: target_compiler.host, + } + } } impl Step for LldWrapper { - type Output = ToolBuildResult; + type Output = BuiltLldWrapper; + + const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() + run.path("src/tools/lld-wrapper") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(LldWrapper { + build_compiler: get_tool_target_compiler( + run.builder, + ToolTargetBuildMode::Build(run.target), + ), + target: run.target, + }); } #[cfg_attr( @@ -858,25 +939,16 @@ impl Step for LldWrapper { level = "debug", name = "LldWrapper::run", skip_all, - fields(build_compiler = ?self.build_compiler, target_compiler = ?self.target_compiler), + fields(build_compiler = ?self.build_compiler), ), )] - fn run(self, builder: &Builder<'_>) -> ToolBuildResult { - if builder.config.dry_run() { - return ToolBuildResult { - tool_path: Default::default(), - build_compiler: self.build_compiler, - target_compiler: self.target_compiler, - }; - } - - let target = self.target_compiler.host; - - let tool_result = builder.ensure(ToolBuild { - compiler: self.build_compiler, - target, + fn run(self, builder: &Builder<'_>) -> Self::Output { + let lld_dir = builder.ensure(llvm::Lld { target: self.target }); + let tool = builder.ensure(ToolBuild { + build_compiler: self.build_compiler, + target: self.target, tool: "lld-wrapper", - mode: Mode::ToolStd, + mode: Mode::ToolTarget, path: "src/tools/lld-wrapper", source_type: SourceType::InTree, extra_features: Vec::new(), @@ -884,38 +956,110 @@ impl Step for LldWrapper { cargo_args: Vec::new(), artifact_kind: ToolArtifactKind::Binary, }); + BuiltLldWrapper { tool, lld_dir } + } - let libdir_bin = builder.sysroot_target_bindir(self.target_compiler, target); - t!(fs::create_dir_all(&libdir_bin)); + fn metadata(&self) -> Option { + Some(StepMetadata::build("LldWrapper", self.target).built_by(self.build_compiler)) + } +} - let lld_install = builder.ensure(llvm::Lld { target }); - let src_exe = exe("lld", target); - let dst_exe = exe("rust-lld", target); +pub(crate) fn copy_lld_artifacts( + builder: &Builder<'_>, + lld_wrapper: BuiltLldWrapper, + target_compiler: Compiler, +) { + let target = target_compiler.host; + + let libdir_bin = builder.sysroot_target_bindir(target_compiler, target); + t!(fs::create_dir_all(&libdir_bin)); + + let src_exe = exe("lld", target); + let dst_exe = exe("rust-lld", target); + + builder.copy_link( + &lld_wrapper.lld_dir.join("bin").join(src_exe), + &libdir_bin.join(dst_exe), + FileType::Executable, + ); + let self_contained_lld_dir = libdir_bin.join("gcc-ld"); + t!(fs::create_dir_all(&self_contained_lld_dir)); + for name in crate::LLD_FILE_NAMES { builder.copy_link( - &lld_install.join("bin").join(src_exe), - &libdir_bin.join(dst_exe), + &lld_wrapper.tool.tool_path, + &self_contained_lld_dir.join(exe(name, target)), FileType::Executable, ); - let self_contained_lld_dir = libdir_bin.join("gcc-ld"); - t!(fs::create_dir_all(&self_contained_lld_dir)); - - for name in crate::LLD_FILE_NAMES { - builder.copy_link( - &tool_result.tool_path, - &self_contained_lld_dir.join(exe(name, target)), - FileType::Executable, - ); + } +} + +/// Builds the `wasm-component-ld` linker wrapper, which is shipped with rustc to be executed on the +/// host platform where rustc runs. +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct WasmComponentLd { + build_compiler: Compiler, + target: TargetSelection, +} + +impl WasmComponentLd { + /// Returns `WasmComponentLd` that should be **used** by the passed compiler. + pub fn for_use_by_compiler(builder: &Builder<'_>, target_compiler: Compiler) -> Self { + Self { + build_compiler: get_tool_target_compiler( + builder, + ToolTargetBuildMode::Dist(target_compiler), + ), + target: target_compiler.host, } + } +} - tool_result +impl Step for WasmComponentLd { + type Output = ToolBuildResult; + + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/wasm-component-ld") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(WasmComponentLd { + build_compiler: get_tool_target_compiler( + run.builder, + ToolTargetBuildMode::Build(run.target), + ), + target: run.target, + }); + } + + #[cfg_attr( + feature = "tracing", + instrument( + level = "debug", + name = "WasmComponentLd::run", + skip_all, + fields(build_compiler = ?self.build_compiler), + ), + )] + fn run(self, builder: &Builder<'_>) -> ToolBuildResult { + builder.ensure(ToolBuild { + build_compiler: self.build_compiler, + target: self.target, + tool: "wasm-component-ld", + mode: Mode::ToolTarget, + path: "src/tools/wasm-component-ld", + source_type: SourceType::InTree, + extra_features: vec![], + allow_features: "", + cargo_args: vec![], + artifact_kind: ToolArtifactKind::Binary, + }) } fn metadata(&self) -> Option { - Some( - StepMetadata::build("LldWrapper", self.target_compiler.host) - .built_by(self.build_compiler), - ) + Some(StepMetadata::build("WasmComponentLd", self.target).built_by(self.build_compiler)) } } @@ -948,7 +1092,7 @@ impl Step for RustAnalyzer { fn run(self, builder: &Builder<'_>) -> ToolBuildResult { builder.ensure(ToolBuild { - compiler: self.compiler, + build_compiler: self.compiler, target: self.target, tool: "rust-analyzer", mode: Mode::ToolRustc, @@ -993,7 +1137,7 @@ impl Step for RustAnalyzerProcMacroSrv { fn run(self, builder: &Builder<'_>) -> Option { let tool_result = builder.ensure(ToolBuild { - compiler: self.compiler, + build_compiler: self.compiler, target: self.target, tool: "rust-analyzer-proc-macro-srv", mode: Mode::ToolRustc, @@ -1021,8 +1165,21 @@ impl Step for RustAnalyzerProcMacroSrv { #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct LlvmBitcodeLinker { - pub build_compiler: Compiler, - pub target: TargetSelection, + build_compiler: Compiler, + target: TargetSelection, +} + +impl LlvmBitcodeLinker { + /// Returns `LlvmBitcodeLinker` that should be **used** by the passed compiler. + pub fn for_use_by_compiler(builder: &Builder<'_>, target_compiler: Compiler) -> Self { + Self { + build_compiler: get_tool_target_compiler( + builder, + ToolTargetBuildMode::Dist(target_compiler), + ), + target: target_compiler.host, + } + } } impl Step for LlvmBitcodeLinker { @@ -1038,9 +1195,10 @@ impl Step for LlvmBitcodeLinker { fn make_run(run: RunConfig<'_>) { run.builder.ensure(LlvmBitcodeLinker { - build_compiler: run - .builder - .compiler(run.builder.top_stage, run.builder.config.host_target), + build_compiler: get_tool_target_compiler( + run.builder, + ToolTargetBuildMode::Build(run.target), + ), target: run.target, }); } @@ -1051,10 +1209,10 @@ impl Step for LlvmBitcodeLinker { )] fn run(self, builder: &Builder<'_>) -> ToolBuildResult { builder.ensure(ToolBuild { - compiler: self.build_compiler, + build_compiler: self.build_compiler, target: self.target, tool: "llvm-bitcode-linker", - mode: Mode::ToolRustc, + mode: Mode::ToolTarget, path: "src/tools/llvm-bitcode-linker", source_type: SourceType::InTree, extra_features: vec![], @@ -1235,7 +1393,7 @@ fn run_tool_build_step( let ToolBuildResult { tool_path, build_compiler, target_compiler } = builder.ensure(ToolBuild { - compiler, + build_compiler: compiler, target, tool: tool_name, mode: Mode::ToolRustc, @@ -1332,7 +1490,7 @@ impl Step for TestFloatParse { let compiler = builder.compiler(builder.top_stage, bootstrap_host); builder.ensure(ToolBuild { - compiler, + build_compiler: compiler, target: bootstrap_host, tool: "test-float-parse", mode: Mode::ToolStd, diff --git a/src/bootstrap/src/core/builder/cargo.rs b/src/bootstrap/src/core/builder/cargo.rs index 065d7e45e0f04..c6f3c876a6f8e 100644 --- a/src/bootstrap/src/core/builder/cargo.rs +++ b/src/bootstrap/src/core/builder/cargo.rs @@ -537,7 +537,7 @@ impl Builder<'_> { } } - let stage = if compiler.stage == 0 && self.local_rebuild { + let build_compiler_stage = if compiler.stage == 0 && self.local_rebuild { // Assume the local-rebuild rustc already has stage1 features. 1 } else { @@ -545,15 +545,17 @@ impl Builder<'_> { }; // We synthetically interpret a stage0 compiler used to build tools as a - // "raw" compiler in that it's the exact snapshot we download. Normally - // the stage0 build means it uses libraries build by the stage0 - // compiler, but for tools we just use the precompiled libraries that - // we've downloaded - let use_snapshot = mode == Mode::ToolBootstrap; - assert!(!use_snapshot || stage == 0 || self.local_rebuild); - - let maybe_sysroot = self.sysroot(compiler); - let sysroot = if use_snapshot { self.rustc_snapshot_sysroot() } else { &maybe_sysroot }; + // "raw" compiler in that it's the exact snapshot we download. For things like + // ToolRustc, we would have to use the artificial stage0-sysroot compiler instead. + let use_snapshot = + mode == Mode::ToolBootstrap || (mode == Mode::ToolTarget && build_compiler_stage == 0); + assert!(!use_snapshot || build_compiler_stage == 0 || self.local_rebuild); + + let sysroot = if use_snapshot { + self.rustc_snapshot_sysroot().to_path_buf() + } else { + self.sysroot(compiler) + }; let libdir = self.rustc_libdir(compiler); let sysroot_str = sysroot.as_os_str().to_str().expect("sysroot should be UTF-8"); @@ -562,7 +564,7 @@ impl Builder<'_> { } let mut rustflags = Rustflags::new(target); - if stage != 0 { + if build_compiler_stage != 0 { if let Ok(s) = env::var("CARGOFLAGS_NOT_BOOTSTRAP") { cargo.args(s.split_whitespace()); } @@ -604,7 +606,7 @@ impl Builder<'_> { // sysroot. Passing this cfg enables raw-dylib support instead, which makes the native // library unnecessary. This can be removed when windows-rs enables raw-dylib // unconditionally. - if let Mode::Rustc | Mode::ToolRustc | Mode::ToolBootstrap = mode { + if let Mode::Rustc | Mode::ToolRustc | Mode::ToolBootstrap | Mode::ToolTarget = mode { rustflags.arg("--cfg=windows_raw_dylib"); } @@ -657,7 +659,7 @@ impl Builder<'_> { // FIXME(rust-lang/cargo#5754) we shouldn't be using special command arguments // to the host invocation here, but rather Cargo should know what flags to pass rustc // itself. - if stage == 0 { + if build_compiler_stage == 0 { hostflags.arg("--cfg=bootstrap"); } @@ -666,7 +668,7 @@ impl Builder<'_> { // #71458. let mut rustdocflags = rustflags.clone(); rustdocflags.propagate_cargo_env("RUSTDOCFLAGS"); - if stage == 0 { + if build_compiler_stage == 0 { rustdocflags.env("RUSTDOCFLAGS_BOOTSTRAP"); } else { rustdocflags.env("RUSTDOCFLAGS_NOT_BOOTSTRAP"); @@ -677,7 +679,7 @@ impl Builder<'_> { } match mode { - Mode::Std | Mode::ToolBootstrap | Mode::ToolStd => {} + Mode::Std | Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolTarget => {} Mode::Rustc | Mode::Codegen | Mode::ToolRustc => { // Build proc macros both for the host and the target unless proc-macros are not // supported by the target. @@ -719,7 +721,7 @@ impl Builder<'_> { // feature on the rustc side. cargo.arg("-Zbinary-dep-depinfo"); let allow_features = match mode { - Mode::ToolBootstrap | Mode::ToolStd => { + Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolTarget => { // Restrict the allowed features so we don't depend on nightly // accidentally. // @@ -827,7 +829,7 @@ impl Builder<'_> { cargo .env("RUSTBUILD_NATIVE_DIR", self.native_dir(target)) .env("RUSTC_REAL", self.rustc(compiler)) - .env("RUSTC_STAGE", stage.to_string()) + .env("RUSTC_STAGE", build_compiler_stage.to_string()) .env("RUSTC_SYSROOT", sysroot) .env("RUSTC_LIBDIR", libdir) .env("RUSTDOC", self.bootstrap_out.join("rustdoc")) @@ -872,7 +874,7 @@ impl Builder<'_> { let debuginfo_level = match mode { Mode::Rustc | Mode::Codegen => self.config.rust_debuginfo_level_rustc, Mode::Std => self.config.rust_debuginfo_level_std, - Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc => { + Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc | Mode::ToolTarget => { self.config.rust_debuginfo_level_tools } }; @@ -884,11 +886,10 @@ impl Builder<'_> { profile_var("DEBUG_ASSERTIONS"), match mode { Mode::Std => self.config.std_debug_assertions, - Mode::Rustc => self.config.rustc_debug_assertions, - Mode::Codegen => self.config.rustc_debug_assertions, - Mode::ToolBootstrap => self.config.tools_debug_assertions, - Mode::ToolStd => self.config.tools_debug_assertions, - Mode::ToolRustc => self.config.tools_debug_assertions, + Mode::Rustc | Mode::Codegen => self.config.rustc_debug_assertions, + Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc | Mode::ToolTarget => { + self.config.tools_debug_assertions + } } .to_string(), ); @@ -959,7 +960,11 @@ impl Builder<'_> { cargo.env("CFG_VIRTUAL_RUSTC_DEV_SOURCE_BASE_DIR", map_to); } } - Mode::Std | Mode::ToolBootstrap | Mode::ToolRustc | Mode::ToolStd => { + Mode::Std + | Mode::ToolBootstrap + | Mode::ToolRustc + | Mode::ToolStd + | Mode::ToolTarget => { if let Some(ref map_to) = self.build.debuginfo_map_to(GitRepo::Rustc, RemapScheme::NonCompiler) { @@ -1274,7 +1279,7 @@ impl Builder<'_> { }; if let Some(limit) = limit - && (stage == 0 + && (build_compiler_stage == 0 || self.config.default_codegen_backend(target).unwrap_or_default() == "llvm") { rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={limit}")); diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index 7464327fde989..34737606020c9 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -959,6 +959,7 @@ impl<'a> Builder<'a> { tool::RemoteTestServer, tool::RemoteTestClient, tool::RustInstaller, + tool::FeaturesStatusDump, tool::Cargo, tool::RustAnalyzer, tool::RustAnalyzerProcMacroSrv, @@ -980,6 +981,8 @@ impl<'a> Builder<'a> { tool::CoverageDump, tool::LlvmBitcodeLinker, tool::RustcPerf, + tool::WasmComponentLd, + tool::LldWrapper ), Kind::Clippy => describe!( clippy::Std, diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs index bbcb58fca14f3..4ca9ff98dee4b 100644 --- a/src/bootstrap/src/core/builder/tests.rs +++ b/src/bootstrap/src/core/builder/tests.rs @@ -769,11 +769,11 @@ mod snapshot { [build] llvm [build] rustc 0 -> rustc 1 [build] rustc 0 -> LldWrapper 1 - [build] rustc 1 -> LlvmBitcodeLinker 2 + [build] rustc 0 -> LlvmBitcodeLinker 1 [build] rustc 1 -> std 1 [build] rustc 1 -> rustc 2 [build] rustc 1 -> LldWrapper 2 - [build] rustc 2 -> LlvmBitcodeLinker 3 + [build] rustc 1 -> LlvmBitcodeLinker 2 [build] rustc 2 -> std 2 [build] rustdoc 1 " @@ -793,17 +793,17 @@ mod snapshot { [build] llvm [build] rustc 0 -> rustc 1 [build] rustc 0 -> LldWrapper 1 - [build] rustc 1 -> LlvmBitcodeLinker 2 + [build] rustc 0 -> LlvmBitcodeLinker 1 [build] rustc 1 -> std 1 [build] rustc 1 -> rustc 2 [build] rustc 1 -> LldWrapper 2 - [build] rustc 2 -> LlvmBitcodeLinker 3 + [build] rustc 1 -> LlvmBitcodeLinker 2 [build] rustc 1 -> std 1 [build] rustc 2 -> std 2 [build] llvm [build] rustc 1 -> rustc 2 [build] rustc 1 -> LldWrapper 2 - [build] rustc 2 -> LlvmBitcodeLinker 3 + [build] rustc 1 -> LlvmBitcodeLinker 2 [build] rustdoc 1 " ); @@ -1062,18 +1062,28 @@ mod snapshot { fn dist_extended() { let ctx = TestCtx::new(); insta::assert_snapshot!( - ctx - .config("dist") - .args(&["--set", "build.extended=true"]) - .render_steps(), @r" + ctx.config("dist") + .args(&[ + "--set", + "build.extended=true", + "--set", + "rust.llvm-bitcode-linker=true", + "--set", + "rust.lld=true", + ]) + .render_steps(), @r" [build] rustc 0 -> UnstableBookGen 1 [build] rustc 0 -> Rustbook 1 [build] llvm [build] rustc 0 -> rustc 1 + [build] rustc 0 -> LldWrapper 1 [build] rustc 0 -> WasmComponentLd 1 + [build] rustc 0 -> LlvmBitcodeLinker 1 [build] rustc 1 -> std 1 [build] rustc 1 -> rustc 2 + [build] rustc 1 -> LldWrapper 2 [build] rustc 1 -> WasmComponentLd 2 + [build] rustc 1 -> LlvmBitcodeLinker 2 [build] rustdoc 1 [doc] std 2 [build] rustc 2 -> std 2 @@ -1092,7 +1102,6 @@ mod snapshot { [build] rustc 0 -> cargo-clippy 1 [build] rustc 0 -> miri 1 [build] rustc 0 -> cargo-miri 1 - [build] rustc 1 -> LlvmBitcodeLinker 2 "); } @@ -1284,6 +1293,7 @@ mod snapshot { [build] rustc 0 -> miri 1 [build] rustc 0 -> cargo-miri 1 [build] rustc 1 -> LlvmBitcodeLinker 2 + [build] rustc 0 -> LlvmBitcodeLinker 1 "); } diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index ef5c28272b8e1..4ec0b089bacf9 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -253,12 +253,24 @@ pub enum Mode { /// These tools are intended to be only executed on the host system that /// invokes bootstrap, and they thus cannot be cross-compiled. /// - /// They are always built using the stage0 compiler, and typically they + /// They are always built using the stage0 compiler, and they /// can be compiled with stable Rust. /// /// These tools also essentially do not participate in staging. ToolBootstrap, + /// Build a cross-compilable helper tool. These tools do not depend on unstable features or + /// compiler internals, but they might be cross-compilable (so we cannot build them using the + /// stage0 compiler, unlike `ToolBootstrap`). + /// + /// Some of these tools are also shipped in our `dist` archives. + /// While we could compile them using the stage0 compiler when not cross-compiling, we instead + /// use the in-tree compiler (and std) to build them, so that we can ship e.g. std security + /// fixes and avoid depending fully on stage0 for the artifacts that we ship. + /// + /// This mode is used e.g. for linkers and linker tools invoked by rustc on its host target. + ToolTarget, + /// Build a tool which uses the locally built std, placing output in the /// "stageN-tools" directory. Its usage is quite rare, mainly used by /// compiletest which needs libtest. @@ -273,11 +285,21 @@ pub enum Mode { impl Mode { pub fn is_tool(&self) -> bool { - matches!(self, Mode::ToolBootstrap | Mode::ToolRustc | Mode::ToolStd) + match self { + Mode::ToolBootstrap | Mode::ToolRustc | Mode::ToolStd | Mode::ToolTarget => true, + Mode::Std | Mode::Codegen | Mode::Rustc => false, + } } pub fn must_support_dlopen(&self) -> bool { - matches!(self, Mode::Std | Mode::Codegen) + match self { + Mode::Std | Mode::Codegen => true, + Mode::ToolBootstrap + | Mode::ToolRustc + | Mode::ToolStd + | Mode::ToolTarget + | Mode::Rustc => false, + } } } @@ -804,17 +826,39 @@ impl Build { /// stage when running with a particular host compiler. /// /// The mode indicates what the root directory is for. - fn stage_out(&self, compiler: Compiler, mode: Mode) -> PathBuf { - let suffix = match mode { - Mode::Std => "-std", - Mode::Rustc => "-rustc", - Mode::Codegen => "-codegen", - Mode::ToolBootstrap => { - return self.out.join(compiler.host).join("bootstrap-tools"); + fn stage_out(&self, build_compiler: Compiler, mode: Mode) -> PathBuf { + use std::fmt::Write; + + fn bootstrap_tool() -> (Option, &'static str) { + (None, "bootstrap-tools") + } + fn staged_tool(build_compiler: Compiler) -> (Option, &'static str) { + (Some(build_compiler.stage), "tools") + } + + let (stage, suffix) = match mode { + Mode::Std => (Some(build_compiler.stage), "std"), + Mode::Rustc => (Some(build_compiler.stage), "rustc"), + Mode::Codegen => (Some(build_compiler.stage), "codegen"), + Mode::ToolBootstrap => bootstrap_tool(), + Mode::ToolStd | Mode::ToolRustc => (Some(build_compiler.stage), "tools"), + Mode::ToolTarget => { + // If we're not cross-compiling (the common case), share the target directory with + // bootstrap tools to reuse the build cache. + if build_compiler.stage == 0 { + bootstrap_tool() + } else { + staged_tool(build_compiler) + } } - Mode::ToolStd | Mode::ToolRustc => "-tools", }; - self.out.join(compiler.host).join(format!("stage{}{}", compiler.stage, suffix)) + let path = self.out.join(build_compiler.host); + let mut dir_name = String::new(); + if let Some(stage) = stage { + write!(dir_name, "stage{stage}-").unwrap(); + } + dir_name.push_str(suffix); + path.join(dir_name) } /// Returns the root output directory for all Cargo output in a given stage,