From 48a07bfb958a48da0c909f4607eb0cf9118fc902 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 5 Jul 2016 21:58:20 -0700 Subject: [PATCH] rustbuild: Remove the `build` directory The organization in rustbuild was a little odd at the moment where the `lib.rs` was quite small but the binary `main.rs` was much larger. Unfortunately as well there was a `build/` directory with the implementation of the build system, but this directory was ignored by GitHub on the file-search prompt which was a little annoying. This commit reorganizes rustbuild slightly where all the library files (the build system) is located directly inside of `src/bootstrap` and all the binaries now live in `src/bootstrap/bin` (they're small). Hopefully this should allow GitHub to index and allow navigating all the files while maintaining a relatively similar layout to the other libraries in `src/`. --- src/bootstrap/Cargo.toml | 6 +- src/bootstrap/{ => bin}/main.rs | 14 +- src/bootstrap/{ => bin}/rustc.rs | 5 +- src/bootstrap/{ => bin}/rustdoc.rs | 5 +- src/bootstrap/build/mod.rs | 871 -------------------------- src/bootstrap/{build => }/cc.rs | 4 +- src/bootstrap/{build => }/channel.rs | 2 +- src/bootstrap/{build => }/check.rs | 5 +- src/bootstrap/{build => }/clean.rs | 2 +- src/bootstrap/{build => }/compile.rs | 4 +- src/bootstrap/{build => }/config.rs | 0 src/bootstrap/{build => }/dist.rs | 4 +- src/bootstrap/{build => }/doc.rs | 4 +- src/bootstrap/{build => }/flags.rs | 0 src/bootstrap/{build => }/job.rs | 0 src/bootstrap/lib.rs | 882 ++++++++++++++++++++++++++- src/bootstrap/{build => }/native.rs | 4 +- src/bootstrap/{build => }/sanity.rs | 2 +- src/bootstrap/{build => }/step.rs | 2 +- src/bootstrap/{build => }/util.rs | 23 +- 20 files changed, 909 insertions(+), 930 deletions(-) rename src/bootstrap/{ => bin}/main.rs (82%) rename src/bootstrap/{ => bin}/rustc.rs (97%) rename src/bootstrap/{ => bin}/rustdoc.rs (89%) delete mode 100644 src/bootstrap/build/mod.rs rename src/bootstrap/{build => }/cc.rs (99%) rename src/bootstrap/{build => }/channel.rs (99%) rename src/bootstrap/{build => }/check.rs (99%) rename src/bootstrap/{build => }/clean.rs (98%) rename src/bootstrap/{build => }/compile.rs (99%) rename src/bootstrap/{build => }/config.rs (100%) rename src/bootstrap/{build => }/dist.rs (99%) rename src/bootstrap/{build => }/doc.rs (99%) rename src/bootstrap/{build => }/flags.rs (100%) rename src/bootstrap/{build => }/job.rs (100%) rename src/bootstrap/{build => }/native.rs (99%) rename src/bootstrap/{build => }/sanity.rs (99%) rename src/bootstrap/{build => }/step.rs (99%) rename src/bootstrap/{build => }/util.rs (86%) diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index f9a64567ffde0..cde4a825be1fb 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -9,15 +9,15 @@ path = "lib.rs" [[bin]] name = "bootstrap" -path = "main.rs" +path = "bin/main.rs" [[bin]] name = "rustc" -path = "rustc.rs" +path = "bin/rustc.rs" [[bin]] name = "rustdoc" -path = "rustdoc.rs" +path = "bin/rustdoc.rs" [dependencies] build_helper = { path = "../build_helper" } diff --git a/src/bootstrap/main.rs b/src/bootstrap/bin/main.rs similarity index 82% rename from src/bootstrap/main.rs rename to src/bootstrap/bin/main.rs index 18d03b5d59c29..c47f4fd8ec64b 100644 --- a/src/bootstrap/main.rs +++ b/src/bootstrap/bin/main.rs @@ -18,22 +18,10 @@ #![deny(warnings)] extern crate bootstrap; -extern crate build_helper; -extern crate cmake; -extern crate filetime; -extern crate gcc; -extern crate getopts; -extern crate libc; -extern crate num_cpus; -extern crate rustc_serialize; -extern crate toml; -extern crate md5; use std::env; -use build::{Flags, Config, Build}; - -mod build; +use bootstrap::{Flags, Config, Build}; fn main() { let args = env::args().skip(1).collect::>(); diff --git a/src/bootstrap/rustc.rs b/src/bootstrap/bin/rustc.rs similarity index 97% rename from src/bootstrap/rustc.rs rename to src/bootstrap/bin/rustc.rs index 97decedf91dce..c64cbb9a74e09 100644 --- a/src/bootstrap/rustc.rs +++ b/src/bootstrap/bin/rustc.rs @@ -53,13 +53,14 @@ fn main() { let rustc = env::var_os(rustc).unwrap(); let libdir = env::var_os(libdir).unwrap(); - let mut dylib_path = bootstrap::dylib_path(); + let mut dylib_path = bootstrap::util::dylib_path(); dylib_path.insert(0, PathBuf::from(libdir)); let mut cmd = Command::new(rustc); cmd.args(&args) .arg("--cfg").arg(format!("stage{}", stage)) - .env(bootstrap::dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + .env(bootstrap::util::dylib_path_var(), + env::join_paths(&dylib_path).unwrap()); if let Some(target) = target { // The stage0 compiler has a special sysroot distinct from what we diff --git a/src/bootstrap/rustdoc.rs b/src/bootstrap/bin/rustdoc.rs similarity index 89% rename from src/bootstrap/rustdoc.rs rename to src/bootstrap/bin/rustdoc.rs index 88ac26d32f6c3..79629bfd71711 100644 --- a/src/bootstrap/rustdoc.rs +++ b/src/bootstrap/bin/rustdoc.rs @@ -23,14 +23,15 @@ fn main() { let rustdoc = env::var_os("RUSTDOC_REAL").unwrap(); let libdir = env::var_os("RUSTC_LIBDIR").unwrap(); - let mut dylib_path = bootstrap::dylib_path(); + let mut dylib_path = bootstrap::util::dylib_path(); dylib_path.insert(0, PathBuf::from(libdir)); let mut cmd = Command::new(rustdoc); cmd.args(&args) .arg("--cfg").arg(format!("stage{}", env::var("RUSTC_STAGE").unwrap())) .arg("--cfg").arg("dox") - .env(bootstrap::dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + .env(bootstrap::util::dylib_path_var(), + env::join_paths(&dylib_path).unwrap()); std::process::exit(match cmd.status() { Ok(s) => s.code().unwrap_or(1), Err(e) => panic!("\n\nfailed to run {:?}: {}\n\n", cmd, e), diff --git a/src/bootstrap/build/mod.rs b/src/bootstrap/build/mod.rs deleted file mode 100644 index 195d1bc90c655..0000000000000 --- a/src/bootstrap/build/mod.rs +++ /dev/null @@ -1,871 +0,0 @@ -// Copyright 2015 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! Implementation of rustbuild, the Rust build system. -//! -//! This module, and its descendants, are the implementation of the Rust build -//! system. Most of this build system is backed by Cargo but the outer layer -//! here serves as the ability to orchestrate calling Cargo, sequencing Cargo -//! builds, building artifacts like LLVM, etc. -//! -//! More documentation can be found in each respective module below. - -use std::cell::RefCell; -use std::collections::HashMap; -use std::env; -use std::fs::{self, File}; -use std::path::{PathBuf, Path}; -use std::process::Command; - -use build_helper::{run_silent, output}; -use gcc; -use num_cpus; - -use build::util::{exe, mtime, libdir, add_lib_path}; - -/// A helper macro to `unwrap` a result except also print out details like: -/// -/// * The file/line of the panic -/// * The expression that failed -/// * The error itself -/// -/// This is currently used judiciously throughout the build system rather than -/// using a `Result` with `try!`, but this may change on day... -macro_rules! t { - ($e:expr) => (match $e { - Ok(e) => e, - Err(e) => panic!("{} failed with {}", stringify!($e), e), - }) -} - -mod cc; -mod channel; -mod check; -mod clean; -mod compile; -mod config; -mod dist; -mod doc; -mod flags; -mod native; -mod sanity; -mod step; -mod util; - -#[cfg(windows)] -mod job; - -#[cfg(not(windows))] -mod job { - pub unsafe fn setup() {} -} - -pub use build::config::Config; -pub use build::flags::Flags; - -/// A structure representing a Rust compiler. -/// -/// Each compiler has a `stage` that it is associated with and a `host` that -/// corresponds to the platform the compiler runs on. This structure is used as -/// a parameter to many methods below. -#[derive(Eq, PartialEq, Clone, Copy, Hash, Debug)] -pub struct Compiler<'a> { - stage: u32, - host: &'a str, -} - -/// Global configuration for the build system. -/// -/// This structure transitively contains all configuration for the build system. -/// All filesystem-encoded configuration is in `config`, all flags are in -/// `flags`, and then parsed or probed information is listed in the keys below. -/// -/// This structure is a parameter of almost all methods in the build system, -/// although most functions are implemented as free functions rather than -/// methods specifically on this structure itself (to make it easier to -/// organize). -pub struct Build { - // User-specified configuration via config.toml - config: Config, - - // User-specified configuration via CLI flags - flags: Flags, - - // Derived properties from the above two configurations - cargo: PathBuf, - rustc: PathBuf, - src: PathBuf, - out: PathBuf, - release: String, - unstable_features: bool, - ver_hash: Option, - short_ver_hash: Option, - ver_date: Option, - version: String, - package_vers: String, - bootstrap_key: String, - bootstrap_key_stage0: String, - - // Probed tools at runtime - gdb_version: Option, - lldb_version: Option, - lldb_python_dir: Option, - - // Runtime state filled in later on - cc: HashMap)>, - cxx: HashMap, - compiler_rt_built: RefCell>, -} - -/// The various "modes" of invoking Cargo. -/// -/// These entries currently correspond to the various output directories of the -/// build system, with each mod generating output in a different directory. -#[derive(Clone, Copy)] -pub enum Mode { - /// This cargo is going to build the standard library, placing output in the - /// "stageN-std" directory. - Libstd, - - /// This cargo is going to build libtest, placing output in the - /// "stageN-test" directory. - Libtest, - - /// This cargo is going to build librustc and compiler libraries, placing - /// output in the "stageN-rustc" directory. - Librustc, - - /// This cargo is going to some build tool, placing output in the - /// "stageN-tools" directory. - Tool, -} - -impl Build { - /// Creates a new set of build configuration from the `flags` on the command - /// line and the filesystem `config`. - /// - /// By default all build output will be placed in the current directory. - pub fn new(flags: Flags, config: Config) -> Build { - let cwd = t!(env::current_dir()); - let src = flags.src.clone().unwrap_or(cwd.clone()); - let out = cwd.join("build"); - - let stage0_root = out.join(&config.build).join("stage0/bin"); - let rustc = match config.rustc { - Some(ref s) => PathBuf::from(s), - None => stage0_root.join(exe("rustc", &config.build)), - }; - let cargo = match config.cargo { - Some(ref s) => PathBuf::from(s), - None => stage0_root.join(exe("cargo", &config.build)), - }; - - Build { - flags: flags, - config: config, - cargo: cargo, - rustc: rustc, - src: src, - out: out, - - release: String::new(), - unstable_features: false, - ver_hash: None, - short_ver_hash: None, - ver_date: None, - version: String::new(), - bootstrap_key: String::new(), - bootstrap_key_stage0: String::new(), - package_vers: String::new(), - cc: HashMap::new(), - cxx: HashMap::new(), - compiler_rt_built: RefCell::new(HashMap::new()), - gdb_version: None, - lldb_version: None, - lldb_python_dir: None, - } - } - - /// Executes the entire build, as configured by the flags and configuration. - pub fn build(&mut self) { - use build::step::Source::*; - - unsafe { - job::setup(); - } - - if self.flags.clean { - return clean::clean(self); - } - - self.verbose("finding compilers"); - cc::find(self); - self.verbose("running sanity check"); - sanity::check(self); - self.verbose("collecting channel variables"); - channel::collect(self); - self.verbose("updating submodules"); - self.update_submodules(); - - // The main loop of the build system. - // - // The `step::all` function returns a topographically sorted list of all - // steps that need to be executed as part of this build. Each step has a - // corresponding entry in `step.rs` and indicates some unit of work that - // needs to be done as part of the build. - // - // Almost all of these are simple one-liners that shell out to the - // corresponding functionality in the extra modules, where more - // documentation can be found. - for target in step::all(self) { - let doc_out = self.out.join(&target.target).join("doc"); - match target.src { - Llvm { _dummy } => { - native::llvm(self, target.target); - } - CompilerRt { _dummy } => { - native::compiler_rt(self, target.target); - } - TestHelpers { _dummy } => { - native::test_helpers(self, target.target); - } - Libstd { compiler } => { - compile::std(self, target.target, &compiler); - } - Libtest { compiler } => { - compile::test(self, target.target, &compiler); - } - Librustc { compiler } => { - compile::rustc(self, target.target, &compiler); - } - LibstdLink { compiler, host } => { - compile::std_link(self, target.target, &compiler, host); - } - LibtestLink { compiler, host } => { - compile::test_link(self, target.target, &compiler, host); - } - LibrustcLink { compiler, host } => { - compile::rustc_link(self, target.target, &compiler, host); - } - Rustc { stage: 0 } => { - // nothing to do... - } - Rustc { stage } => { - compile::assemble_rustc(self, stage, target.target); - } - ToolLinkchecker { stage } => { - compile::tool(self, stage, target.target, "linkchecker"); - } - ToolRustbook { stage } => { - compile::tool(self, stage, target.target, "rustbook"); - } - ToolErrorIndex { stage } => { - compile::tool(self, stage, target.target, - "error_index_generator"); - } - ToolCargoTest { stage } => { - compile::tool(self, stage, target.target, "cargotest"); - } - ToolTidy { stage } => { - compile::tool(self, stage, target.target, "tidy"); - } - ToolCompiletest { stage } => { - compile::tool(self, stage, target.target, "compiletest"); - } - DocBook { stage } => { - doc::rustbook(self, stage, target.target, "book", &doc_out); - } - DocNomicon { stage } => { - doc::rustbook(self, stage, target.target, "nomicon", - &doc_out); - } - DocStyle { stage } => { - doc::rustbook(self, stage, target.target, "style", - &doc_out); - } - DocStandalone { stage } => { - doc::standalone(self, stage, target.target, &doc_out); - } - DocStd { stage } => { - doc::std(self, stage, target.target, &doc_out); - } - DocTest { stage } => { - doc::test(self, stage, target.target, &doc_out); - } - DocRustc { stage } => { - doc::rustc(self, stage, target.target, &doc_out); - } - DocErrorIndex { stage } => { - doc::error_index(self, stage, target.target, &doc_out); - } - - CheckLinkcheck { stage } => { - check::linkcheck(self, stage, target.target); - } - CheckCargoTest { stage } => { - check::cargotest(self, stage, target.target); - } - CheckTidy { stage } => { - check::tidy(self, stage, target.target); - } - CheckRPass { compiler } => { - check::compiletest(self, &compiler, target.target, - "run-pass", "run-pass"); - } - CheckRPassFull { compiler } => { - check::compiletest(self, &compiler, target.target, - "run-pass", "run-pass-fulldeps"); - } - CheckCFail { compiler } => { - check::compiletest(self, &compiler, target.target, - "compile-fail", "compile-fail"); - } - CheckCFailFull { compiler } => { - check::compiletest(self, &compiler, target.target, - "compile-fail", "compile-fail-fulldeps") - } - CheckPFail { compiler } => { - check::compiletest(self, &compiler, target.target, - "parse-fail", "parse-fail"); - } - CheckRFail { compiler } => { - check::compiletest(self, &compiler, target.target, - "run-fail", "run-fail"); - } - CheckRFailFull { compiler } => { - check::compiletest(self, &compiler, target.target, - "run-fail", "run-fail-fulldeps"); - } - CheckPretty { compiler } => { - check::compiletest(self, &compiler, target.target, - "pretty", "pretty"); - } - CheckPrettyRPass { compiler } => { - check::compiletest(self, &compiler, target.target, - "pretty", "run-pass"); - } - CheckPrettyRPassFull { compiler } => { - check::compiletest(self, &compiler, target.target, - "pretty", "run-pass-fulldeps"); - } - CheckPrettyRFail { compiler } => { - check::compiletest(self, &compiler, target.target, - "pretty", "run-fail"); - } - CheckPrettyRFailFull { compiler } => { - check::compiletest(self, &compiler, target.target, - "pretty", "run-fail-fulldeps"); - } - CheckPrettyRPassValgrind { compiler } => { - check::compiletest(self, &compiler, target.target, - "pretty", "run-pass-valgrind"); - } - CheckCodegen { compiler } => { - check::compiletest(self, &compiler, target.target, - "codegen", "codegen"); - } - CheckCodegenUnits { compiler } => { - check::compiletest(self, &compiler, target.target, - "codegen-units", "codegen-units"); - } - CheckIncremental { compiler } => { - check::compiletest(self, &compiler, target.target, - "incremental", "incremental"); - } - CheckUi { compiler } => { - check::compiletest(self, &compiler, target.target, - "ui", "ui"); - } - CheckDebuginfo { compiler } => { - if target.target.contains("msvc") { - // nothing to do - } else if target.target.contains("apple") { - check::compiletest(self, &compiler, target.target, - "debuginfo-lldb", "debuginfo"); - } else { - check::compiletest(self, &compiler, target.target, - "debuginfo-gdb", "debuginfo"); - } - } - CheckRustdoc { compiler } => { - check::compiletest(self, &compiler, target.target, - "rustdoc", "rustdoc"); - } - CheckRPassValgrind { compiler } => { - check::compiletest(self, &compiler, target.target, - "run-pass-valgrind", "run-pass-valgrind"); - } - CheckDocs { compiler } => { - check::docs(self, &compiler); - } - CheckErrorIndex { compiler } => { - check::error_index(self, &compiler); - } - CheckRMake { compiler } => { - check::compiletest(self, &compiler, target.target, - "run-make", "run-make") - } - CheckCrateStd { compiler } => { - check::krate(self, &compiler, target.target, Mode::Libstd) - } - CheckCrateTest { compiler } => { - check::krate(self, &compiler, target.target, Mode::Libtest) - } - CheckCrateRustc { compiler } => { - check::krate(self, &compiler, target.target, Mode::Librustc) - } - - DistDocs { stage } => dist::docs(self, stage, target.target), - DistMingw { _dummy } => dist::mingw(self, target.target), - DistRustc { stage } => dist::rustc(self, stage, target.target), - DistStd { compiler } => dist::std(self, &compiler, target.target), - - DebuggerScripts { stage } => { - let compiler = Compiler::new(stage, target.target); - dist::debugger_scripts(self, - &self.sysroot(&compiler), - target.target); - } - - AndroidCopyLibs { compiler } => { - check::android_copy_libs(self, &compiler, target.target); - } - - // pseudo-steps - Dist { .. } | - Doc { .. } | - CheckTarget { .. } | - Check { .. } => {} - } - } - } - - /// Updates all git submodules that we have. - /// - /// This will detect if any submodules are out of date an run the necessary - /// commands to sync them all with upstream. - fn update_submodules(&self) { - if !self.config.submodules { - return - } - if fs::metadata(self.src.join(".git")).is_err() { - return - } - let git_submodule = || { - let mut cmd = Command::new("git"); - cmd.current_dir(&self.src).arg("submodule"); - return cmd - }; - - // FIXME: this takes a seriously long time to execute on Windows and a - // nontrivial amount of time on Unix, we should have a better way - // of detecting whether we need to run all the submodule commands - // below. - let out = output(git_submodule().arg("status")); - if !out.lines().any(|l| l.starts_with("+") || l.starts_with("-")) { - return - } - - self.run(git_submodule().arg("sync")); - self.run(git_submodule().arg("init")); - self.run(git_submodule().arg("update")); - self.run(git_submodule().arg("update").arg("--recursive")); - self.run(git_submodule().arg("status").arg("--recursive")); - self.run(git_submodule().arg("foreach").arg("--recursive") - .arg("git").arg("clean").arg("-fdx")); - self.run(git_submodule().arg("foreach").arg("--recursive") - .arg("git").arg("checkout").arg(".")); - } - - /// Clear out `dir` if `input` is newer. - /// - /// After this executes, it will also ensure that `dir` exists. - fn clear_if_dirty(&self, dir: &Path, input: &Path) { - let stamp = dir.join(".stamp"); - if mtime(&stamp) < mtime(input) { - self.verbose(&format!("Dirty - {}", dir.display())); - let _ = fs::remove_dir_all(dir); - } - t!(fs::create_dir_all(dir)); - t!(File::create(stamp)); - } - - /// Prepares an invocation of `cargo` to be run. - /// - /// This will create a `Command` that represents a pending execution of - /// Cargo. This cargo will be configured to use `compiler` as the actual - /// rustc compiler, its output will be scoped by `mode`'s output directory, - /// it will pass the `--target` flag for the specified `target`, and will be - /// executing the Cargo command `cmd`. - fn cargo(&self, - compiler: &Compiler, - mode: Mode, - target: &str, - cmd: &str) -> Command { - let mut cargo = Command::new(&self.cargo); - let out_dir = self.stage_out(compiler, mode); - cargo.env("CARGO_TARGET_DIR", out_dir) - .arg(cmd) - .arg("-j").arg(self.jobs().to_string()) - .arg("--target").arg(target); - - let stage; - if compiler.stage == 0 && self.config.local_rebuild { - // Assume the local-rebuild rustc already has stage1 features. - stage = 1; - } else { - stage = compiler.stage; - } - - // Customize the compiler we're running. Specify the compiler to cargo - // as our shim and then pass it some various options used to configure - // how the actual compiler itself is called. - // - // These variables are primarily all read by - // src/bootstrap/{rustc,rustdoc.rs} - cargo.env("RUSTC", self.out.join("bootstrap/debug/rustc")) - .env("RUSTC_REAL", self.compiler_path(compiler)) - .env("RUSTC_STAGE", stage.to_string()) - .env("RUSTC_DEBUGINFO", self.config.rust_debuginfo.to_string()) - .env("RUSTC_CODEGEN_UNITS", - self.config.rust_codegen_units.to_string()) - .env("RUSTC_DEBUG_ASSERTIONS", - self.config.rust_debug_assertions.to_string()) - .env("RUSTC_SNAPSHOT", &self.rustc) - .env("RUSTC_SYSROOT", self.sysroot(compiler)) - .env("RUSTC_LIBDIR", self.rustc_libdir(compiler)) - .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_snapshot_libdir()) - .env("RUSTC_RPATH", self.config.rust_rpath.to_string()) - .env("RUSTDOC", self.out.join("bootstrap/debug/rustdoc")) - .env("RUSTDOC_REAL", self.rustdoc(compiler)) - .env("RUSTC_FLAGS", self.rustc_flags(target).join(" ")); - - self.add_bootstrap_key(compiler, &mut cargo); - - // Specify some various options for build scripts used throughout - // the build. - // - // FIXME: the guard against msvc shouldn't need to be here - if !target.contains("msvc") { - cargo.env(format!("CC_{}", target), self.cc(target)) - .env(format!("AR_{}", target), self.ar(target).unwrap()) // only msvc is None - .env(format!("CFLAGS_{}", target), self.cflags(target).join(" ")); - } - - // If we're building for OSX, inform the compiler and the linker that - // we want to build a compiler runnable on 10.7 - if target.contains("apple-darwin") { - cargo.env("MACOSX_DEPLOYMENT_TARGET", "10.7"); - } - - // Environment variables *required* needed throughout the build - // - // FIXME: should update code to not require this env var - cargo.env("CFG_COMPILER_HOST_TRIPLE", target); - - if self.config.verbose || self.flags.verbose { - cargo.arg("-v"); - } - if self.config.rust_optimize { - cargo.arg("--release"); - } - return cargo - } - - /// Get a path to the compiler specified. - fn compiler_path(&self, compiler: &Compiler) -> PathBuf { - if compiler.is_snapshot(self) { - self.rustc.clone() - } else { - self.sysroot(compiler).join("bin").join(exe("rustc", compiler.host)) - } - } - - /// Get the specified tool built by the specified compiler - fn tool(&self, compiler: &Compiler, tool: &str) -> PathBuf { - self.cargo_out(compiler, Mode::Tool, compiler.host) - .join(exe(tool, compiler.host)) - } - - /// Get the `rustdoc` executable next to the specified compiler - fn rustdoc(&self, compiler: &Compiler) -> PathBuf { - let mut rustdoc = self.compiler_path(compiler); - rustdoc.pop(); - rustdoc.push(exe("rustdoc", compiler.host)); - return rustdoc - } - - /// Get a `Command` which is ready to run `tool` in `stage` built for - /// `host`. - fn tool_cmd(&self, compiler: &Compiler, tool: &str) -> Command { - let mut cmd = Command::new(self.tool(&compiler, tool)); - let host = compiler.host; - let paths = vec![ - self.cargo_out(compiler, Mode::Libstd, host).join("deps"), - self.cargo_out(compiler, Mode::Libtest, host).join("deps"), - self.cargo_out(compiler, Mode::Librustc, host).join("deps"), - self.cargo_out(compiler, Mode::Tool, host).join("deps"), - ]; - add_lib_path(paths, &mut cmd); - return cmd - } - - /// Get the space-separated set of activated features for the standard - /// library. - fn std_features(&self) -> String { - let mut features = String::new(); - if self.config.debug_jemalloc { - features.push_str(" debug-jemalloc"); - } - if self.config.use_jemalloc { - features.push_str(" jemalloc"); - } - return features - } - - /// Get the space-separated set of activated features for the compiler. - fn rustc_features(&self) -> String { - let mut features = String::new(); - if self.config.use_jemalloc { - features.push_str(" jemalloc"); - } - return features - } - - /// Component directory that Cargo will produce output into (e.g. - /// release/debug) - fn cargo_dir(&self) -> &'static str { - if self.config.rust_optimize {"release"} else {"debug"} - } - - /// Returns the sysroot for the `compiler` specified that *this build system - /// generates*. - /// - /// That is, the sysroot for the stage0 compiler is not what the compiler - /// thinks it is by default, but it's the same as the default for stages - /// 1-3. - fn sysroot(&self, compiler: &Compiler) -> PathBuf { - if compiler.stage == 0 { - self.out.join(compiler.host).join("stage0-sysroot") - } else { - self.out.join(compiler.host).join(format!("stage{}", compiler.stage)) - } - } - - /// Returns the libdir where the standard library and other artifacts are - /// found for a compiler's sysroot. - fn sysroot_libdir(&self, compiler: &Compiler, target: &str) -> PathBuf { - self.sysroot(compiler).join("lib").join("rustlib") - .join(target).join("lib") - } - - /// Returns the root directory for all output generated in a particular - /// stage when running with a particular host compiler. - /// - /// The mode indicates what the root directory is for. - fn stage_out(&self, compiler: &Compiler, mode: Mode) -> PathBuf { - let suffix = match mode { - Mode::Libstd => "-std", - Mode::Libtest => "-test", - Mode::Tool => "-tools", - Mode::Librustc => "-rustc", - }; - self.out.join(compiler.host) - .join(format!("stage{}{}", compiler.stage, suffix)) - } - - /// Returns the root output directory for all Cargo output in a given stage, - /// running a particular comipler, wehther or not we're building the - /// standard library, and targeting the specified architecture. - fn cargo_out(&self, - compiler: &Compiler, - mode: Mode, - target: &str) -> PathBuf { - self.stage_out(compiler, mode).join(target).join(self.cargo_dir()) - } - - /// Root output directory for LLVM compiled for `target` - /// - /// Note that if LLVM is configured externally then the directory returned - /// will likely be empty. - fn llvm_out(&self, target: &str) -> PathBuf { - self.out.join(target).join("llvm") - } - - /// Returns the path to `llvm-config` for the specified target. - /// - /// If a custom `llvm-config` was specified for target then that's returned - /// instead. - fn llvm_config(&self, target: &str) -> PathBuf { - let target_config = self.config.target_config.get(target); - if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { - s.clone() - } else { - self.llvm_out(&self.config.build).join("bin") - .join(exe("llvm-config", target)) - } - } - - /// Returns the path to `FileCheck` binary for the specified target - fn llvm_filecheck(&self, target: &str) -> PathBuf { - let target_config = self.config.target_config.get(target); - if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { - s.parent().unwrap().join(exe("FileCheck", target)) - } else { - let base = self.llvm_out(&self.config.build).join("build"); - let exe = exe("FileCheck", target); - if self.config.build.contains("msvc") { - base.join("Release/bin").join(exe) - } else { - base.join("bin").join(exe) - } - } - } - - /// Root output directory for compiler-rt compiled for `target` - fn compiler_rt_out(&self, target: &str) -> PathBuf { - self.out.join(target).join("compiler-rt") - } - - /// Root output directory for rust_test_helpers library compiled for - /// `target` - fn test_helpers_out(&self, target: &str) -> PathBuf { - self.out.join(target).join("rust-test-helpers") - } - - /// Adds the compiler's directory of dynamic libraries to `cmd`'s dynamic - /// library lookup path. - fn add_rustc_lib_path(&self, compiler: &Compiler, cmd: &mut Command) { - // Windows doesn't need dylib path munging because the dlls for the - // compiler live next to the compiler and the system will find them - // automatically. - if cfg!(windows) { - return - } - - add_lib_path(vec![self.rustc_libdir(compiler)], cmd); - } - - /// Adds the compiler's bootstrap key to the environment of `cmd`. - fn add_bootstrap_key(&self, compiler: &Compiler, cmd: &mut Command) { - // In stage0 we're using a previously released stable compiler, so we - // use the stage0 bootstrap key. Otherwise we use our own build's - // bootstrap key. - let bootstrap_key = if compiler.is_snapshot(self) && !self.config.local_rebuild { - &self.bootstrap_key_stage0 - } else { - &self.bootstrap_key - }; - cmd.env("RUSTC_BOOTSTRAP_KEY", bootstrap_key); - } - - /// Returns the compiler's libdir where it stores the dynamic libraries that - /// it itself links against. - /// - /// For example this returns `/lib` on Unix and `/bin` on - /// Windows. - fn rustc_libdir(&self, compiler: &Compiler) -> PathBuf { - if compiler.is_snapshot(self) { - self.rustc_snapshot_libdir() - } else { - self.sysroot(compiler).join(libdir(compiler.host)) - } - } - - /// Returns the libdir of the snapshot compiler. - fn rustc_snapshot_libdir(&self) -> PathBuf { - self.rustc.parent().unwrap().parent().unwrap() - .join(libdir(&self.config.build)) - } - - /// Runs a command, printing out nice contextual information if it fails. - fn run(&self, cmd: &mut Command) { - self.verbose(&format!("running: {:?}", cmd)); - run_silent(cmd) - } - - /// Prints a message if this build is configured in verbose mode. - fn verbose(&self, msg: &str) { - if self.flags.verbose || self.config.verbose { - println!("{}", msg); - } - } - - /// Returns the number of parallel jobs that have been configured for this - /// build. - fn jobs(&self) -> u32 { - self.flags.jobs.unwrap_or(num_cpus::get() as u32) - } - - /// Returns the path to the C compiler for the target specified. - fn cc(&self, target: &str) -> &Path { - self.cc[target].0.path() - } - - /// Returns a list of flags to pass to the C compiler for the target - /// specified. - fn cflags(&self, target: &str) -> Vec { - // Filter out -O and /O (the optimization flags) that we picked up from - // gcc-rs because the build scripts will determine that for themselves. - let mut base = self.cc[target].0.args().iter() - .map(|s| s.to_string_lossy().into_owned()) - .filter(|s| !s.starts_with("-O") && !s.starts_with("/O")) - .collect::>(); - - // If we're compiling on OSX then we add a few unconditional flags - // indicating that we want libc++ (more filled out than libstdc++) and - // we want to compile for 10.7. This way we can ensure that - // LLVM/jemalloc/etc are all properly compiled. - if target.contains("apple-darwin") { - base.push("-stdlib=libc++".into()); - base.push("-mmacosx-version-min=10.7".into()); - } - return base - } - - /// Returns the path to the `ar` archive utility for the target specified. - fn ar(&self, target: &str) -> Option<&Path> { - self.cc[target].1.as_ref().map(|p| &**p) - } - - /// Returns the path to the C++ compiler for the target specified, may panic - /// if no C++ compiler was configured for the target. - fn cxx(&self, target: &str) -> &Path { - self.cxx[target].path() - } - - /// Returns flags to pass to the compiler to generate code for `target`. - fn rustc_flags(&self, target: &str) -> Vec { - // New flags should be added here with great caution! - // - // It's quite unfortunate to **require** flags to generate code for a - // target, so it should only be passed here if absolutely necessary! - // Most default configuration should be done through target specs rather - // than an entry here. - - let mut base = Vec::new(); - if target != self.config.build && !target.contains("msvc") { - base.push(format!("-Clinker={}", self.cc(target).display())); - } - return base - } -} - -impl<'a> Compiler<'a> { - /// Creates a new complier for the specified stage/host - fn new(stage: u32, host: &'a str) -> Compiler<'a> { - Compiler { stage: stage, host: host } - } - - /// Returns whether this is a snapshot compiler for `build`'s configuration - fn is_snapshot(&self, build: &Build) -> bool { - self.stage == 0 && self.host == build.config.build - } -} diff --git a/src/bootstrap/build/cc.rs b/src/bootstrap/cc.rs similarity index 99% rename from src/bootstrap/build/cc.rs rename to src/bootstrap/cc.rs index ff0941a97dce1..e2bde4a658611 100644 --- a/src/bootstrap/build/cc.rs +++ b/src/bootstrap/cc.rs @@ -36,8 +36,8 @@ use std::process::Command; use build_helper::{cc2ar, output}; use gcc; -use build::Build; -use build::config::Target; +use Build; +use config::Target; pub fn find(build: &mut Build) { // For all targets we're going to need a C compiler for building some shims diff --git a/src/bootstrap/build/channel.rs b/src/bootstrap/channel.rs similarity index 99% rename from src/bootstrap/build/channel.rs rename to src/bootstrap/channel.rs index 76d061eb43e0c..879c383404a4c 100644 --- a/src/bootstrap/build/channel.rs +++ b/src/bootstrap/channel.rs @@ -22,7 +22,7 @@ use std::process::Command; use build_helper::output; use md5; -use build::Build; +use Build; pub fn collect(build: &mut Build) { // Currently the canonical source for the release number (e.g. 1.10.0) and diff --git a/src/bootstrap/build/check.rs b/src/bootstrap/check.rs similarity index 99% rename from src/bootstrap/build/check.rs rename to src/bootstrap/check.rs index 0a096f8e4de41..3d8b1438125e6 100644 --- a/src/bootstrap/build/check.rs +++ b/src/bootstrap/check.rs @@ -20,10 +20,9 @@ use std::path::{PathBuf, Path}; use std::process::Command; use build_helper::output; -use bootstrap::{dylib_path, dylib_path_var}; -use build::{Build, Compiler, Mode}; -use build::util; +use {Build, Compiler, Mode}; +use util::{self, dylib_path, dylib_path_var}; const ADB_TEST_DIR: &'static str = "/data/tmp"; diff --git a/src/bootstrap/build/clean.rs b/src/bootstrap/clean.rs similarity index 98% rename from src/bootstrap/build/clean.rs rename to src/bootstrap/clean.rs index 91334bdb91e22..a466e2e6897f8 100644 --- a/src/bootstrap/build/clean.rs +++ b/src/bootstrap/clean.rs @@ -18,7 +18,7 @@ use std::fs; use std::path::Path; -use build::Build; +use Build; pub fn clean(build: &Build) { rm_rf(build, "tmp".as_ref()); diff --git a/src/bootstrap/build/compile.rs b/src/bootstrap/compile.rs similarity index 99% rename from src/bootstrap/build/compile.rs rename to src/bootstrap/compile.rs index 5ed9c1c18c218..8ec9c7f0109f3 100644 --- a/src/bootstrap/build/compile.rs +++ b/src/bootstrap/compile.rs @@ -23,8 +23,8 @@ use std::process::Command; use build_helper::output; -use build::util::{exe, staticlib, libdir, mtime, is_dylib, copy}; -use build::{Build, Compiler, Mode}; +use util::{exe, staticlib, libdir, mtime, is_dylib, copy}; +use {Build, Compiler, Mode}; /// Build the standard library. /// diff --git a/src/bootstrap/build/config.rs b/src/bootstrap/config.rs similarity index 100% rename from src/bootstrap/build/config.rs rename to src/bootstrap/config.rs diff --git a/src/bootstrap/build/dist.rs b/src/bootstrap/dist.rs similarity index 99% rename from src/bootstrap/build/dist.rs rename to src/bootstrap/dist.rs index 6eed7eaf206f4..1cf71c3aaecd6 100644 --- a/src/bootstrap/build/dist.rs +++ b/src/bootstrap/dist.rs @@ -23,8 +23,8 @@ use std::io::Write; use std::path::{PathBuf, Path}; use std::process::Command; -use build::{Build, Compiler}; -use build::util::{cp_r, libdir, is_dylib}; +use {Build, Compiler}; +use util::{cp_r, libdir, is_dylib}; fn package_vers(build: &Build) -> &str { match &build.config.channel[..] { diff --git a/src/bootstrap/build/doc.rs b/src/bootstrap/doc.rs similarity index 99% rename from src/bootstrap/build/doc.rs rename to src/bootstrap/doc.rs index f7cc742277a63..ac90ab547373b 100644 --- a/src/bootstrap/build/doc.rs +++ b/src/bootstrap/doc.rs @@ -22,8 +22,8 @@ use std::io::prelude::*; use std::path::Path; use std::process::Command; -use build::{Build, Compiler, Mode}; -use build::util::{up_to_date, cp_r}; +use {Build, Compiler, Mode}; +use util::{up_to_date, cp_r}; /// Invoke `rustbook` as compiled in `stage` for `target` for the doc book /// `name` into the `out` path. diff --git a/src/bootstrap/build/flags.rs b/src/bootstrap/flags.rs similarity index 100% rename from src/bootstrap/build/flags.rs rename to src/bootstrap/flags.rs diff --git a/src/bootstrap/build/job.rs b/src/bootstrap/job.rs similarity index 100% rename from src/bootstrap/build/job.rs rename to src/bootstrap/job.rs diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index ef6184d6ca76c..943271fc8a641 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -8,30 +8,872 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -//! A small helper library shared between the build system's executables +//! Implementation of rustbuild, the Rust build system. //! -//! Currently this just has some simple utilities for modifying the dynamic -//! library lookup path. +//! This module, and its descendants, are the implementation of the Rust build +//! system. Most of this build system is backed by Cargo but the outer layer +//! here serves as the ability to orchestrate calling Cargo, sequencing Cargo +//! builds, building artifacts like LLVM, etc. +//! +//! More documentation can be found in each respective module below. + +extern crate build_helper; +extern crate cmake; +extern crate filetime; +extern crate gcc; +extern crate getopts; +extern crate md5; +extern crate num_cpus; +extern crate rustc_serialize; +extern crate toml; +use std::cell::RefCell; +use std::collections::HashMap; use std::env; -use std::ffi::OsString; -use std::path::PathBuf; - -/// Returns the environment variable which the dynamic library lookup path -/// resides in for this platform. -pub fn dylib_path_var() -> &'static str { - if cfg!(target_os = "windows") { - "PATH" - } else if cfg!(target_os = "macos") { - "DYLD_LIBRARY_PATH" - } else { - "LD_LIBRARY_PATH" +use std::fs::{self, File}; +use std::path::{PathBuf, Path}; +use std::process::Command; + +use build_helper::{run_silent, output}; + +use util::{exe, mtime, libdir, add_lib_path}; + +/// A helper macro to `unwrap` a result except also print out details like: +/// +/// * The file/line of the panic +/// * The expression that failed +/// * The error itself +/// +/// This is currently used judiciously throughout the build system rather than +/// using a `Result` with `try!`, but this may change on day... +macro_rules! t { + ($e:expr) => (match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {}", stringify!($e), e), + }) +} + +mod cc; +mod channel; +mod check; +mod clean; +mod compile; +mod config; +mod dist; +mod doc; +mod flags; +mod native; +mod sanity; +mod step; +pub mod util; + +#[cfg(windows)] +mod job; + +#[cfg(not(windows))] +mod job { + pub unsafe fn setup() {} +} + +pub use config::Config; +pub use flags::Flags; + +/// A structure representing a Rust compiler. +/// +/// Each compiler has a `stage` that it is associated with and a `host` that +/// corresponds to the platform the compiler runs on. This structure is used as +/// a parameter to many methods below. +#[derive(Eq, PartialEq, Clone, Copy, Hash, Debug)] +pub struct Compiler<'a> { + stage: u32, + host: &'a str, +} + +/// Global configuration for the build system. +/// +/// This structure transitively contains all configuration for the build system. +/// All filesystem-encoded configuration is in `config`, all flags are in +/// `flags`, and then parsed or probed information is listed in the keys below. +/// +/// This structure is a parameter of almost all methods in the build system, +/// although most functions are implemented as free functions rather than +/// methods specifically on this structure itself (to make it easier to +/// organize). +pub struct Build { + // User-specified configuration via config.toml + config: Config, + + // User-specified configuration via CLI flags + flags: Flags, + + // Derived properties from the above two configurations + cargo: PathBuf, + rustc: PathBuf, + src: PathBuf, + out: PathBuf, + release: String, + unstable_features: bool, + ver_hash: Option, + short_ver_hash: Option, + ver_date: Option, + version: String, + package_vers: String, + bootstrap_key: String, + bootstrap_key_stage0: String, + + // Probed tools at runtime + gdb_version: Option, + lldb_version: Option, + lldb_python_dir: Option, + + // Runtime state filled in later on + cc: HashMap)>, + cxx: HashMap, + compiler_rt_built: RefCell>, +} + +/// The various "modes" of invoking Cargo. +/// +/// These entries currently correspond to the various output directories of the +/// build system, with each mod generating output in a different directory. +#[derive(Clone, Copy)] +pub enum Mode { + /// This cargo is going to build the standard library, placing output in the + /// "stageN-std" directory. + Libstd, + + /// This cargo is going to build libtest, placing output in the + /// "stageN-test" directory. + Libtest, + + /// This cargo is going to build librustc and compiler libraries, placing + /// output in the "stageN-rustc" directory. + Librustc, + + /// This cargo is going to some build tool, placing output in the + /// "stageN-tools" directory. + Tool, +} + +impl Build { + /// Creates a new set of build configuration from the `flags` on the command + /// line and the filesystem `config`. + /// + /// By default all build output will be placed in the current directory. + pub fn new(flags: Flags, config: Config) -> Build { + let cwd = t!(env::current_dir()); + let src = flags.src.clone().unwrap_or(cwd.clone()); + let out = cwd.join("build"); + + let stage0_root = out.join(&config.build).join("stage0/bin"); + let rustc = match config.rustc { + Some(ref s) => PathBuf::from(s), + None => stage0_root.join(exe("rustc", &config.build)), + }; + let cargo = match config.cargo { + Some(ref s) => PathBuf::from(s), + None => stage0_root.join(exe("cargo", &config.build)), + }; + + Build { + flags: flags, + config: config, + cargo: cargo, + rustc: rustc, + src: src, + out: out, + + release: String::new(), + unstable_features: false, + ver_hash: None, + short_ver_hash: None, + ver_date: None, + version: String::new(), + bootstrap_key: String::new(), + bootstrap_key_stage0: String::new(), + package_vers: String::new(), + cc: HashMap::new(), + cxx: HashMap::new(), + compiler_rt_built: RefCell::new(HashMap::new()), + gdb_version: None, + lldb_version: None, + lldb_python_dir: None, + } + } + + /// Executes the entire build, as configured by the flags and configuration. + pub fn build(&mut self) { + use step::Source::*; + + unsafe { + job::setup(); + } + + if self.flags.clean { + return clean::clean(self); + } + + self.verbose("finding compilers"); + cc::find(self); + self.verbose("running sanity check"); + sanity::check(self); + self.verbose("collecting channel variables"); + channel::collect(self); + self.verbose("updating submodules"); + self.update_submodules(); + + // The main loop of the build system. + // + // The `step::all` function returns a topographically sorted list of all + // steps that need to be executed as part of this build. Each step has a + // corresponding entry in `step.rs` and indicates some unit of work that + // needs to be done as part of the build. + // + // Almost all of these are simple one-liners that shell out to the + // corresponding functionality in the extra modules, where more + // documentation can be found. + for target in step::all(self) { + let doc_out = self.out.join(&target.target).join("doc"); + match target.src { + Llvm { _dummy } => { + native::llvm(self, target.target); + } + CompilerRt { _dummy } => { + native::compiler_rt(self, target.target); + } + TestHelpers { _dummy } => { + native::test_helpers(self, target.target); + } + Libstd { compiler } => { + compile::std(self, target.target, &compiler); + } + Libtest { compiler } => { + compile::test(self, target.target, &compiler); + } + Librustc { compiler } => { + compile::rustc(self, target.target, &compiler); + } + LibstdLink { compiler, host } => { + compile::std_link(self, target.target, &compiler, host); + } + LibtestLink { compiler, host } => { + compile::test_link(self, target.target, &compiler, host); + } + LibrustcLink { compiler, host } => { + compile::rustc_link(self, target.target, &compiler, host); + } + Rustc { stage: 0 } => { + // nothing to do... + } + Rustc { stage } => { + compile::assemble_rustc(self, stage, target.target); + } + ToolLinkchecker { stage } => { + compile::tool(self, stage, target.target, "linkchecker"); + } + ToolRustbook { stage } => { + compile::tool(self, stage, target.target, "rustbook"); + } + ToolErrorIndex { stage } => { + compile::tool(self, stage, target.target, + "error_index_generator"); + } + ToolCargoTest { stage } => { + compile::tool(self, stage, target.target, "cargotest"); + } + ToolTidy { stage } => { + compile::tool(self, stage, target.target, "tidy"); + } + ToolCompiletest { stage } => { + compile::tool(self, stage, target.target, "compiletest"); + } + DocBook { stage } => { + doc::rustbook(self, stage, target.target, "book", &doc_out); + } + DocNomicon { stage } => { + doc::rustbook(self, stage, target.target, "nomicon", + &doc_out); + } + DocStyle { stage } => { + doc::rustbook(self, stage, target.target, "style", + &doc_out); + } + DocStandalone { stage } => { + doc::standalone(self, stage, target.target, &doc_out); + } + DocStd { stage } => { + doc::std(self, stage, target.target, &doc_out); + } + DocTest { stage } => { + doc::test(self, stage, target.target, &doc_out); + } + DocRustc { stage } => { + doc::rustc(self, stage, target.target, &doc_out); + } + DocErrorIndex { stage } => { + doc::error_index(self, stage, target.target, &doc_out); + } + + CheckLinkcheck { stage } => { + check::linkcheck(self, stage, target.target); + } + CheckCargoTest { stage } => { + check::cargotest(self, stage, target.target); + } + CheckTidy { stage } => { + check::tidy(self, stage, target.target); + } + CheckRPass { compiler } => { + check::compiletest(self, &compiler, target.target, + "run-pass", "run-pass"); + } + CheckRPassFull { compiler } => { + check::compiletest(self, &compiler, target.target, + "run-pass", "run-pass-fulldeps"); + } + CheckCFail { compiler } => { + check::compiletest(self, &compiler, target.target, + "compile-fail", "compile-fail"); + } + CheckCFailFull { compiler } => { + check::compiletest(self, &compiler, target.target, + "compile-fail", "compile-fail-fulldeps") + } + CheckPFail { compiler } => { + check::compiletest(self, &compiler, target.target, + "parse-fail", "parse-fail"); + } + CheckRFail { compiler } => { + check::compiletest(self, &compiler, target.target, + "run-fail", "run-fail"); + } + CheckRFailFull { compiler } => { + check::compiletest(self, &compiler, target.target, + "run-fail", "run-fail-fulldeps"); + } + CheckPretty { compiler } => { + check::compiletest(self, &compiler, target.target, + "pretty", "pretty"); + } + CheckPrettyRPass { compiler } => { + check::compiletest(self, &compiler, target.target, + "pretty", "run-pass"); + } + CheckPrettyRPassFull { compiler } => { + check::compiletest(self, &compiler, target.target, + "pretty", "run-pass-fulldeps"); + } + CheckPrettyRFail { compiler } => { + check::compiletest(self, &compiler, target.target, + "pretty", "run-fail"); + } + CheckPrettyRFailFull { compiler } => { + check::compiletest(self, &compiler, target.target, + "pretty", "run-fail-fulldeps"); + } + CheckPrettyRPassValgrind { compiler } => { + check::compiletest(self, &compiler, target.target, + "pretty", "run-pass-valgrind"); + } + CheckCodegen { compiler } => { + check::compiletest(self, &compiler, target.target, + "codegen", "codegen"); + } + CheckCodegenUnits { compiler } => { + check::compiletest(self, &compiler, target.target, + "codegen-units", "codegen-units"); + } + CheckIncremental { compiler } => { + check::compiletest(self, &compiler, target.target, + "incremental", "incremental"); + } + CheckUi { compiler } => { + check::compiletest(self, &compiler, target.target, + "ui", "ui"); + } + CheckDebuginfo { compiler } => { + if target.target.contains("msvc") { + // nothing to do + } else if target.target.contains("apple") { + check::compiletest(self, &compiler, target.target, + "debuginfo-lldb", "debuginfo"); + } else { + check::compiletest(self, &compiler, target.target, + "debuginfo-gdb", "debuginfo"); + } + } + CheckRustdoc { compiler } => { + check::compiletest(self, &compiler, target.target, + "rustdoc", "rustdoc"); + } + CheckRPassValgrind { compiler } => { + check::compiletest(self, &compiler, target.target, + "run-pass-valgrind", "run-pass-valgrind"); + } + CheckDocs { compiler } => { + check::docs(self, &compiler); + } + CheckErrorIndex { compiler } => { + check::error_index(self, &compiler); + } + CheckRMake { compiler } => { + check::compiletest(self, &compiler, target.target, + "run-make", "run-make") + } + CheckCrateStd { compiler } => { + check::krate(self, &compiler, target.target, Mode::Libstd) + } + CheckCrateTest { compiler } => { + check::krate(self, &compiler, target.target, Mode::Libtest) + } + CheckCrateRustc { compiler } => { + check::krate(self, &compiler, target.target, Mode::Librustc) + } + + DistDocs { stage } => dist::docs(self, stage, target.target), + DistMingw { _dummy } => dist::mingw(self, target.target), + DistRustc { stage } => dist::rustc(self, stage, target.target), + DistStd { compiler } => dist::std(self, &compiler, target.target), + + DebuggerScripts { stage } => { + let compiler = Compiler::new(stage, target.target); + dist::debugger_scripts(self, + &self.sysroot(&compiler), + target.target); + } + + AndroidCopyLibs { compiler } => { + check::android_copy_libs(self, &compiler, target.target); + } + + // pseudo-steps + Dist { .. } | + Doc { .. } | + CheckTarget { .. } | + Check { .. } => {} + } + } + } + + /// Updates all git submodules that we have. + /// + /// This will detect if any submodules are out of date an run the necessary + /// commands to sync them all with upstream. + fn update_submodules(&self) { + if !self.config.submodules { + return + } + if fs::metadata(self.src.join(".git")).is_err() { + return + } + let git_submodule = || { + let mut cmd = Command::new("git"); + cmd.current_dir(&self.src).arg("submodule"); + return cmd + }; + + // FIXME: this takes a seriously long time to execute on Windows and a + // nontrivial amount of time on Unix, we should have a better way + // of detecting whether we need to run all the submodule commands + // below. + let out = output(git_submodule().arg("status")); + if !out.lines().any(|l| l.starts_with("+") || l.starts_with("-")) { + return + } + + self.run(git_submodule().arg("sync")); + self.run(git_submodule().arg("init")); + self.run(git_submodule().arg("update")); + self.run(git_submodule().arg("update").arg("--recursive")); + self.run(git_submodule().arg("status").arg("--recursive")); + self.run(git_submodule().arg("foreach").arg("--recursive") + .arg("git").arg("clean").arg("-fdx")); + self.run(git_submodule().arg("foreach").arg("--recursive") + .arg("git").arg("checkout").arg(".")); + } + + /// Clear out `dir` if `input` is newer. + /// + /// After this executes, it will also ensure that `dir` exists. + fn clear_if_dirty(&self, dir: &Path, input: &Path) { + let stamp = dir.join(".stamp"); + if mtime(&stamp) < mtime(input) { + self.verbose(&format!("Dirty - {}", dir.display())); + let _ = fs::remove_dir_all(dir); + } + t!(fs::create_dir_all(dir)); + t!(File::create(stamp)); + } + + /// Prepares an invocation of `cargo` to be run. + /// + /// This will create a `Command` that represents a pending execution of + /// Cargo. This cargo will be configured to use `compiler` as the actual + /// rustc compiler, its output will be scoped by `mode`'s output directory, + /// it will pass the `--target` flag for the specified `target`, and will be + /// executing the Cargo command `cmd`. + fn cargo(&self, + compiler: &Compiler, + mode: Mode, + target: &str, + cmd: &str) -> Command { + let mut cargo = Command::new(&self.cargo); + let out_dir = self.stage_out(compiler, mode); + cargo.env("CARGO_TARGET_DIR", out_dir) + .arg(cmd) + .arg("-j").arg(self.jobs().to_string()) + .arg("--target").arg(target); + + let stage; + if compiler.stage == 0 && self.config.local_rebuild { + // Assume the local-rebuild rustc already has stage1 features. + stage = 1; + } else { + stage = compiler.stage; + } + + // Customize the compiler we're running. Specify the compiler to cargo + // as our shim and then pass it some various options used to configure + // how the actual compiler itself is called. + // + // These variables are primarily all read by + // src/bootstrap/{rustc,rustdoc.rs} + cargo.env("RUSTC", self.out.join("bootstrap/debug/rustc")) + .env("RUSTC_REAL", self.compiler_path(compiler)) + .env("RUSTC_STAGE", stage.to_string()) + .env("RUSTC_DEBUGINFO", self.config.rust_debuginfo.to_string()) + .env("RUSTC_CODEGEN_UNITS", + self.config.rust_codegen_units.to_string()) + .env("RUSTC_DEBUG_ASSERTIONS", + self.config.rust_debug_assertions.to_string()) + .env("RUSTC_SNAPSHOT", &self.rustc) + .env("RUSTC_SYSROOT", self.sysroot(compiler)) + .env("RUSTC_LIBDIR", self.rustc_libdir(compiler)) + .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_snapshot_libdir()) + .env("RUSTC_RPATH", self.config.rust_rpath.to_string()) + .env("RUSTDOC", self.out.join("bootstrap/debug/rustdoc")) + .env("RUSTDOC_REAL", self.rustdoc(compiler)) + .env("RUSTC_FLAGS", self.rustc_flags(target).join(" ")); + + self.add_bootstrap_key(compiler, &mut cargo); + + // Specify some various options for build scripts used throughout + // the build. + // + // FIXME: the guard against msvc shouldn't need to be here + if !target.contains("msvc") { + cargo.env(format!("CC_{}", target), self.cc(target)) + .env(format!("AR_{}", target), self.ar(target).unwrap()) // only msvc is None + .env(format!("CFLAGS_{}", target), self.cflags(target).join(" ")); + } + + // If we're building for OSX, inform the compiler and the linker that + // we want to build a compiler runnable on 10.7 + if target.contains("apple-darwin") { + cargo.env("MACOSX_DEPLOYMENT_TARGET", "10.7"); + } + + // Environment variables *required* needed throughout the build + // + // FIXME: should update code to not require this env var + cargo.env("CFG_COMPILER_HOST_TRIPLE", target); + + if self.config.verbose || self.flags.verbose { + cargo.arg("-v"); + } + if self.config.rust_optimize { + cargo.arg("--release"); + } + return cargo + } + + /// Get a path to the compiler specified. + fn compiler_path(&self, compiler: &Compiler) -> PathBuf { + if compiler.is_snapshot(self) { + self.rustc.clone() + } else { + self.sysroot(compiler).join("bin").join(exe("rustc", compiler.host)) + } + } + + /// Get the specified tool built by the specified compiler + fn tool(&self, compiler: &Compiler, tool: &str) -> PathBuf { + self.cargo_out(compiler, Mode::Tool, compiler.host) + .join(exe(tool, compiler.host)) + } + + /// Get the `rustdoc` executable next to the specified compiler + fn rustdoc(&self, compiler: &Compiler) -> PathBuf { + let mut rustdoc = self.compiler_path(compiler); + rustdoc.pop(); + rustdoc.push(exe("rustdoc", compiler.host)); + return rustdoc + } + + /// Get a `Command` which is ready to run `tool` in `stage` built for + /// `host`. + fn tool_cmd(&self, compiler: &Compiler, tool: &str) -> Command { + let mut cmd = Command::new(self.tool(&compiler, tool)); + let host = compiler.host; + let paths = vec![ + self.cargo_out(compiler, Mode::Libstd, host).join("deps"), + self.cargo_out(compiler, Mode::Libtest, host).join("deps"), + self.cargo_out(compiler, Mode::Librustc, host).join("deps"), + self.cargo_out(compiler, Mode::Tool, host).join("deps"), + ]; + add_lib_path(paths, &mut cmd); + return cmd + } + + /// Get the space-separated set of activated features for the standard + /// library. + fn std_features(&self) -> String { + let mut features = String::new(); + if self.config.debug_jemalloc { + features.push_str(" debug-jemalloc"); + } + if self.config.use_jemalloc { + features.push_str(" jemalloc"); + } + return features + } + + /// Get the space-separated set of activated features for the compiler. + fn rustc_features(&self) -> String { + let mut features = String::new(); + if self.config.use_jemalloc { + features.push_str(" jemalloc"); + } + return features + } + + /// Component directory that Cargo will produce output into (e.g. + /// release/debug) + fn cargo_dir(&self) -> &'static str { + if self.config.rust_optimize {"release"} else {"debug"} + } + + /// Returns the sysroot for the `compiler` specified that *this build system + /// generates*. + /// + /// That is, the sysroot for the stage0 compiler is not what the compiler + /// thinks it is by default, but it's the same as the default for stages + /// 1-3. + fn sysroot(&self, compiler: &Compiler) -> PathBuf { + if compiler.stage == 0 { + self.out.join(compiler.host).join("stage0-sysroot") + } else { + self.out.join(compiler.host).join(format!("stage{}", compiler.stage)) + } + } + + /// Returns the libdir where the standard library and other artifacts are + /// found for a compiler's sysroot. + fn sysroot_libdir(&self, compiler: &Compiler, target: &str) -> PathBuf { + self.sysroot(compiler).join("lib").join("rustlib") + .join(target).join("lib") + } + + /// Returns the root directory for all output generated in a particular + /// stage when running with a particular host compiler. + /// + /// The mode indicates what the root directory is for. + fn stage_out(&self, compiler: &Compiler, mode: Mode) -> PathBuf { + let suffix = match mode { + Mode::Libstd => "-std", + Mode::Libtest => "-test", + Mode::Tool => "-tools", + Mode::Librustc => "-rustc", + }; + self.out.join(compiler.host) + .join(format!("stage{}{}", compiler.stage, suffix)) + } + + /// Returns the root output directory for all Cargo output in a given stage, + /// running a particular comipler, wehther or not we're building the + /// standard library, and targeting the specified architecture. + fn cargo_out(&self, + compiler: &Compiler, + mode: Mode, + target: &str) -> PathBuf { + self.stage_out(compiler, mode).join(target).join(self.cargo_dir()) + } + + /// Root output directory for LLVM compiled for `target` + /// + /// Note that if LLVM is configured externally then the directory returned + /// will likely be empty. + fn llvm_out(&self, target: &str) -> PathBuf { + self.out.join(target).join("llvm") + } + + /// Returns the path to `llvm-config` for the specified target. + /// + /// If a custom `llvm-config` was specified for target then that's returned + /// instead. + fn llvm_config(&self, target: &str) -> PathBuf { + let target_config = self.config.target_config.get(target); + if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { + s.clone() + } else { + self.llvm_out(&self.config.build).join("bin") + .join(exe("llvm-config", target)) + } + } + + /// Returns the path to `FileCheck` binary for the specified target + fn llvm_filecheck(&self, target: &str) -> PathBuf { + let target_config = self.config.target_config.get(target); + if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { + s.parent().unwrap().join(exe("FileCheck", target)) + } else { + let base = self.llvm_out(&self.config.build).join("build"); + let exe = exe("FileCheck", target); + if self.config.build.contains("msvc") { + base.join("Release/bin").join(exe) + } else { + base.join("bin").join(exe) + } + } + } + + /// Root output directory for compiler-rt compiled for `target` + fn compiler_rt_out(&self, target: &str) -> PathBuf { + self.out.join(target).join("compiler-rt") + } + + /// Root output directory for rust_test_helpers library compiled for + /// `target` + fn test_helpers_out(&self, target: &str) -> PathBuf { + self.out.join(target).join("rust-test-helpers") + } + + /// Adds the compiler's directory of dynamic libraries to `cmd`'s dynamic + /// library lookup path. + fn add_rustc_lib_path(&self, compiler: &Compiler, cmd: &mut Command) { + // Windows doesn't need dylib path munging because the dlls for the + // compiler live next to the compiler and the system will find them + // automatically. + if cfg!(windows) { + return + } + + add_lib_path(vec![self.rustc_libdir(compiler)], cmd); + } + + /// Adds the compiler's bootstrap key to the environment of `cmd`. + fn add_bootstrap_key(&self, compiler: &Compiler, cmd: &mut Command) { + // In stage0 we're using a previously released stable compiler, so we + // use the stage0 bootstrap key. Otherwise we use our own build's + // bootstrap key. + let bootstrap_key = if compiler.is_snapshot(self) && !self.config.local_rebuild { + &self.bootstrap_key_stage0 + } else { + &self.bootstrap_key + }; + cmd.env("RUSTC_BOOTSTRAP_KEY", bootstrap_key); + } + + /// Returns the compiler's libdir where it stores the dynamic libraries that + /// it itself links against. + /// + /// For example this returns `/lib` on Unix and `/bin` on + /// Windows. + fn rustc_libdir(&self, compiler: &Compiler) -> PathBuf { + if compiler.is_snapshot(self) { + self.rustc_snapshot_libdir() + } else { + self.sysroot(compiler).join(libdir(compiler.host)) + } + } + + /// Returns the libdir of the snapshot compiler. + fn rustc_snapshot_libdir(&self) -> PathBuf { + self.rustc.parent().unwrap().parent().unwrap() + .join(libdir(&self.config.build)) + } + + /// Runs a command, printing out nice contextual information if it fails. + fn run(&self, cmd: &mut Command) { + self.verbose(&format!("running: {:?}", cmd)); + run_silent(cmd) + } + + /// Prints a message if this build is configured in verbose mode. + fn verbose(&self, msg: &str) { + if self.flags.verbose || self.config.verbose { + println!("{}", msg); + } + } + + /// Returns the number of parallel jobs that have been configured for this + /// build. + fn jobs(&self) -> u32 { + self.flags.jobs.unwrap_or(num_cpus::get() as u32) + } + + /// Returns the path to the C compiler for the target specified. + fn cc(&self, target: &str) -> &Path { + self.cc[target].0.path() + } + + /// Returns a list of flags to pass to the C compiler for the target + /// specified. + fn cflags(&self, target: &str) -> Vec { + // Filter out -O and /O (the optimization flags) that we picked up from + // gcc-rs because the build scripts will determine that for themselves. + let mut base = self.cc[target].0.args().iter() + .map(|s| s.to_string_lossy().into_owned()) + .filter(|s| !s.starts_with("-O") && !s.starts_with("/O")) + .collect::>(); + + // If we're compiling on OSX then we add a few unconditional flags + // indicating that we want libc++ (more filled out than libstdc++) and + // we want to compile for 10.7. This way we can ensure that + // LLVM/jemalloc/etc are all properly compiled. + if target.contains("apple-darwin") { + base.push("-stdlib=libc++".into()); + base.push("-mmacosx-version-min=10.7".into()); + } + return base + } + + /// Returns the path to the `ar` archive utility for the target specified. + fn ar(&self, target: &str) -> Option<&Path> { + self.cc[target].1.as_ref().map(|p| &**p) + } + + /// Returns the path to the C++ compiler for the target specified, may panic + /// if no C++ compiler was configured for the target. + fn cxx(&self, target: &str) -> &Path { + self.cxx[target].path() + } + + /// Returns flags to pass to the compiler to generate code for `target`. + fn rustc_flags(&self, target: &str) -> Vec { + // New flags should be added here with great caution! + // + // It's quite unfortunate to **require** flags to generate code for a + // target, so it should only be passed here if absolutely necessary! + // Most default configuration should be done through target specs rather + // than an entry here. + + let mut base = Vec::new(); + if target != self.config.build && !target.contains("msvc") { + base.push(format!("-Clinker={}", self.cc(target).display())); + } + return base } } -/// Parses the `dylib_path_var()` environment variable, returning a list of -/// paths that are members of this lookup path. -pub fn dylib_path() -> Vec { - env::split_paths(&env::var_os(dylib_path_var()).unwrap_or(OsString::new())) - .collect() +impl<'a> Compiler<'a> { + /// Creates a new complier for the specified stage/host + fn new(stage: u32, host: &'a str) -> Compiler<'a> { + Compiler { stage: stage, host: host } + } + + /// Returns whether this is a snapshot compiler for `build`'s configuration + fn is_snapshot(&self, build: &Build) -> bool { + self.stage == 0 && self.host == build.config.build + } } diff --git a/src/bootstrap/build/native.rs b/src/bootstrap/native.rs similarity index 99% rename from src/bootstrap/build/native.rs rename to src/bootstrap/native.rs index f6030cfd090d2..83e9393fbaef7 100644 --- a/src/bootstrap/build/native.rs +++ b/src/bootstrap/native.rs @@ -26,8 +26,8 @@ use build_helper::output; use cmake; use gcc; -use build::Build; -use build::util::{staticlib, up_to_date}; +use Build; +use util::{staticlib, up_to_date}; /// Compile LLVM for `target`. pub fn llvm(build: &Build, target: &str) { diff --git a/src/bootstrap/build/sanity.rs b/src/bootstrap/sanity.rs similarity index 99% rename from src/bootstrap/build/sanity.rs rename to src/bootstrap/sanity.rs index 5eced00e13973..7c0f09c322f24 100644 --- a/src/bootstrap/build/sanity.rs +++ b/src/bootstrap/sanity.rs @@ -26,7 +26,7 @@ use std::process::Command; use build_helper::output; -use build::Build; +use Build; pub fn check(build: &mut Build) { let mut checked = HashSet::new(); diff --git a/src/bootstrap/build/step.rs b/src/bootstrap/step.rs similarity index 99% rename from src/bootstrap/build/step.rs rename to src/bootstrap/step.rs index 7cbbd6740a265..4b3be04b57c57 100644 --- a/src/bootstrap/build/step.rs +++ b/src/bootstrap/step.rs @@ -22,7 +22,7 @@ use std::collections::HashSet; -use build::{Build, Compiler}; +use {Build, Compiler}; #[derive(Hash, Eq, PartialEq, Clone, Debug)] pub struct Step<'a> { diff --git a/src/bootstrap/build/util.rs b/src/bootstrap/util.rs similarity index 86% rename from src/bootstrap/build/util.rs rename to src/bootstrap/util.rs index 36ce064914227..3ef7f8cab2d1b 100644 --- a/src/bootstrap/build/util.rs +++ b/src/bootstrap/util.rs @@ -14,11 +14,11 @@ //! not a lot of interesting happenings here unfortunately. use std::env; -use std::path::{Path, PathBuf}; +use std::ffi::OsString; use std::fs; +use std::path::{Path, PathBuf}; use std::process::Command; -use bootstrap::{dylib_path, dylib_path_var}; use filetime::FileTime; /// Returns the `name` as the filename of a static library for `target`. @@ -121,3 +121,22 @@ fn dir_up_to_date(src: &Path, threshold: &FileTime) -> bool { } }) } + +/// Returns the environment variable which the dynamic library lookup path +/// resides in for this platform. +pub fn dylib_path_var() -> &'static str { + if cfg!(target_os = "windows") { + "PATH" + } else if cfg!(target_os = "macos") { + "DYLD_LIBRARY_PATH" + } else { + "LD_LIBRARY_PATH" + } +} + +/// Parses the `dylib_path_var()` environment variable, returning a list of +/// paths that are members of this lookup path. +pub fn dylib_path() -> Vec { + env::split_paths(&env::var_os(dylib_path_var()).unwrap_or(OsString::new())) + .collect() +}