Auto merge of #38667 - alexcrichton:stage0-tools, r=brson

rustbuild: Compile all support tools in stage0

This commit changes all tools and such to get compiled in stage0, not in
later stages. The purpose of this commit is to cut down dependencies on later
stages for future modifications to the build system. Notably we're going to be
adding builders that produce a full suite of cross-compiled artifacts for a
particular host, and that shouldn't compile the `x86_64-unknown-linux-gnu`
compiler more than once. Currently dependencies on, for example, the error index
end up compiling the `x86_64-unknown-linux-gnu` compiler more than necessary.

As a result here we move many dependencies on these tools to being produced by a
stage0 compiler, not a stage1+ compiler. None of these tools actually need to be
staged at all, so they'll exhibit consistent behavior across the stages.

Author: bors

src/Cargo.lock

@@ -89,7 +89,9 @@ fn main() {
         // When we build Rust dylibs they're all intended for intermediate
         // usage, so make sure we pass the -Cprefer-dynamic flag instead of
         // linking all deps statically into the dylib.
-        cmd.arg("-Cprefer-dynamic");
+        if env::var_os("RUSTC_NO_PREFER_DYNAMIC").is_none() {
+            cmd.arg("-Cprefer-dynamic");
+        }
 
         // Help the libc crate compile by assisting it in finding the MUSL
         // native libraries.

src/bootstrap/bin/rustc.rs

@@ -62,9 +62,9 @@ impl fmt::Display for TestKind {
 ///
 /// This tool in `src/tools` will verify the validity of all our links in the
 /// documentation to ensure we don't have a bunch of dead ones.
-pub fn linkcheck(build: &Build, stage: u32, host: &str) {
-    println!("Linkcheck stage{} ({})", stage, host);
-    let compiler = Compiler::new(stage, host);
+pub fn linkcheck(build: &Build, host: &str) {
+    println!("Linkcheck ({})", host);
+    let compiler = Compiler::new(0, host);
 
     let _time = util::timeit();
     build.run(build.tool_cmd(&compiler, "linkchecker")
@@ -93,20 +93,21 @@ pub fn cargotest(build: &Build, stage: u32, host: &str) {
     t!(fs::create_dir_all(&out_dir));
 
     let _time = util::timeit();
-    build.run(build.tool_cmd(compiler, "cargotest")
-                   .env("PATH", newpath)
-                   .arg(&build.cargo)
-                   .arg(&out_dir));
+    let mut cmd = Command::new(build.tool(&Compiler::new(0, host), "cargotest"));
+    build.prepare_tool_cmd(compiler, &mut cmd);
+    build.run(cmd.env("PATH", newpath)
+                 .arg(&build.cargo)
+                 .arg(&out_dir));
 }
 
 /// Runs the `tidy` tool as compiled in `stage` by the `host` compiler.
 ///
 /// This tool in `src/tools` checks up on various bits and pieces of style and
 /// otherwise just implements a few lint-like checks that are specific to the
 /// compiler itself.
-pub fn tidy(build: &Build, stage: u32, host: &str) {
-    println!("tidy check stage{} ({})", stage, host);
-    let compiler = Compiler::new(stage, host);
+pub fn tidy(build: &Build, host: &str) {
+    println!("tidy check ({})", host);
+    let compiler = Compiler::new(0, host);
     build.run(build.tool_cmd(&compiler, "tidy")
                    .arg(build.src.join("src")));
 }
@@ -127,7 +128,9 @@ pub fn compiletest(build: &Build,
                    suite: &str) {
     println!("Check compiletest suite={} mode={} ({} -> {})",
              suite, mode, compiler.host, target);
-    let mut cmd = build.tool_cmd(compiler, "compiletest");
+    let mut cmd = Command::new(build.tool(&Compiler::new(0, compiler.host),
+                                          "compiletest"));
+    build.prepare_tool_cmd(compiler, &mut cmd);
 
     // compiletest currently has... a lot of arguments, so let's just pass all
     // of them!
@@ -287,7 +290,8 @@ pub fn error_index(build: &Build, compiler: &Compiler) {
     let output = dir.join("error-index.md");
 
     let _time = util::timeit();
-    build.run(build.tool_cmd(compiler, "error_index_generator")
+    build.run(build.tool_cmd(&Compiler::new(0, compiler.host),
+                             "error_index_generator")
                    .arg("markdown")
                    .arg(&output)
                    .env("CFG_BUILD", &build.config.build));

src/bootstrap/check.rs

@@ -379,6 +379,11 @@ pub fn tool(build: &Build, stage: u32, host: &str, tool: &str) {
     let mut cargo = build.cargo(&compiler, Mode::Tool, host, "build");
     cargo.arg("--manifest-path")
          .arg(build.src.join(format!("src/tools/{}/Cargo.toml", tool)));
+
+    // We don't want to build tools dynamically as they'll be running across
+    // stages and such and it's just easier if they're not dynamically linked.
+    cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
+
     build.run(&mut cargo);
 }
 

src/bootstrap/compile.rs

@@ -29,19 +29,19 @@ use util::{up_to_date, cp_r};
 ///
 /// This will not actually generate any documentation if the documentation has
 /// already been generated.
-pub fn rustbook(build: &Build, stage: u32, target: &str, name: &str) {
+pub fn rustbook(build: &Build, target: &str, name: &str) {
     let out = build.doc_out(target);
     t!(fs::create_dir_all(&out));
 
     let out = out.join(name);
-    let compiler = Compiler::new(stage, &build.config.build);
+    let compiler = Compiler::new(0, &build.config.build);
     let src = build.src.join("src/doc").join(name);
     let index = out.join("index.html");
     let rustbook = build.tool(&compiler, "rustbook");
     if up_to_date(&src, &index) && up_to_date(&rustbook, &index) {
         return
     }
-    println!("Rustbook stage{} ({}) - {}", stage, target, name);
+    println!("Rustbook ({}) - {}", target, name);
     let _ = fs::remove_dir_all(&out);
     build.run(build.tool_cmd(&compiler, "rustbook")
                    .arg("build")
@@ -214,11 +214,11 @@ pub fn rustc(build: &Build, stage: u32, target: &str) {
 
 /// Generates the HTML rendered error-index by running the
 /// `error_index_generator` tool.
-pub fn error_index(build: &Build, stage: u32, target: &str) {
-    println!("Documenting stage{} error index ({})", stage, target);
+pub fn error_index(build: &Build, target: &str) {
+    println!("Documenting error index ({})", target);
     let out = build.doc_out(target);
     t!(fs::create_dir_all(&out));
-    let compiler = Compiler::new(stage, &build.config.build);
+    let compiler = Compiler::new(0, &build.config.build);
     let mut index = build.tool_cmd(&compiler, "error_index_generator");
     index.arg("html");
     index.arg(out.join("error-index.html"));

src/bootstrap/doc.rs

@@ -570,6 +570,15 @@ impl Build {
     /// `host`.
     fn tool_cmd(&self, compiler: &Compiler, tool: &str) -> Command {
         let mut cmd = Command::new(self.tool(&compiler, tool));
+        self.prepare_tool_cmd(compiler, &mut cmd);
+        return cmd
+    }
+
+    /// Prepares the `cmd` provided to be able to run the `compiler` provided.
+    ///
+    /// Notably this munges the dynamic library lookup path to point to the
+    /// right location to run `compiler`.
+    fn prepare_tool_cmd(&self, compiler: &Compiler, cmd: &mut Command) {
         let host = compiler.host;
         let mut paths = vec![
             self.sysroot_libdir(compiler, compiler.host),
@@ -593,8 +602,7 @@ impl Build {
                 }
             }
         }
-        add_lib_path(paths, &mut cmd);
-        return cmd
+        add_lib_path(paths, cmd);
     }
 
     /// Get the space-separated set of activated features for the standard

src/bootstrap/lib.rs

@@ -69,7 +69,7 @@ distcheck:
 install:
 	$(Q)$(BOOTSTRAP) dist --install $(BOOTSTRAP_ARGS)
 tidy:
-	$(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS) --stage 0
+	$(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS)
 
 check-stage2-T-arm-linux-androideabi-H-x86_64-unknown-linux-gnu:
 	$(Q)$(BOOTSTRAP) test --target arm-linux-androideabi

src/bootstrap/mk/Makefile.in

@@ -293,7 +293,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
         let mut suite = |name, path, mode, dir| {
             rules.test(name, path)
                  .dep(|s| s.name("libtest"))
-                 .dep(|s| s.name("tool-compiletest").target(s.host))
+                 .dep(|s| s.name("tool-compiletest").target(s.host).stage(0))
                  .dep(|s| s.name("test-helpers"))
                  .dep(|s| s.name("android-copy-libs"))
                  .default(mode != "pretty") // pretty tests don't run everywhere
@@ -325,7 +325,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
         rules.test("check-debuginfo", "src/test/debuginfo")
              .default(true)
              .dep(|s| s.name("libtest"))
-             .dep(|s| s.name("tool-compiletest").target(s.host))
+             .dep(|s| s.name("tool-compiletest").target(s.host).stage(0))
              .dep(|s| s.name("test-helpers"))
              .dep(|s| s.name("debugger-scripts"))
              .run(move |s| check::compiletest(build, &s.compiler(), s.target,
@@ -334,7 +334,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
         rules.test("check-debuginfo", "src/test/debuginfo")
              .default(true)
              .dep(|s| s.name("libtest"))
-             .dep(|s| s.name("tool-compiletest").target(s.host))
+             .dep(|s| s.name("tool-compiletest").target(s.host).stage(0))
              .dep(|s| s.name("test-helpers"))
              .dep(|s| s.name("debugger-scripts"))
              .dep(|s| s.name("android-copy-libs"))
@@ -351,7 +351,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
             rules.test(name, path)
                  .dep(|s| s.name("librustc"))
                  .dep(|s| s.name("test-helpers"))
-                 .dep(|s| s.name("tool-compiletest").target(s.host))
+                 .dep(|s| s.name("tool-compiletest").target(s.host).stage(0))
                  .default(mode != "pretty")
                  .host(true)
                  .run(move |s| {
@@ -441,24 +441,24 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
                                     Mode::Librustc, TestKind::Test, None));
 
     rules.test("check-linkchecker", "src/tools/linkchecker")
-         .dep(|s| s.name("tool-linkchecker"))
+         .dep(|s| s.name("tool-linkchecker").stage(0))
          .dep(|s| s.name("default:doc"))
          .default(true)
          .host(true)
-         .run(move |s| check::linkcheck(build, s.stage, s.target));
+         .run(move |s| check::linkcheck(build, s.target));
     rules.test("check-cargotest", "src/tools/cargotest")
-         .dep(|s| s.name("tool-cargotest"))
+         .dep(|s| s.name("tool-cargotest").stage(0))
          .dep(|s| s.name("librustc"))
          .host(true)
          .run(move |s| check::cargotest(build, s.stage, s.target));
     rules.test("check-tidy", "src/tools/tidy")
          .dep(|s| s.name("tool-tidy").stage(0))
          .default(true)
          .host(true)
-         .run(move |s| check::tidy(build, 0, s.target));
+         .run(move |s| check::tidy(build, s.target));
     rules.test("check-error-index", "src/tools/error_index_generator")
          .dep(|s| s.name("libstd"))
-         .dep(|s| s.name("tool-error-index").host(s.host))
+         .dep(|s| s.name("tool-error-index").host(s.host).stage(0))
          .default(true)
          .host(true)
          .run(move |s| check::error_index(build, &s.compiler()));
@@ -504,23 +504,23 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     // ========================================================================
     // Documentation targets
     rules.doc("doc-book", "src/doc/book")
-         .dep(move |s| s.name("tool-rustbook").target(&build.config.build))
+         .dep(move |s| s.name("tool-rustbook").target(&build.config.build).stage(0))
          .default(build.config.docs)
-         .run(move |s| doc::rustbook(build, s.stage, s.target, "book"));
+         .run(move |s| doc::rustbook(build, s.target, "book"));
     rules.doc("doc-nomicon", "src/doc/nomicon")
-         .dep(move |s| s.name("tool-rustbook").target(&build.config.build))
+         .dep(move |s| s.name("tool-rustbook").target(&build.config.build).stage(0))
          .default(build.config.docs)
-         .run(move |s| doc::rustbook(build, s.stage, s.target, "nomicon"));
+         .run(move |s| doc::rustbook(build, s.target, "nomicon"));
     rules.doc("doc-standalone", "src/doc")
          .dep(move |s| s.name("rustc").host(&build.config.build).target(&build.config.build))
          .default(build.config.docs)
          .run(move |s| doc::standalone(build, s.stage, s.target));
     rules.doc("doc-error-index", "src/tools/error_index_generator")
-         .dep(move |s| s.name("tool-error-index").target(&build.config.build))
-         .dep(move |s| s.name("librustc-link"))
+         .dep(move |s| s.name("tool-error-index").target(&build.config.build).stage(0))
+         .dep(move |s| s.name("librustc-link").stage(0))
          .default(build.config.docs)
          .host(true)
-         .run(move |s| doc::error_index(build, s.stage, s.target));
+         .run(move |s| doc::error_index(build, s.target));
     for (krate, path, default) in krates("std_shim") {
         rules.doc(&krate.doc_step, path)
              .dep(|s| s.name("libstd-link"))

src/bootstrap/step.rs

@@ -7,4 +7,4 @@ build = "build.rs"
 [dependencies]
 log = "0.3"
 env_logger = { version = "0.3.5", default-features = false }
-serialize = { path = "../../libserialize" }
+rustc-serialize = "0.3"

src/tools/compiletest/Cargo.toml

@@ -21,6 +21,10 @@
 extern crate libc;
 extern crate test;
 extern crate getopts;
+
+#[cfg(cargobuild)]
+extern crate rustc_serialize;
+#[cfg(not(cargobuild))]
 extern crate serialize as rustc_serialize;
 
 #[macro_use]