Merge pull request #1302 from bjorn3/build_system_rework4

Allow specifying where build artifacts should be written to

Author: bjorn3

.github/workflows/main.yml

@@ -63,7 +63,7 @@ jobs:
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
-        path: target
+        path: build/cg_clif
         key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
 
     - name: Install MinGW toolchain and wine
@@ -164,7 +164,7 @@ jobs:
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
-        path: target
+        path: build/cg_clif
         key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
 
     - name: Set MinGW as the default toolchain

.github/workflows/rustc.yml

@@ -27,7 +27,7 @@ jobs:
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
-        path: target
+        path: build/cg_clif
         key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
 
     - name: Prepare dependencies
@@ -65,7 +65,7 @@ jobs:
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
-        path: target
+        path: build/cg_clif
         key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
 
     - name: Prepare dependencies

build_system/abi_cafe.rs

@@ -2,19 +2,20 @@ use std::path::Path;
 
 use super::build_sysroot;
 use super::config;
+use super::path::Dirs;
 use super::prepare::GitRepo;
 use super::utils::{spawn_and_wait, CargoProject, Compiler};
 use super::SysrootKind;
 
 pub(crate) static ABI_CAFE_REPO: GitRepo =
     GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe");
 
-static ABI_CAFE: CargoProject = CargoProject::git(&ABI_CAFE_REPO, ".");
+static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe");
 
 pub(crate) fn run(
     channel: &str,
     sysroot_kind: SysrootKind,
-    dist_dir: &Path,
+    dirs: &Dirs,
     cg_clif_dylib: &Path,
     host_triple: &str,
     target_triple: &str,
@@ -31,9 +32,9 @@ pub(crate) fn run(
 
     eprintln!("Building sysroot for abi-cafe");
     build_sysroot::build_sysroot(
+        dirs,
         channel,
         sysroot_kind,
-        dist_dir,
         cg_clif_dylib,
         host_triple,
         target_triple,
@@ -43,13 +44,13 @@ pub(crate) fn run(
 
     let pairs = ["rustc_calls_cgclif", "cgclif_calls_rustc", "cgclif_calls_cc", "cc_calls_cgclif"];
 
-    let mut cmd = ABI_CAFE.run(&Compiler::host());
+    let mut cmd = ABI_CAFE.run(&Compiler::host(), dirs);
     cmd.arg("--");
     cmd.arg("--pairs");
     cmd.args(pairs);
     cmd.arg("--add-rustc-codegen-backend");
     cmd.arg(format!("cgclif:{}", cg_clif_dylib.display()));
-    cmd.current_dir(ABI_CAFE.source_dir());
+    cmd.current_dir(ABI_CAFE.source_dir(dirs));
 
     spawn_and_wait(cmd);
 }

build_system/build_backend.rs

@@ -1,17 +1,19 @@
 use std::env;
 use std::path::PathBuf;
 
+use super::path::{Dirs, RelPath};
 use super::rustc_info::get_file_name;
 use super::utils::{is_ci, CargoProject, Compiler};
 
-static CG_CLIF: CargoProject = CargoProject::local(".");
+static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif");
 
 pub(crate) fn build_backend(
+    dirs: &Dirs,
     channel: &str,
     host_triple: &str,
     use_unstable_features: bool,
 ) -> PathBuf {
-    let mut cmd = CG_CLIF.build(&Compiler::host());
+    let mut cmd = CG_CLIF.build(&Compiler::host(), dirs);
 
     cmd.env("CARGO_BUILD_INCREMENTAL", "true"); // Force incr comp even in release mode
 
@@ -43,8 +45,7 @@ pub(crate) fn build_backend(
     super::utils::spawn_and_wait(cmd);
 
     CG_CLIF
-        .source_dir()
-        .join("target")
+        .target_dir(dirs)
         .join(host_triple)
         .join(channel)
         .join(get_file_name("rustc_codegen_cranelift", "dylib"))

build_system/build_sysroot.rs

@@ -1,37 +1,41 @@
 use std::fs;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::process::{self, Command};
 
+use super::path::{Dirs, RelPath};
 use super::rustc_info::{get_file_name, get_rustc_version, get_wrapper_file_name};
 use super::utils::{spawn_and_wait, try_hard_link, CargoProject, Compiler};
 use super::SysrootKind;
 
+static DIST_DIR: RelPath = RelPath::DIST;
+static BIN_DIR: RelPath = RelPath::DIST.join("bin");
+static LIB_DIR: RelPath = RelPath::DIST.join("lib");
+static RUSTLIB_DIR: RelPath = LIB_DIR.join("rustlib");
+
 pub(crate) fn build_sysroot(
+    dirs: &Dirs,
     channel: &str,
     sysroot_kind: SysrootKind,
-    dist_dir: &Path,
     cg_clif_dylib_src: &Path,
     host_triple: &str,
     target_triple: &str,
 ) {
     eprintln!("[BUILD] sysroot {:?}", sysroot_kind);
 
-    if dist_dir.exists() {
-        fs::remove_dir_all(dist_dir).unwrap();
-    }
-    fs::create_dir_all(dist_dir.join("bin")).unwrap();
-    fs::create_dir_all(dist_dir.join("lib")).unwrap();
+    DIST_DIR.ensure_fresh(dirs);
+    BIN_DIR.ensure_exists(dirs);
+    LIB_DIR.ensure_exists(dirs);
 
     // Copy the backend
-    let cg_clif_dylib_path = dist_dir
-        .join(if cfg!(windows) {
-            // Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the
-            // binaries.
-            "bin"
-        } else {
-            "lib"
-        })
-        .join(get_file_name("rustc_codegen_cranelift", "dylib"));
+    let cg_clif_dylib_path = if cfg!(windows) {
+        // Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the
+        // binaries.
+        BIN_DIR
+    } else {
+        LIB_DIR
+    }
+    .to_path(dirs)
+    .join(get_file_name("rustc_codegen_cranelift", "dylib"));
     try_hard_link(cg_clif_dylib_src, &cg_clif_dylib_path);
 
     // Build and copy rustc and cargo wrappers
@@ -40,18 +44,17 @@ pub(crate) fn build_sysroot(
 
         let mut build_cargo_wrapper_cmd = Command::new("rustc");
         build_cargo_wrapper_cmd
-            .arg(PathBuf::from("scripts").join(format!("{wrapper}.rs")))
+            .arg(RelPath::SCRIPTS.to_path(dirs).join(&format!("{wrapper}.rs")))
             .arg("-o")
-            .arg(dist_dir.join(wrapper_name))
+            .arg(DIST_DIR.to_path(dirs).join(wrapper_name))
             .arg("-g");
         spawn_and_wait(build_cargo_wrapper_cmd);
     }
 
     let default_sysroot = super::rustc_info::get_default_sysroot();
 
-    let rustlib = dist_dir.join("lib").join("rustlib");
-    let host_rustlib_lib = rustlib.join(host_triple).join("lib");
-    let target_rustlib_lib = rustlib.join(target_triple).join("lib");
+    let host_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(host_triple).join("lib");
+    let target_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(target_triple).join("lib");
     fs::create_dir_all(&host_rustlib_lib).unwrap();
     fs::create_dir_all(&target_rustlib_lib).unwrap();
 
@@ -112,13 +115,7 @@ pub(crate) fn build_sysroot(
             }
         }
         SysrootKind::Clif => {
-            build_clif_sysroot_for_triple(
-                channel,
-                dist_dir,
-                host_triple,
-                &cg_clif_dylib_path,
-                None,
-            );
+            build_clif_sysroot_for_triple(dirs, channel, host_triple, &cg_clif_dylib_path, None);
 
             if host_triple != target_triple {
                 // When cross-compiling it is often necessary to manually pick the right linker
@@ -128,8 +125,8 @@ pub(crate) fn build_sysroot(
                     None
                 };
                 build_clif_sysroot_for_triple(
+                    dirs,
                     channel,
-                    dist_dir,
                     target_triple,
                     &cg_clif_dylib_path,
                     linker,
@@ -142,23 +139,26 @@ pub(crate) fn build_sysroot(
                 let file = file.unwrap().path();
                 let filename = file.file_name().unwrap().to_str().unwrap();
                 if filename.contains("std-") && !filename.contains(".rlib") {
-                    try_hard_link(&file, dist_dir.join("lib").join(file.file_name().unwrap()));
+                    try_hard_link(&file, LIB_DIR.to_path(dirs).join(file.file_name().unwrap()));
                 }
             }
         }
     }
 }
 
-static STANDARD_LIBRARY: CargoProject = CargoProject::local("build_sysroot");
+// FIXME move to download/ or dist/
+pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = RelPath::BUILD_SYSROOT.join("rustc_version");
+pub(crate) static SYSROOT_SRC: RelPath = RelPath::BUILD_SYSROOT.join("sysroot_src");
+static STANDARD_LIBRARY: CargoProject = CargoProject::new(&RelPath::BUILD_SYSROOT, "build_sysroot");
 
 fn build_clif_sysroot_for_triple(
+    dirs: &Dirs,
     channel: &str,
-    dist_dir: &Path,
     triple: &str,
     cg_clif_dylib_path: &Path,
     linker: Option<&str>,
 ) {
-    match fs::read_to_string(Path::new("build_sysroot").join("rustc_version")) {
+    match fs::read_to_string(SYSROOT_RUSTC_VERSION.to_path(dirs)) {
         Err(e) => {
             eprintln!("Failed to get rustc version for patched sysroot source: {}", e);
             eprintln!("Hint: Try `./y.rs prepare` to patch the sysroot source");
@@ -176,7 +176,7 @@ fn build_clif_sysroot_for_triple(
         }
     }
 
-    let build_dir = Path::new("build_sysroot").join("target").join(triple).join(channel);
+    let build_dir = STANDARD_LIBRARY.target_dir(dirs).join(triple).join(channel);
 
     if !super::config::get_bool("keep_sysroot") {
         // Cleanup the deps dir, but keep build scripts and the incremental cache for faster
@@ -189,7 +189,7 @@ fn build_clif_sysroot_for_triple(
     // Build sysroot
     let mut rustflags = "-Zforce-unstable-if-unmarked -Cpanic=abort".to_string();
     rustflags.push_str(&format!(" -Zcodegen-backend={}", cg_clif_dylib_path.to_str().unwrap()));
-    rustflags.push_str(&format!(" --sysroot={}", dist_dir.to_str().unwrap()));
+    rustflags.push_str(&format!(" --sysroot={}", DIST_DIR.to_path(dirs).to_str().unwrap()));
     if channel == "release" {
         rustflags.push_str(" -Zmir-opt-level=3");
     }
@@ -199,18 +199,15 @@ fn build_clif_sysroot_for_triple(
     }
     let mut compiler = Compiler::with_triple(triple.to_owned());
     compiler.rustflags = rustflags;
-    let mut build_cmd = STANDARD_LIBRARY.build(&compiler);
+    let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs);
     if channel == "release" {
         build_cmd.arg("--release");
     }
     build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
     spawn_and_wait(build_cmd);
 
     // Copy all relevant files to the sysroot
-    for entry in
-        fs::read_dir(Path::new("build_sysroot/target").join(triple).join(channel).join("deps"))
-            .unwrap()
-    {
+    for entry in fs::read_dir(build_dir.join("deps")).unwrap() {
         let entry = entry.unwrap();
         if let Some(ext) = entry.path().extension() {
             if ext == "rmeta" || ext == "d" || ext == "dSYM" || ext == "clif" {
@@ -221,7 +218,7 @@ fn build_clif_sysroot_for_triple(
         };
         try_hard_link(
             entry.path(),
-            dist_dir.join("lib").join("rustlib").join(triple).join("lib").join(entry.file_name()),
+            RUSTLIB_DIR.to_path(dirs).join(triple).join("lib").join(entry.file_name()),
         );
     }
 }