Auto merge of rust-lang#138307 - Kobzol:citool-alias, r=<try>

Allow specifying glob patterns for try jobs

This PR modifies the `try-job` lookup logic to allow glob patterns. So you can e.g. request all MSVC-related jobs with `try-job: *msvc*`.

Best reviewed commit by commit.

r? `@marcoieni`

try-job: `*msvc*`

Author: bors

src/ci/citool/Cargo.lock

@@ -107,6 +107,7 @@ dependencies = [
  "build_helper",
  "clap",
  "csv",
+ "glob-match",
  "insta",
  "serde",
  "serde_json",
@@ -308,6 +309,12 @@ dependencies = [
  "wasi",
 ]
 
+[[package]]
+name = "glob-match"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9985c9503b412198aa4197559e9a318524ebc4519c229bfa05a535828c950b9d"
+
 [[package]]
 name = "hashbrown"
 version = "0.15.2"

src/ci/citool/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 anyhow = "1"
 clap = { version = "4.5", features = ["derive"] }
 csv = "1"
+glob-match = "0.2"
 serde = { version = "1", features = ["derive"] }
 serde_yaml = "0.9"
 serde_json = "1"

src/ci/citool/src/jobs.rs

@@ -0,0 +1,244 @@
+#[cfg(test)]
+mod tests;
+
+use std::collections::BTreeMap;
+
+use serde_yaml::Value;
+
+use crate::GitHubContext;
+
+/// Representation of a job loaded from the `src/ci/github-actions/jobs.yml` file.
+#[derive(serde::Deserialize, Debug, Clone)]
+pub struct Job {
+    /// Name of the job, e.g. mingw-check
+    pub name: String,
+    /// GitHub runner on which the job should be executed
+    pub os: String,
+    pub env: BTreeMap<String, Value>,
+    /// Should the job be only executed on a specific channel?
+    #[serde(default)]
+    pub only_on_channel: Option<String>,
+    /// Do not cancel the whole workflow if this job fails.
+    #[serde(default)]
+    pub continue_on_error: Option<bool>,
+    /// Free additional disk space in the job, by removing unused packages.
+    #[serde(default)]
+    pub free_disk: Option<bool>,
+}
+
+impl Job {
+    /// By default, the Docker image of a job is based on its name.
+    /// However, it can be overridden by its IMAGE environment variable.
+    pub fn image(&self) -> String {
+        self.env
+            .get("IMAGE")
+            .map(|v| v.as_str().expect("IMAGE value should be a string").to_string())
+            .unwrap_or_else(|| self.name.clone())
+    }
+
+    fn is_linux(&self) -> bool {
+        self.os.contains("ubuntu")
+    }
+}
+
+#[derive(serde::Deserialize, Debug)]
+struct JobEnvironments {
+    #[serde(rename = "pr")]
+    pr_env: BTreeMap<String, Value>,
+    #[serde(rename = "try")]
+    try_env: BTreeMap<String, Value>,
+    #[serde(rename = "auto")]
+    auto_env: BTreeMap<String, Value>,
+}
+
+#[derive(serde::Deserialize, Debug)]
+pub struct JobDatabase {
+    #[serde(rename = "pr")]
+    pub pr_jobs: Vec<Job>,
+    #[serde(rename = "try")]
+    pub try_jobs: Vec<Job>,
+    #[serde(rename = "auto")]
+    pub auto_jobs: Vec<Job>,
+
+    /// Shared environments for the individual run types.
+    envs: JobEnvironments,
+}
+
+impl JobDatabase {
+    /// Find `auto` jobs that correspond to the passed `pattern`.
+    /// Patterns are matched using the glob syntax.
+    /// For example `dist-*` matches all jobs starting with `dist-`.
+    fn find_auto_jobs_by_pattern(&self, pattern: &str) -> Vec<Job> {
+        self.auto_jobs
+            .iter()
+            .filter(|j| glob_match::glob_match(pattern, &j.name))
+            .cloned()
+            .collect()
+    }
+}
+
+pub fn load_job_db(db: &str) -> anyhow::Result<JobDatabase> {
+    let mut db: Value = serde_yaml::from_str(&db)?;
+
+    // We need to expand merge keys (<<), because serde_yaml can't deal with them
+    // `apply_merge` only applies the merge once, so do it a few times to unwrap nested merges.
+    db.apply_merge()?;
+    db.apply_merge()?;
+
+    let db: JobDatabase = serde_yaml::from_value(db)?;
+    Ok(db)
+}
+
+/// Representation of a job outputted to a GitHub Actions workflow.
+#[derive(serde::Serialize, Debug)]
+struct GithubActionsJob {
+    /// The main identifier of the job, used by CI scripts to determine what should be executed.
+    name: String,
+    /// Helper label displayed in GitHub Actions interface, containing the job name and a run type
+    /// prefix (PR/try/auto).
+    full_name: String,
+    os: String,
+    env: BTreeMap<String, serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    continue_on_error: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    free_disk: Option<bool>,
+}
+
+/// Skip CI jobs that are not supposed to be executed on the given `channel`.
+fn skip_jobs(jobs: Vec<Job>, channel: &str) -> Vec<Job> {
+    jobs.into_iter()
+        .filter(|job| {
+            job.only_on_channel.is_none() || job.only_on_channel.as_deref() == Some(channel)
+        })
+        .collect()
+}
+
+/// Type of workflow that is being executed on CI
+#[derive(Debug)]
+pub enum RunType {
+    /// Workflows that run after a push to a PR branch
+    PullRequest,
+    /// Try run started with @bors try
+    TryJob { job_patterns: Option<Vec<String>> },
+    /// Merge attempt workflow
+    AutoJob,
+}
+
+/// Maximum number of custom try jobs that can be requested in a single
+/// `@bors try` request.
+const MAX_TRY_JOBS_COUNT: usize = 20;
+
+fn calculate_jobs(
+    run_type: &RunType,
+    db: &JobDatabase,
+    channel: &str,
+) -> anyhow::Result<Vec<GithubActionsJob>> {
+    let (jobs, prefix, base_env) = match run_type {
+        RunType::PullRequest => (db.pr_jobs.clone(), "PR", &db.envs.pr_env),
+        RunType::TryJob { job_patterns } => {
+            let jobs = if let Some(patterns) = job_patterns {
+                let mut jobs: Vec<Job> = vec![];
+                let mut unknown_patterns = vec![];
+                for pattern in patterns {
+                    let matched_jobs = db.find_auto_jobs_by_pattern(pattern);
+                    if matched_jobs.is_empty() {
+                        unknown_patterns.push(pattern.clone());
+                    } else {
+                        for job in matched_jobs {
+                            if !jobs.iter().any(|j| j.name == job.name) {
+                                jobs.push(job);
+                            }
+                        }
+                    }
+                }
+                if !unknown_patterns.is_empty() {
+                    return Err(anyhow::anyhow!(
+                        "Patterns `{}` did not match any auto jobs",
+                        unknown_patterns.join(", ")
+                    ));
+                }
+                if jobs.len() > MAX_TRY_JOBS_COUNT {
+                    return Err(anyhow::anyhow!(
+                        "It is only possible to schedule up to {MAX_TRY_JOBS_COUNT} custom jobs, received {} custom jobs expanded from {} pattern(s)",
+                        jobs.len(),
+                        patterns.len()
+                    ));
+                }
+                jobs
+            } else {
+                db.try_jobs.clone()
+            };
+            (jobs, "try", &db.envs.try_env)
+        }
+        RunType::AutoJob => (db.auto_jobs.clone(), "auto", &db.envs.auto_env),
+    };
+    let jobs = skip_jobs(jobs, channel);
+    let jobs = jobs
+        .into_iter()
+        .map(|job| {
+            let mut env: BTreeMap<String, serde_json::Value> = crate::yaml_map_to_json(base_env);
+            env.extend(crate::yaml_map_to_json(&job.env));
+            let full_name = format!("{prefix} - {}", job.name);
+
+            GithubActionsJob {
+                name: job.name,
+                full_name,
+                os: job.os,
+                env,
+                continue_on_error: job.continue_on_error,
+                free_disk: job.free_disk,
+            }
+        })
+        .collect();
+
+    Ok(jobs)
+}
+
+pub fn calculate_job_matrix(
+    db: JobDatabase,
+    gh_ctx: GitHubContext,
+    channel: &str,
+) -> anyhow::Result<()> {
+    let run_type = gh_ctx.get_run_type().ok_or_else(|| {
+        anyhow::anyhow!("Cannot determine the type of workflow that is being executed")
+    })?;
+    eprintln!("Run type: {run_type:?}");
+
+    let jobs = calculate_jobs(&run_type, &db, channel)?;
+    if jobs.is_empty() {
+        return Err(anyhow::anyhow!("Computed job list is empty"));
+    }
+
+    let run_type = match run_type {
+        RunType::PullRequest => "pr",
+        RunType::TryJob { .. } => "try",
+        RunType::AutoJob => "auto",
+    };
+
+    eprintln!("Output");
+    eprintln!("jobs={jobs:?}");
+    eprintln!("run_type={run_type}");
+    println!("jobs={}", serde_json::to_string(&jobs)?);
+    println!("run_type={run_type}");
+
+    Ok(())
+}
+
+pub fn find_linux_job<'a>(jobs: &'a [Job], name: &str) -> anyhow::Result<&'a Job> {
+    let Some(job) = jobs.iter().find(|j| j.name == name) else {
+        let available_jobs: Vec<&Job> = jobs.iter().filter(|j| j.is_linux()).collect();
+        let mut available_jobs =
+            available_jobs.iter().map(|j| j.name.to_string()).collect::<Vec<_>>();
+        available_jobs.sort();
+        return Err(anyhow::anyhow!(
+            "Job {name} not found. The following jobs are available:\n{}",
+            available_jobs.join(", ")
+        ));
+    };
+    if !job.is_linux() {
+        return Err(anyhow::anyhow!("Only Linux jobs can be executed locally"));
+    }
+
+    Ok(job)
+}

src/ci/citool/src/jobs/tests.rs

@@ -0,0 +1,64 @@
+use crate::jobs::{JobDatabase, load_job_db};
+
+#[test]
+fn lookup_job_pattern() {
+    let db = load_job_db(
+        r#"
+envs:
+  pr:
+  try:
+  auto:
+
+pr:
+try:
+auto:
+    - name: dist-a
+      os: ubuntu
+      env: {}
+    - name: dist-a-alt
+      os: ubuntu
+      env: {}
+    - name: dist-b
+      os: ubuntu
+      env: {}
+    - name: dist-b-alt
+      os: ubuntu
+      env: {}
+    - name: test-a
+      os: ubuntu
+      env: {}
+    - name: test-a-alt
+      os: ubuntu
+      env: {}
+    - name: test-i686
+      os: ubuntu
+      env: {}
+    - name: dist-i686
+      os: ubuntu
+      env: {}
+    - name: test-msvc-i686-1
+      os: ubuntu
+      env: {}
+    - name: test-msvc-i686-2
+      os: ubuntu
+      env: {}
+"#,
+    )
+    .unwrap();
+    check_pattern(&db, "dist-*", &["dist-a", "dist-a-alt", "dist-b", "dist-b-alt", "dist-i686"]);
+    check_pattern(&db, "*-alt", &["dist-a-alt", "dist-b-alt", "test-a-alt"]);
+    check_pattern(&db, "dist*-alt", &["dist-a-alt", "dist-b-alt"]);
+    check_pattern(
+        &db,
+        "*i686*",
+        &["test-i686", "dist-i686", "test-msvc-i686-1", "test-msvc-i686-2"],
+    );
+}
+
+#[track_caller]
+fn check_pattern(db: &JobDatabase, pattern: &str, expected: &[&str]) {
+    let jobs =
+        db.find_auto_jobs_by_pattern(pattern).into_iter().map(|j| j.name).collect::<Vec<_>>();
+
+    assert_eq!(jobs, expected);
+}