diff --git a/Cargo.lock b/Cargo.lock index 59fc3be1..b2cb0f6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -695,6 +695,22 @@ dependencies = [ "serde", ] +[[package]] +name = "cargo-util-schemas" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26a31f1bb58068aa01b7809533b02c26b1e64a7810ae99131da5af1a4b8e7fc2" +dependencies = [ + "semver", + "serde", + "serde-untagged", + "serde-value", + "thiserror", + "toml", + "unicode-xid", + "url", +] + [[package]] name = "cargo_metadata" version = "0.18.1" @@ -871,6 +887,7 @@ dependencies = [ "aws-sdk-s3", "base64 0.21.7", "bytes", + "cargo-util-schemas", "cargo_metadata", "chrono", "clap", @@ -1265,6 +1282,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "erased-serde" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e2389d65ab4fab27dc2a5de7b191e1f6617d1f1c8855c0dc569c94a4cbb18d" +dependencies = [ + "serde", + "typeid", +] + [[package]] name = "errno" version = "0.3.9" @@ -3085,6 +3112,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "outref" version = "0.5.1" @@ -3803,6 +3839,27 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-untagged" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2676ba99bd82f75cae5cbd2c8eda6fa0b8760f18978ea840e980dd5567b5c5b6" +dependencies = [ + "erased-serde", + "serde", + "typeid", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.209" @@ -4377,6 +4434,12 @@ dependencies = [ "utf-8", ] +[[package]] +name = "typeid" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e13db2e0ccd5e14a544e8a246ba2312cd25223f616442d7f2cb0e3db614236e" + [[package]] name = "typenum" version = "1.17.0" @@ -4484,6 +4547,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 8cdf7878..f43bee17 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,9 +18,13 @@ aws-sdk-s3 = "1.7" base64 = "0.21.5" bytes = "1" cargo_metadata = "0.18.1" +cargo-util-schemas = "0.7.1" chrono = { version = "0.4", features = ["serde"] } clap = { version = "4", features = ["derive"] } -crates-index = { version = "2.2.0", default-features = false, features = ["git-performance", "git-https"] } +crates-index = { version = "2.2.0", default-features = false, features = [ + "git-performance", + "git-https", +] } crossbeam-channel = "0.5" csv = "1.0.2" ctrlc = "3.1.3" @@ -46,7 +50,10 @@ remove_dir_all = "0.7" reqwest = { version = "0.11", features = ["blocking", "json"] } rusqlite = { version = "0.32.1", features = ["chrono", "functions", "bundled"] } rust_team_data = { git = "https://github.com/rust-lang/team" } -rustwide = { version = "0.19.0", features = ["unstable", "unstable-toolchain-ci"] } +rustwide = { version = "0.19.0", features = [ + "unstable", + "unstable-toolchain-ci", +] } serde = "1.0" serde_derive = "1.0" serde_json = "1.0" diff --git a/src/crates/mod.rs b/src/crates/mod.rs index a3d6b8c8..f36518cd 100644 --- a/src/crates/mod.rs +++ b/src/crates/mod.rs @@ -74,50 +74,164 @@ impl TryFrom<&'_ PackageId> for Crate { type Error = anyhow::Error; fn try_from(pkgid: &PackageId) -> Fallible { - let parts = &pkgid - .repr - .split_ascii_whitespace() - .flat_map(|s| { - // remove () - s.trim_matches(|c: char| c.is_ascii_punctuation()) - // split resource and protocol - .split('+') - }) - .collect::>(); - - match parts[..] { - [name, version, "registry", _] => Ok(Crate::Registry(RegistryCrate { - name: name.to_string(), - version: version.to_string(), - })), - [_, _, "path", path] => Ok(Crate::Path(path.to_string())), - [_, _, "git", repo] => { - if repo.starts_with("https://github.com") { - Ok(Crate::GitHub(repo.replace('#', "/").parse()?)) - } else { - let mut parts = repo.split('#').rev().collect::>(); - let url = parts.pop(); - let sha = parts.pop(); - - match (url, sha) { - (Some(url), None) => Ok(Crate::Git(GitRepo { - url: url.to_string(), - sha: None, - })), - (Some(url), Some(sha)) => Ok(Crate::Git(GitRepo { - // remove additional queries if the sha is present - // as the crate version is already uniquely determined - url: url.split('?').next().unwrap().to_string(), - sha: Some(sha.to_string()), - })), - _ => bail!("malformed git repo: {}", repo), + if pkgid.repr.contains(|c: char| c.is_ascii_whitespace()) { + let parts = &pkgid + .repr + .split_ascii_whitespace() + .flat_map(|s| { + // remove () + s.trim_matches(|c: char| c.is_ascii_punctuation()) + // split resource and protocol + .split('+') + }) + .collect::>(); + + match parts[..] { + [name, version, "registry", _] => Ok(Crate::Registry(RegistryCrate { + name: name.to_string(), + version: version.to_string(), + })), + [_, _, "path", path] => Ok(Crate::Path(path.to_string())), + [_, _, "git", repo] => { + if repo.starts_with("https://github.com") { + Ok(Crate::GitHub(repo.replace('#', "/").parse()?)) + } else { + let mut parts = repo.split('#').rev().collect::>(); + let url = parts.pop(); + let sha = parts.pop(); + + match (url, sha) { + (Some(url), None) => Ok(Crate::Git(GitRepo { + url: url.to_string(), + sha: None, + })), + (Some(url), Some(sha)) => Ok(Crate::Git(GitRepo { + // remove additional queries if the sha is present + // as the crate version is already uniquely determined + url: url.split('?').next().unwrap().to_string(), + sha: Some(sha.to_string()), + })), + _ => bail!("malformed git repo: {}", repo), + } } } + _ => bail!( + "malformed pkgid format: {}\n maybe the representation has changed?", + pkgid.repr + ), + } + } else { + use cargo_util_schemas::core::*; + + let package_id = PackageIdSpec::parse(&pkgid.repr)?; + + match package_id.kind() { + Some(SourceKind::LocalRegistry) => Ok(Crate::Local(package_id.name().to_string())), + Some(SourceKind::Git(rev)) => { + if let Some(url) = package_id.url() { + if url.domain() == Some("github.com") { + if let Some(mut path) = url.path_segments() { + let Some(org) = path.next() else { + bail!("Github URL path is too short") + }; + + let Some(repo_name) = path.next() else { + bail!("Github URL path is too short") + }; + + Ok(Crate::GitHub(GitHubRepo { + org: org.to_string(), + name: repo_name.trim_end_matches(".git").to_string(), + sha: match rev { + GitReference::Rev(rev) + if rev.chars().all(|c| c.is_ascii_hexdigit()) => + { + Some(rev.to_string()) + } + _ => None, + }, + })) + } else { + bail!("Github Git URL doesn't have a valid path") + } + } else { + Ok(Crate::Git(GitRepo { + url: url.to_string(), + sha: match rev { + GitReference::Rev(rev) + if rev.chars().all(|c| c.is_ascii_hexdigit()) => + { + Some(rev.to_string()) + } + _ => None, + }, + })) + } + } else { + bail!("Package Id with SourceKind Git should have a URL") + } + } + Some(SourceKind::Path) => { + if let Some(url) = package_id.url() { + Ok(Crate::Path(url.path().to_string())) + } else { + bail!("Package Id with SourceKind Path should have a URL") + } + } + Some(SourceKind::Registry | SourceKind::SparseRegistry) => { + Ok(Crate::Registry(RegistryCrate { + name: package_id.name().to_string(), + version: package_id + .version() + .ok_or_else(|| anyhow!("missing version for registry crate"))? + .to_string(), + })) + } + Some(SourceKind::Directory) => { + bail!("Unsupported SourceKind Directory") + } + None => match package_id.url() { + None => Ok(Crate::Registry(RegistryCrate { + name: package_id.name().to_string(), + version: package_id + .version() + .ok_or_else(|| anyhow!("missing version for registry crate"))? + .to_string(), + })), + Some(url) => match url.scheme() { + "http" | "https" | "git" | "ssh" => { + if url.domain() == Some("github.com") { + if let Some(mut path) = url.path_segments() { + let Some(org) = path.next() else { + bail!("Github URL path is too short") + }; + + let Some(repo_name) = path.next() else { + bail!("Github URL path is too short") + }; + + Ok(Crate::GitHub(GitHubRepo { + org: org.to_string(), + name: repo_name.trim_end_matches(".git").to_string(), + sha: None, + })) + } else { + bail!("Github Git URL doesn't have a valid path") + } + } else { + Ok(Crate::Git(GitRepo { + url: url.to_string(), + sha: None, + })) + } + } + "file" => Ok(Crate::Path(url.path().to_string())), + other => { + bail!(format!("Unsuported Protocol: {other}")) + } + }, + }, } - _ => bail!( - "malformed pkgid format: {}\n maybe the representation has changed?", - pkgid.repr - ), } } } @@ -252,8 +366,60 @@ mod tests { .to_string(), sha: None }), + + // package id spec + + "registry+https://github.com/rust-lang/crates.io-index#cookie@0.15.0" => Crate::Registry(RegistryCrate { + name: "cookie".to_string(), + version: "0.15.0".to_string(), + }), + "sparse+https://github.com/rust-lang/crates.io-index#cookie@0.15.0" => Crate::Registry(RegistryCrate { + name: "cookie".to_string(), + version: "0.15.0".to_string(), + }), + "regex@1.4.3" => Crate::Registry(RegistryCrate { + name: "regex".to_string(), + version: "1.4.3".to_string(), + }), + + "https://github.com/rust-lang/cargo#0.52.0" => Crate::GitHub(GitHubRepo { + org: "rust-lang".to_string(), + name: "cargo".to_string(), + sha: None + }), + "https://github.com/rust-lang/cargo#cargo-platform@0.1.2" => Crate::GitHub(GitHubRepo { + org: "rust-lang".to_string(), + name: "cargo".to_string(), // repo name not crate name + sha: None + }), + "ssh://git@github.com/rust-lang/regex.git#regex@1.4.3" => Crate::GitHub(GitHubRepo { + org: "rust-lang".to_string(), + name: "regex".to_string(), + sha: None + }), + "git+ssh://git@github.com/rust-lang/regex.git#regex@1.4.3" => Crate::GitHub(GitHubRepo { + org: "rust-lang".to_string(), + name: "regex".to_string(), + sha: None + }), + "git+ssh://git@github.com/rust-lang/regex.git?branch=dev#regex@1.4.3" => Crate::GitHub(GitHubRepo { + org: "rust-lang".to_string(), + name: "regex".to_string(), + sha: None + }), + + "git+https://gitlab.com/dummy_org/dummy?rev=9823f01cf4948a41279f6a3febcf793130cab4f6" => Crate::Git(GitRepo { + url: "https://gitlab.com/dummy_org/dummy" + .to_string(), + sha: Some("9823f01cf4948a41279f6a3febcf793130cab4f6".to_string()) + }), + + "file:///path/to/my/project/foo" => Crate::Path("/path/to/my/project/foo".to_string()), + "file:///path/to/my/project/foo#1.1.8" => Crate::Path("/path/to/my/project/foo".to_string()), + "path+file:///path/to/my/project/foo#1.1.8" => Crate::Path("/path/to/my/project/foo".to_string()), } + // while `invalid` is a valid package name it is missing a version assert!(Crate::try_from(&PackageId { repr: "invalid".to_string() }) diff --git a/tests/minicrater/full/full.html.context.expected.json b/tests/minicrater/full/full.html.context.expected.json index ddb108a5..1f45bd77 100644 --- a/tests/minicrater/full/full.html.context.expected.json +++ b/tests/minicrater/full/full.html.context.expected.json @@ -5,8 +5,29 @@ 0, { "Tree": { - "count": 0, - "tree": {} + "count": 1, + "tree": { + "rust-lang/crater/f190933e896443e285e3bb6962fb87d7439b8d65": [ + { + "color_idx": 0, + "name": "beta-faulty-deps (local)", + "res": "regressed", + "runs": [ + { + "color_idx": 7, + "log": "stable/local/beta-faulty-deps", + "name_idx": 0 + }, + { + "color_idx": 0, + "log": "beta/local/beta-faulty-deps", + "name_idx": 1 + } + ], + "url": "https://github.com/rust-lang/crater/tree/master/local-crates/beta-faulty-deps" + } + ] + } } } ], @@ -26,12 +47,12 @@ { "color_idx": 0, "log": "stable/local/ice-regression", - "name_idx": 2 + "name_idx": 3 }, { "color_idx": 0, "log": "beta/local/ice-regression", - "name_idx": 3 + "name_idx": 4 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/ice-regression" @@ -51,31 +72,13 @@ { "color_idx": 0, "log": "beta/local/error-code", - "name_idx": 2 + "name_idx": 3 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/error-code" } ], "build failed (unknown)": [ - { - "color_idx": 0, - "name": "beta-faulty-deps (local)", - "res": "regressed", - "runs": [ - { - "color_idx": 7, - "log": "stable/local/beta-faulty-deps", - "name_idx": 0 - }, - { - "color_idx": 0, - "log": "beta/local/beta-faulty-deps", - "name_idx": 1 - } - ], - "url": "https://github.com/rust-lang/crater/tree/master/local-crates/beta-faulty-deps" - }, { "color_idx": 0, "name": "beta-regression (local)", @@ -89,7 +92,7 @@ { "color_idx": 0, "log": "beta/local/beta-regression", - "name_idx": 1 + "name_idx": 2 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/beta-regression" @@ -125,7 +128,7 @@ { "color_idx": 0, "log": "stable/local/beta-fixed", - "name_idx": 1 + "name_idx": 2 }, { "color_idx": 7, @@ -143,12 +146,12 @@ { "color_idx": 0, "log": "stable/local/network-access", - "name_idx": 1 + "name_idx": 2 }, { "color_idx": 3, "log": "beta/local/network-access", - "name_idx": 4 + "name_idx": 5 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/network-access" @@ -171,12 +174,12 @@ { "color_idx": 2, "log": "stable/local/broken-cargotoml", - "name_idx": 5 + "name_idx": 6 }, { "color_idx": 2, "log": "beta/local/broken-cargotoml", - "name_idx": 5 + "name_idx": 6 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/broken-cargotoml" @@ -189,12 +192,12 @@ { "color_idx": 2, "log": "stable/local/yanked-deps", - "name_idx": 6 + "name_idx": 7 }, { "color_idx": 2, "log": "beta/local/yanked-deps", - "name_idx": 6 + "name_idx": 7 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/yanked-deps" @@ -215,12 +218,12 @@ { "color_idx": 0, "log": "stable/local/build-fail", - "name_idx": 1 + "name_idx": 2 }, { "color_idx": 0, "log": "beta/local/build-fail", - "name_idx": 1 + "name_idx": 2 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/build-fail" @@ -375,12 +378,12 @@ { "color_idx": 3, "log": "stable/local/test-fail", - "name_idx": 4 + "name_idx": 5 }, { "color_idx": 3, "log": "beta/local/test-fail", - "name_idx": 4 + "name_idx": 5 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/test-fail" @@ -448,6 +451,7 @@ ], "result_names": [ "test passed", + "build faulty deps", "build failed (unknown)", "build compiler error", "build ICE", diff --git a/tests/minicrater/full/index.html.context.expected.json b/tests/minicrater/full/index.html.context.expected.json index 18535b78..45a999eb 100644 --- a/tests/minicrater/full/index.html.context.expected.json +++ b/tests/minicrater/full/index.html.context.expected.json @@ -5,8 +5,29 @@ 0, { "Tree": { - "count": 0, - "tree": {} + "count": 1, + "tree": { + "rust-lang/crater/f190933e896443e285e3bb6962fb87d7439b8d65": [ + { + "color_idx": 0, + "name": "beta-faulty-deps (local)", + "res": "regressed", + "runs": [ + { + "color_idx": 7, + "log": "stable/local/beta-faulty-deps", + "name_idx": 0 + }, + { + "color_idx": 0, + "log": "beta/local/beta-faulty-deps", + "name_idx": 1 + } + ], + "url": "https://github.com/rust-lang/crater/tree/master/local-crates/beta-faulty-deps" + } + ] + } } } ], @@ -26,12 +47,12 @@ { "color_idx": 0, "log": "stable/local/ice-regression", - "name_idx": 2 + "name_idx": 3 }, { "color_idx": 0, "log": "beta/local/ice-regression", - "name_idx": 3 + "name_idx": 4 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/ice-regression" @@ -51,31 +72,13 @@ { "color_idx": 0, "log": "beta/local/error-code", - "name_idx": 2 + "name_idx": 3 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/error-code" } ], "build failed (unknown)": [ - { - "color_idx": 0, - "name": "beta-faulty-deps (local)", - "res": "regressed", - "runs": [ - { - "color_idx": 7, - "log": "stable/local/beta-faulty-deps", - "name_idx": 0 - }, - { - "color_idx": 0, - "log": "beta/local/beta-faulty-deps", - "name_idx": 1 - } - ], - "url": "https://github.com/rust-lang/crater/tree/master/local-crates/beta-faulty-deps" - }, { "color_idx": 0, "name": "beta-regression (local)", @@ -89,7 +92,7 @@ { "color_idx": 0, "log": "beta/local/beta-regression", - "name_idx": 1 + "name_idx": 2 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/beta-regression" @@ -125,7 +128,7 @@ { "color_idx": 0, "log": "stable/local/beta-fixed", - "name_idx": 1 + "name_idx": 2 }, { "color_idx": 7, @@ -143,12 +146,12 @@ { "color_idx": 0, "log": "stable/local/network-access", - "name_idx": 1 + "name_idx": 2 }, { "color_idx": 3, "log": "beta/local/network-access", - "name_idx": 4 + "name_idx": 5 } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/network-access" @@ -218,6 +221,7 @@ ], "result_names": [ "test passed", + "build faulty deps", "build failed (unknown)", "build compiler error", "build ICE", diff --git a/tests/minicrater/full/markdown.md.context.expected.json b/tests/minicrater/full/markdown.md.context.expected.json index f7c639eb..2f4ea88a 100644 --- a/tests/minicrater/full/markdown.md.context.expected.json +++ b/tests/minicrater/full/markdown.md.context.expected.json @@ -4,29 +4,38 @@ "regressed", { "Complete": { - "orphans": [], - "res": [ + "orphans": [ [ { - "krate": { - "Local": "beta-faulty-deps" - }, - "name": "beta-faulty-deps (local)", - "res": "regressed", - "runs": [ - { - "log": "stable/local/beta-faulty-deps", - "res": "test-pass" - }, - { - "log": "beta/local/beta-faulty-deps", - "res": "build-fail:unknown" - } - ], - "url": "https://github.com/rust-lang/crater/tree/master/local-crates/beta-faulty-deps" + "GitHub": { + "name": "crater", + "org": "rust-lang", + "sha": "f190933e896443e285e3bb6962fb87d7439b8d65" + } }, - [] - ], + [ + { + "krate": { + "Local": "beta-faulty-deps" + }, + "name": "beta-faulty-deps (local)", + "res": "regressed", + "runs": [ + { + "log": "stable/local/beta-faulty-deps", + "res": "test-pass" + }, + { + "log": "beta/local/beta-faulty-deps", + "res": "build-fail:depends-on(gh/rust-lang/crater/f190933e896443e285e3bb6962fb87d7439b8d65)" + } + ], + "url": "https://github.com/rust-lang/crater/tree/master/local-crates/beta-faulty-deps" + } + ] + ] + ], + "res": [ [ { "krate": { diff --git a/tests/minicrater/full/results.expected.json b/tests/minicrater/full/results.expected.json index d3f70a5f..edbc54a5 100644 --- a/tests/minicrater/full/results.expected.json +++ b/tests/minicrater/full/results.expected.json @@ -13,7 +13,7 @@ }, { "log": "beta/local/beta-faulty-deps", - "res": "build-fail:unknown" + "res": "build-fail:depends-on(gh/rust-lang/crater/f190933e896443e285e3bb6962fb87d7439b8d65)" } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/beta-faulty-deps" @@ -171,11 +171,11 @@ "runs": [ { "log": "stable/local/faulty-deps", - "res": "build-fail:unknown" + "res": "build-fail:depends-on(reg/lazy_static/0.1.0, gh/rust-lang/crater/c3f462bdab37a93c24b2b172b90564749e892cbc)" }, { "log": "beta/local/faulty-deps", - "res": "build-fail:unknown" + "res": "build-fail:depends-on(reg/lazy_static/0.1.0, gh/rust-lang/crater/c3f462bdab37a93c24b2b172b90564749e892cbc)" } ], "url": "https://github.com/rust-lang/crater/tree/master/local-crates/faulty-deps"