Skip to content

Commit bf20ec1

Browse files
ishitatsuyukiishitatsuyuki
committed
Download crates in parallel
1 parent 7cbad92 commit bf20ec1

File tree

22 files changed

+382
-217
lines changed

22 files changed

+382
-217
lines changed

‎Cargo.toml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ atty = "0.2"
2121
crates-io = { path = "src/crates-io", version = "0.16" }
2222
crossbeam = "0.3"
2323
crypto-hash = "0.3"
24-
curl = "0.4.6"
24+
curl = "0.4.11"
2525
docopt = "0.8.1"
2626
env_logger = "0.5"
2727
failure = "0.1.1"
@@ -91,3 +91,9 @@ doc = false
9191
[[test]]
9292
name = "testsuite"
9393
path = "tests/testsuite/lib.rs"
94+
95+
[patch.crates-io]
96+
curl = { git = "https://github.com/alexcrichton/curl-rust" }
97+
curl-sys = { git = "https://github.com/alexcrichton/curl-rust" }
98+
openssl = { git = "https://github.com/sfackler/rust-openssl", branch = "more-sync" }
99+
openssl-sys = { git = "https://github.com/sfackler/rust-openssl", branch = "more-sync" }

‎src/cargo/core/package.rs‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -197,22 +197,32 @@ impl<'cfg> PackageSet<'cfg> {
197197
Box::new(self.packages.keys())
198198
}
199199

200-
pub fn get(&self, id: &PackageId) -> CargoResult<&Package> {
201-
let slot = self.packages.get(id).ok_or_else(|| {
202-
internal(format!("couldn't find `{}` in package set", id))
203-
})?;
204-
if let Some(pkg) = slot.borrow() {
205-
return Ok(pkg)
200+
pub fn get(&self, ids: &[&PackageId]) -> CargoResult<Vec<&Package>> {
201+
let mut pending = BTreeMap::new();
202+
for &id in ids {
203+
let slot = self.packages.get(id).ok_or_else(|| {
204+
internal(format!("couldn't find `{}` in package set", id))
205+
})?;
206+
if slot.borrow().is_none() {
207+
let &mut (ref mut pending_ids, ref mut slots) = pending.entry(id.source_id()).or_insert_with(|| (Vec::new(), Vec::new()));
208+
pending_ids.push(id);
209+
slots.push(slot);
210+
}
206211
}
207212
let mut sources = self.sources.borrow_mut();
208-
let source = sources.get_mut(id.source_id()).ok_or_else(|| {
209-
internal(format!("couldn't find source for `{}`", id))
210-
})?;
211-
let pkg = source.download(id).chain_err(|| {
212-
format_err!("unable to get packages from source")
213-
})?;
214-
assert!(slot.fill(pkg).is_ok());
215-
Ok(slot.borrow().unwrap())
213+
for (source_id, &(ref pending_ids, ref slots)) in &pending {
214+
let source = sources.get_mut(source_id).ok_or_else(|| {
215+
internal(format!("couldn't find source `{}`", source_id))
216+
})?;
217+
let pkgs = source.download(&*pending_ids).chain_err(|| {
218+
format_err!("unable to get packages from source")
219+
})?;
220+
for (pkg, slot) in pkgs.into_iter().zip(slots) {
221+
assert!(slot.fill(pkg).is_ok());
222+
}
223+
}
224+
225+
Ok(ids.iter().map(|id| self.packages.get(id).unwrap().borrow().unwrap()).collect())
216226
}
217227

218228
pub fn sources(&self) -> Ref<SourceMap<'cfg>> {

‎src/cargo/core/source/mod.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ pub trait Source: Registry {
2020

2121
/// The download method fetches the full package for each name and
2222
/// version specified.
23-
fn download(&mut self, package: &PackageId) -> CargoResult<Package>;
23+
fn download(&mut self, ids: &[&PackageId]) -> CargoResult<Vec<Package>>;
2424

2525
/// Generates a unique string which represents the fingerprint of the
2626
/// current state of the source.
@@ -57,8 +57,8 @@ impl<'a, T: Source + ?Sized + 'a> Source for Box<T> {
5757
}
5858

5959
/// Forwards to `Source::download`
60-
fn download(&mut self, id: &PackageId) -> CargoResult<Package> {
61-
(**self).download(id)
60+
fn download(&mut self, ids: &[&PackageId]) -> CargoResult<Vec<Package>> {
61+
(**self).download(ids)
6262
}
6363

6464
/// Forwards to `Source::fingerprint`

‎src/cargo/ops/cargo_clean.rs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,10 @@ pub fn clean(ws: &Workspace, opts: &CleanOptions) -> CargoResult<()> {
4545
profiles)?;
4646
let mut units = Vec::new();
4747

48-
for spec in opts.spec {
49-
// Translate the spec to a Package
50-
let pkgid = resolve.query(spec)?;
51-
let pkg = packages.get(pkgid)?;
48+
let pkg_ids: CargoResult<Vec<_>> = opts.spec.iter().map(|spec| resolve.query(spec)).collect();
49+
let pkgs = packages.get(&*pkg_ids?)?;
5250

51+
for pkg in pkgs {
5352
// Generate all relevant `Unit` targets for this package
5453
for target in pkg.targets() {
5554
for kind in [Kind::Host, Kind::Target].iter() {

‎src/cargo/ops/cargo_compile.rs‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -247,12 +247,11 @@ pub fn compile_ws<'a>(ws: &Workspace<'a>,
247247
)?;
248248
let (packages, resolve_with_overrides) = resolve;
249249

250-
let to_builds = specs.iter().map(|p| {
251-
let pkgid = p.query(resolve_with_overrides.iter())?;
252-
let p = packages.get(pkgid)?;
253-
p.manifest().print_teapot(ws.config());
254-
Ok(p)
255-
}).collect::<CargoResult<Vec<_>>>()?;
250+
let pkg_ids: CargoResult<Vec<_>> = specs.iter().map(|p| p.query(resolve_with_overrides.iter())).collect();
251+
let to_builds = packages.get(&*pkg_ids?)?;
252+
for pkg in &to_builds {
253+
pkg.manifest().print_teapot(ws.config());
254+
}
256255

257256
let mut general_targets = Vec::new();
258257
let mut package_targets = Vec::new();

‎src/cargo/ops/cargo_doc.rs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,8 @@ pub fn doc(ws: &Workspace, options: &DocOptions) -> CargoResult<()> {
2222
&specs)?;
2323
let (packages, resolve_with_overrides) = resolve;
2424

25-
let pkgs = specs.iter().map(|p| {
26-
let pkgid = p.query(resolve_with_overrides.iter())?;
27-
packages.get(pkgid)
28-
}).collect::<CargoResult<Vec<_>>>()?;
25+
let pkg_ids: CargoResult<Vec<_>> = specs.iter().map(|p| p.query(resolve_with_overrides.iter())).collect();
26+
let pkgs = packages.get(&*pkg_ids?)?;
2927

3028
let mut lib_names = HashMap::new();
3129
let mut bin_names = HashMap::new();

‎src/cargo/ops/cargo_fetch.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,9 @@ use util::CargoResult;
55
/// Executes `cargo fetch`.
66
pub fn fetch<'a>(ws: &Workspace<'a>) -> CargoResult<(Resolve, PackageSet<'a>)> {
77
let (packages, resolve) = ops::resolve_ws(ws)?;
8-
for id in resolve.iter() {
9-
packages.get(id)?;
8+
{
9+
let pkg_ids: Vec<_> = resolve.iter().collect();
10+
packages.get(&*pkg_ids)?;
1011
}
1112
Ok((resolve, packages))
1213
}

‎src/cargo/ops/cargo_install.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -397,8 +397,8 @@ fn select_pkg<'a, T>(mut source: T,
397397
let deps = source.query_vec(&dep)?;
398398
match deps.iter().map(|p| p.package_id()).max() {
399399
Some(pkgid) => {
400-
let pkg = source.download(pkgid)?;
401-
Ok((pkg, Box::new(source)))
400+
let pkg = source.download(&[pkgid])?;
401+
Ok((pkg.into_iter().next().unwrap(), Box::new(source)))
402402
}
403403
None => {
404404
let vers_info = vers.map(|v| format!(" with version `{}`", v))

‎src/cargo/ops/cargo_output_metadata.rs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,8 @@ fn metadata_full(ws: &Workspace,
5454
&specs)?;
5555
let (packages, resolve) = deps;
5656

57-
let packages = packages.package_ids()
58-
.map(|i| packages.get(i).map(|p| p.clone()))
59-
.collect::<CargoResult<Vec<_>>>()?;
57+
let package_ids: Vec<_> = packages.package_ids().collect();
58+
let packages: Vec<_> = packages.get(&*package_ids)?.into_iter().cloned().collect();
6059

6160
Ok(ExportInfo {
6261
packages,

‎src/cargo/ops/cargo_rustc/context.rs‎

Lines changed: 18 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -780,7 +780,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
780780

781781
let id = unit.pkg.package_id();
782782
let deps = self.resolve.deps(id);
783-
let mut ret = deps.filter(|dep| {
783+
let pkg_ids: Vec<_> = deps.filter(|dep| {
784784
unit.pkg.dependencies().iter().filter(|d| {
785785
d.name() == dep.name() && d.version_req().matches(dep.version())
786786
}).any(|d| {
@@ -814,22 +814,19 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
814814
// actually used!
815815
true
816816
})
817-
}).filter_map(|id| {
818-
match self.get_package(id) {
819-
Ok(pkg) => {
820-
pkg.targets().iter().find(|t| t.is_lib()).map(|t| {
821-
let unit = Unit {
822-
pkg,
823-
target: t,
824-
profile: self.lib_or_check_profile(unit, t),
825-
kind: unit.kind.for_target(t),
826-
};
827-
Ok(unit)
828-
})
817+
}).collect();
818+
819+
let pkgs = self.get_packages(&*pkg_ids)?;
820+
let mut ret: Vec<_> = pkgs.into_iter().filter_map(|pkg| {
821+
pkg.targets().iter().find(|t| t.is_lib()).map(|t| {
822+
Unit {
823+
pkg,
824+
target: t,
825+
profile: self.lib_or_check_profile(unit, t),
826+
kind: unit.kind.for_target(t),
829827
}
830-
Err(e) => Some(Err(e))
831-
}
832-
}).collect::<CargoResult<Vec<_>>>()?;
828+
})
829+
}).collect();
833830

834831
// If this target is a build script, then what we've collected so far is
835832
// all we need. If this isn't a build script, then it depends on the
@@ -913,7 +910,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
913910

914911
/// Returns the dependencies necessary to document a package
915912
fn doc_deps(&self, unit: &Unit<'a>) -> CargoResult<Vec<Unit<'a>>> {
916-
let deps = self.resolve.deps(unit.pkg.package_id()).filter(|dep| {
913+
let dep_ids: Vec<_> = self.resolve.deps(unit.pkg.package_id()).filter(|dep| {
917914
unit.pkg.dependencies().iter().filter(|d| {
918915
d.name() == dep.name()
919916
}).any(|dep| {
@@ -923,16 +920,15 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
923920
_ => false,
924921
}
925922
})
926-
}).map(|dep| {
927-
self.get_package(dep)
928-
});
923+
}).collect();
924+
925+
let deps = self.get_packages(&*dep_ids)?;
929926

930927
// To document a library, we depend on dependencies actually being
931928
// built. If we're documenting *all* libraries, then we also depend on
932929
// the documentation of the library being built.
933930
let mut ret = Vec::new();
934931
for dep in deps {
935-
let dep = dep?;
936932
let lib = match dep.targets().iter().find(|t| t.is_lib()) {
937933
Some(lib) => lib,
938934
None => continue,
@@ -1007,7 +1003,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
10071003
}
10081004

10091005
/// Gets a package for the given package id.
1010-
pub fn get_package(&self, id: &PackageId) -> CargoResult<&'a Package> {
1006+
pub fn get_packages(&self, id: &[&PackageId]) -> CargoResult<Vec<&'a Package>> {
10111007
self.packages.get(id)
10121008
}
10131009

0 commit comments

Comments
 (0)