Merge pull request #43 from rust-math/config-download

New package downloader

Author: termoshtt

.github/workflows/rust.yml

@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-18.04
     strategy:
       matrix:
-        target: ["test", "seek", "package"]
+        target: ["test", "seek", "package", "download"]
     steps:
     - uses: actions/checkout@v1
     - name: Test with mkl-rust container

intel-mkl-tool/Makefile

@@ -11,3 +11,6 @@ seek:
 
 package:
 	docker run $(DOCKER_OPTION) $(IMAGE) cargo run --release -- package
+
+download:
+	docker run $(DOCKER_OPTION) $(IMAGE) cargo run --release -- download -o ./test_download

intel-mkl-tool/src/cli.rs

@@ -9,7 +9,11 @@ use structopt::StructOpt;
 enum Opt {
     /// Download Intel-MKL library
     Download {
-        /// Install destination. Default is `$XDG_DATA_HOME/intel-mkl-tool`
+        /// Archive name, e.g. "mkl-static-lp64-iomp". Download all archives if None
+        #[structopt(long = "name")]
+        name: Option<String>,
+        /// Install destination. Default is `$XDG_DATA_HOME/intel-mkl-tool/${MKL_VERSION}/`
+        #[structopt(short = "o", long = "path")]
         path: Option<PathBuf>,
     },
 
@@ -22,8 +26,10 @@ enum Opt {
 
     /// Package Intel MKL libraries into an archive
     Package {
+        #[structopt(long = "name")]
         name: Option<String>,
-        out: Option<PathBuf>,
+        #[structopt(short = "o", long = "path")]
+        path: Option<PathBuf>,
     },
 }
 
@@ -34,13 +40,21 @@ fn main() -> Result<()> {
     let opt = Opt::from_args();
 
     match opt {
-        Opt::Download { path } => {
-            let path = if let Some(path) = path {
-                path
+        Opt::Download { name, path } => {
+            let path = path.unwrap_or(xdg_home_path());
+            if let Some(name) = name {
+                let cfg = Config::from_str(&name)?;
+                cfg.download(&path.join(cfg.name()))?;
             } else {
-                xdg_home_path()
-            };
-            download_default(&path)?;
+                for cfg in Config::possibles() {
+                    info!(
+                        "Download archive {:<22} into {}",
+                        cfg.name(),
+                        path.display()
+                    );
+                    cfg.download(&path.join(cfg.name()))?;
+                }
+            }
         }
 
         Opt::Seek {} => {
@@ -56,26 +70,26 @@ fn main() -> Result<()> {
             }
         }
 
-        Opt::Package { name, out } => {
-            let out = out.unwrap_or(env::current_dir().unwrap());
+        Opt::Package { name, path } => {
+            let path = path.unwrap_or(env::current_dir().unwrap());
             if let Some(name) = name {
                 let cfg = Config::from_str(&name)?;
                 let entry = Entry::from_config(cfg)?;
-                let out = if let Ok(version) = entry.version() {
-                    out.join(format!("{}.{}", version.0, version.1))
+                let path = if let Ok(version) = entry.version() {
+                    path.join(format!("{}.{}", version.0, version.1))
                 } else {
-                    out
+                    path
                 };
-                let package = entry.package(&out)?;
+                let package = entry.package(&path)?;
                 info!("Pacakge created: {}", package.display());
             } else {
                 for entry in Entry::available() {
-                    let out = if let Ok(version) = entry.version() {
-                        out.join(format!("{}.{}", version.0, version.1))
+                    let path = if let Ok(version) = entry.version() {
+                        path.join(format!("{}.{}", version.0, version.1))
                     } else {
-                        out.clone()
+                        path.clone()
                     };
-                    let package = entry.package(&out)?;
+                    let package = entry.package(&path)?;
                     info!("Pacakge created: {}", package.display());
                 }
             }

intel-mkl-tool/src/config.rs

@@ -1,5 +1,7 @@
-use anyhow::*;
-use derive_more::Display;
+use crate::*;
+use curl::easy::Easy;
+use derive_more::*;
+use std::fs;
 
 pub const VALID_CONFIGS: &[&str] = &[
     "mkl-dynamic-ilp64-iomp",
@@ -80,6 +82,13 @@ impl Config {
         })
     }
 
+    pub fn possibles() -> Vec<Self> {
+        VALID_CONFIGS
+            .iter()
+            .map(|name| Self::from_str(name).unwrap())
+            .collect()
+    }
+
     /// identifier used in pkg-config
     pub fn name(&self) -> String {
         format!("mkl-{}-{}-{}", self.link, self.index_size, self.parallel)
@@ -130,6 +139,41 @@ impl Config {
             }
         }
     }
+
+    /// Download archive from AWS S3, and expand into `${out_dir}/*.so`
+    pub fn download<P: AsRef<Path>>(&self, out_dir: P) -> Result<()> {
+        let out_dir = out_dir.as_ref();
+        if out_dir.exists() {
+            fs::create_dir_all(&out_dir)?;
+        }
+        let data = read_from_url(&format!("{}/{}.tar.zst", s3_addr(), self.name()))?;
+        let zstd = zstd::stream::read::Decoder::new(data.as_slice())?;
+        let mut arc = tar::Archive::new(zstd);
+        arc.unpack(&out_dir)?;
+        Ok(())
+    }
+}
+
+/// Helper for download file from URL
+///
+/// - This function expands obtained data into memory space
+///
+fn read_from_url(url: &str) -> Result<Vec<u8>> {
+    let mut data = Vec::new();
+    let mut handle = Easy::new();
+    handle.fail_on_error(true)?;
+    handle.url(url)?;
+    {
+        let mut transfer = handle.transfer();
+        transfer
+            .write_function(|new_data| {
+                data.extend_from_slice(new_data);
+                Ok(new_data.len())
+            })
+            .unwrap();
+        transfer.perform().unwrap();
+    }
+    Ok(data)
 }
 
 #[cfg(test)]

intel-mkl-tool/src/download.rs

@@ -1,5 +1,4 @@
 use crate::*;
-use anyhow::*;
 use derive_more::Deref;
 use std::{
     collections::{HashMap, HashSet},

intel-mkl-tool/src/entry.rs

@@ -90,32 +90,31 @@
 //! |mkl_cdft_core           |             libmkl_cdft_core.so|             libmkl_cdft_core.a|
 //!
 
+use anyhow::*;
 use log::*;
 use std::path::*;
 
 mod config;
-mod download;
 mod entry;
 
 pub use config::*;
-pub use download::*;
 pub use entry::*;
 
 const S3_ADDR: &'static str = "https://s3-ap-northeast-1.amazonaws.com/rust-intel-mkl";
 
 #[cfg(all(target_os = "linux", target_arch = "x86_64"))]
 mod mkl {
-    pub const ARCHIVE: &'static str = "mkl_linux64";
+    pub const OS: &str = "linux";
     pub const EXTENSION_STATIC: &'static str = "a";
     pub const EXTENSION_SHARED: &'static str = "so";
     pub const PREFIX: &'static str = "lib";
-    pub const VERSION_YEAR: u32 = 2019;
-    pub const VERSION_UPDATE: u32 = 5;
+    pub const VERSION_YEAR: u32 = 2020;
+    pub const VERSION_UPDATE: u32 = 1;
 }
 
 #[cfg(all(target_os = "macos", target_arch = "x86_64"))]
 mod mkl {
-    pub const ARCHIVE: &'static str = "mkl_macos64";
+    pub const OS: &str = "macos";
     pub const EXTENSION_STATIC: &'static str = "a";
     pub const EXTENSION_SHARED: &'static str = "dylib";
     pub const PREFIX: &'static str = "lib";
@@ -125,29 +124,34 @@ mod mkl {
 
 #[cfg(all(target_os = "windows", target_arch = "x86_64"))]
 mod mkl {
-    pub const ARCHIVE: &'static str = "mkl_windows64";
+    pub const OS: &str = "windows";
     pub const EXTENSION_STATIC: &'static str = "lib";
     pub const EXTENSION_SHARED: &'static str = "lib";
     pub const PREFIX: &'static str = "";
     pub const VERSION_YEAR: u32 = 2019;
     pub const VERSION_UPDATE: u32 = 5;
 }
 
-pub fn archive_filename() -> String {
+fn s3_addr() -> String {
     format!(
-        "{}_{}_{}.tar.zst",
-        mkl::ARCHIVE,
+        "{}/{}/{}.{}",
+        S3_ADDR,
+        mkl::OS,
         mkl::VERSION_YEAR,
         mkl::VERSION_UPDATE
     )
 }
 
 pub fn xdg_home_path() -> PathBuf {
-    dirs::data_local_dir().unwrap().join("intel-mkl-tool")
+    dirs::data_local_dir().unwrap().join(format!(
+        "intel-mkl-tool/{}.{}",
+        mkl::VERSION_YEAR,
+        mkl::VERSION_UPDATE
+    ))
 }
 
 pub fn seek_pkg_config() -> Option<PathBuf> {
-    if let Ok(lib) = pkg_config::probe_library("mkl-dynamic-lp64-iomp") {
+    if let Ok(lib) = pkg_config::probe_library("mkl-dynamic-lp64-seq") {
         if lib.libs.len() > 1 {
             warn!("Found {} MKL libraries. Use first found.", lib.libs.len())
         }
@@ -156,10 +160,19 @@ pub fn seek_pkg_config() -> Option<PathBuf> {
     None
 }
 
-pub fn seek_home() -> Option<PathBuf> {
-    let home_lib = xdg_home_path();
-    if home_lib.is_dir() {
-        return Some(home_lib);
+pub fn download_default<P: AsRef<Path>>(out_dir: P) -> Result<()> {
+    let cfg = Config::from_str("mkl-dynamic-lp64-seq").unwrap();
+    cfg.download(out_dir)?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn download() -> Result<()> {
+        download_default("./test_download")?;
+        Ok(())
     }
-    None
 }