diff --git a/Cargo.lock b/Cargo.lock index 5ac286d14c2e7..d216ed9cdc5b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,6 +68,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "alga" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658f9468113d34781f6ca9d014d174c74b73de870f1e0e3ad32079bbab253b19" +dependencies = [ + "approx", + "libm", + "num-complex", + "num-traits", +] + [[package]] name = "ansi_term" version = "0.11.0" @@ -104,6 +116,15 @@ dependencies = [ "xdg", ] +[[package]] +name = "approx" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0e60b75072ecd4168020818c0107f2857bb6c4e64252d8d3983f6263b40a5c3" +dependencies = [ + "num-traits", +] + [[package]] name = "arc-swap" version = "0.4.4" @@ -1429,6 +1450,7 @@ version = "2.0.0-alpha.4" dependencies = [ "frame-support", "frame-system", + "linregress", "parity-scale-codec", "sp-api", "sp-io", @@ -2595,6 +2617,12 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "libm" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a" + [[package]] name = "libp2p" version = "0.16.2" @@ -3063,6 +3091,17 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "linregress" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9290cf6f928576eeb9c096c6fad9d8d452a0a1a70a2bbffa6e36064eedc0aac9" +dependencies = [ + "failure", + "nalgebra", + "statrs", +] + [[package]] name = "lite-json" version = "0.1.0" @@ -3123,6 +3162,15 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "matrixmultiply" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4f7ec66360130972f34830bfad9ef05c6610a43938a467bcc9ab9369ab3478f" +dependencies = [ + "rawpointer", +] + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -3272,6 +3320,23 @@ dependencies = [ "unsigned-varint", ] +[[package]] +name = "nalgebra" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaa9fddbc34c8c35dd2108515587b8ce0cab396f17977b8c738568e4edb521a2" +dependencies = [ + "alga", + "approx", + "generic-array", + "matrixmultiply", + "num-complex", + "num-rational", + "num-traits", + "rand 0.6.5", + "typenum", +] + [[package]] name = "names" version = "0.11.0" @@ -3721,6 +3786,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" +dependencies = [ + "autocfg 1.0.0", + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.42" @@ -5122,6 +5197,19 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "rand" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c618c47cd3ebd209790115ab837de41425723956ad3ce2e6a7f09890947cacb9" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "winapi 0.3.8", +] + [[package]] name = "rand" version = "0.6.5" @@ -5291,6 +5379,12 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.3.0" @@ -7532,6 +7626,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "statrs" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10102ac8d55e35db2b3fafc26f81ba8647da2e15879ab686a67e6d19af2685e8" +dependencies = [ + "rand 0.5.6", +] + [[package]] name = "stream-cipher" version = "0.3.2" diff --git a/frame/benchmarking/Cargo.toml b/frame/benchmarking/Cargo.toml index efd15cf004dc1..d12b4151591fe 100644 --- a/frame/benchmarking/Cargo.toml +++ b/frame/benchmarking/Cargo.toml @@ -9,6 +9,7 @@ repository = "https://github.com/paritytech/substrate/" description = "Macro for benchmarking a FRAME runtime." [dependencies] +linregress = "0.1" codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false } sp-api = { version = "2.0.0-alpha.4", path = "../../primitives/api", default-features = false } sp-runtime-interface = { version = "2.0.0-alpha.4", path = "../../primitives/runtime-interface", default-features = false } diff --git a/frame/benchmarking/src/analysis.rs b/frame/benchmarking/src/analysis.rs new file mode 100644 index 0000000000000..fdf1210832cad --- /dev/null +++ b/frame/benchmarking/src/analysis.rs @@ -0,0 +1,243 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Tools for analysing the benchmark results. + +use std::collections::BTreeMap; +use linregress::{FormulaRegressionBuilder, RegressionDataBuilder, RegressionModel}; +use crate::BenchmarkResults; + +pub struct Analysis { + base: u128, + slopes: Vec, + names: Vec, + value_dists: Option, u128, u128)>>, + model: Option, +} + +impl Analysis { + pub fn median_slopes(r: &Vec) -> Option { + let results = r[0].0.iter().enumerate().map(|(i, &(param, _))| { + let mut counted = BTreeMap::, usize>::new(); + for (params, _, _) in r.iter() { + let mut p = params.iter().map(|x| x.1).collect::>(); + p[i] = 0; + *counted.entry(p).or_default() += 1; + } + let others: Vec = counted.iter().max_by_key(|i| i.1).expect("r is not empty; qed").0.clone(); + let values = r.iter() + .filter(|v| + v.0.iter() + .map(|x| x.1) + .zip(others.iter()) + .enumerate() + .all(|(j, (v1, v2))| j == i || v1 == *v2) + ).map(|(ps, v, _)| (ps[i].1, *v)) + .collect::>(); + (format!("{:?}", param), i, others, values) + }).collect::>(); + + let models = results.iter().map(|(_, _, _, ref values)| { + let mut slopes = vec![]; + for (i, &(x1, y1)) in values.iter().enumerate() { + for &(x2, y2) in values.iter().skip(i + 1) { + if x1 != x2 { + slopes.push((y1 as f64 - y2 as f64) / (x1 as f64 - x2 as f64)); + } + } + } + slopes.sort_by(|a, b| a.partial_cmp(b).expect("values well defined; qed")); + let slope = slopes[slopes.len() / 2]; + + let mut offsets = vec![]; + for &(x, y) in values.iter() { + offsets.push(y as f64 - slope * x as f64); + } + offsets.sort_by(|a, b| a.partial_cmp(b).expect("values well defined; qed")); + let offset = offsets[offsets.len() / 2]; + + (offset, slope) + }).collect::>(); + + let models = models.iter() + .zip(results.iter()) + .map(|((offset, slope), (_, i, others, _))| { + let over = others.iter() + .enumerate() + .filter(|(j, _)| j != i) + .map(|(j, v)| models[j].1 * *v as f64) + .fold(0f64, |acc, i| acc + i); + (*offset - over, *slope) + }) + .collect::>(); + + let base = models[0].0.max(0f64) as u128; + let slopes = models.iter().map(|x| x.1.max(0f64) as u128).collect::>(); + + Some(Self { + base, + slopes, + names: results.into_iter().map(|x| x.0).collect::>(), + value_dists: None, + model: None, + }) + } + + pub fn min_squares_iqr(r: &Vec) -> Option { + let mut results = BTreeMap::, Vec>::new(); + for &(ref params, t, _) in r.iter() { + let p = params.iter().map(|x| x.1).collect::>(); + results.entry(p).or_default().push(t); + } + for (_, rs) in results.iter_mut() { + rs.sort(); + let ql = rs.len() / 4; + *rs = rs[ql..rs.len() - ql].to_vec(); + } + + let mut data = vec![("Y", results.iter().flat_map(|x| x.1.iter().map(|v| *v as f64)).collect())]; + + let names = r[0].0.iter().map(|x| format!("{:?}", x.0)).collect::>(); + data.extend(names.iter() + .enumerate() + .map(|(i, p)| ( + p.as_str(), + results.iter() + .flat_map(|x| Some(x.0[i] as f64) + .into_iter() + .cycle() + .take(x.1.len()) + ).collect::>() + )) + ); + + let data = RegressionDataBuilder::new().build_from(data).ok()?; + + let model = FormulaRegressionBuilder::new() + .data(&data) + .formula(format!("Y ~ {}", names.join(" + "))) + .fit() + .ok()?; + + let slopes = model.parameters.regressor_values.iter() + .enumerate() + .map(|(_, x)| (*x + 0.5) as u128) + .collect(); + + let value_dists = results.iter().map(|(p, vs)| { + let total = vs.iter() + .fold(0u128, |acc, v| acc + *v); + let mean = total / vs.len() as u128; + let sum_sq_diff = vs.iter() + .fold(0u128, |acc, v| { + let d = mean.max(*v) - mean.min(*v); + acc + d * d + }); + let stddev = (sum_sq_diff as f64 / vs.len() as f64).sqrt() as u128; + (p.clone(), mean, stddev) + }).collect::>(); + + Some(Self { + base: (model.parameters.intercept_value + 0.5) as u128, + slopes, + names, + value_dists: Some(value_dists), + model: Some(model), + }) + } +} + +fn ms(mut nanos: u128) -> String { + let mut x = 100_000u128; + while x > 1 { + if nanos > x * 1_000 { + nanos = nanos / x * x; + break; + } + x /= 10; + } + format!("{}", nanos as f64 / 1_000f64) +} + +impl std::fmt::Display for Analysis { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + if let Some(ref value_dists) = self.value_dists { + writeln!(f, "\nData points distribution:")?; + writeln!(f, "{} mean µs sigma µs %", self.names.iter().map(|p| format!("{:>5}", p)).collect::>().join(" "))?; + for (param_values, mean, sigma) in value_dists.iter() { + writeln!(f, "{} {:>8} {:>8} {:>3}.{}%", + param_values.iter().map(|v| format!("{:>5}", v)).collect::>().join(" "), + ms(*mean), + ms(*sigma), + (sigma * 100 / mean), + (sigma * 1000 / mean % 10) + )?; + } + } + if let Some(ref model) = self.model { + writeln!(f, "\nQuality and confidence:")?; + writeln!(f, "param error")?; + for (p, se) in self.names.iter().zip(model.se.regressor_values.iter()) { + writeln!(f, "{} {:>8}", p, ms(*se as u128))?; + } + } + + writeln!(f, "\nModel:")?; + writeln!(f, "Time ~= {:>8}", ms(self.base))?; + for (&t, n) in self.slopes.iter().zip(self.names.iter()) { + writeln!(f, " + {} {:>8}", n, ms(t))?; + } + writeln!(f, " µs") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::BenchmarkParameter; + + #[test] + fn analysis_median_slopes_should_work() { + let a = Analysis::median_slopes(&vec![ + (vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0), + (vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0), + (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0), + (vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0), + (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0), + (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0), + (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0), + (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0), + ]).unwrap(); + assert_eq!(a.base, 10_000_000); + assert_eq!(a.slopes, vec![1_000_000, 100_000]); + } + + #[test] + fn analysis_median_min_squares_should_work() { + let a = Analysis::min_squares_iqr(&vec![ + (vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0), + (vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0), + (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0), + (vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0), + (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0), + (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0), + (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0), + (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0), + ]).unwrap(); + assert_eq!(a.base, 10_000_000); + assert_eq!(a.slopes, vec![1_000_000, 100_000]); + } +} diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index a18048d305337..f6094739bfded 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -20,7 +20,12 @@ mod tests; mod utils; +#[cfg(feature = "std")] +mod analysis; + pub use utils::*; +#[cfg(feature = "std")] +pub use analysis::Analysis; #[doc(hidden)] pub use sp_io::storage::root as storage_root; pub use sp_runtime::traits::Dispatchable; @@ -157,7 +162,7 @@ macro_rules! benchmarks_iter { $( $rest:tt )* ) => { $crate::benchmarks_iter! { - { $( $common )* } ( $( $names )* ) $name { $( $code )* }: { + { $( $common )* } ( $( $names )* ) $name { $( $code )* }: { as $crate::Dispatchable>::dispatch(Call::::$dispatch($($arg),*), $origin.into())?; } $( $rest )* } diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index b2aa4bd6a2517..1c02a754016e3 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -22,7 +22,7 @@ use sc_client_db::BenchmarkingState; use sc_service::{Configuration, ChainSpec}; use sc_executor::{NativeExecutor, NativeExecutionDispatch}; use codec::{Encode, Decode}; -use frame_benchmarking::BenchmarkResults; +use frame_benchmarking::{BenchmarkResults, Analysis}; use sp_core::{ tasks, traits::KeystoreExt, @@ -163,6 +163,17 @@ impl BenchmarkCmd { print!("{:?},{:?}\n", result.1, result.2); }); + print!("\n"); + + // Conduct analysis. + if let Some(analysis) = Analysis::median_slopes(&results) { + println!("Median Slopes Analysis\n========\n{}", analysis); + } + + if let Some(analysis) = Analysis::min_squares_iqr(&results) { + println!("Min Squares Analysis\n========\n{}", analysis); + } + eprintln!("Done."); } Err(error) => eprintln!("Error: {:?}", error),