[CPP-30]Backend CPU Usage benchmark.

Author: john-michaelburke

.gitattributes

@@ -1,3 +1,3 @@
 src/main/fbs/* linguist-vendored
 src/main/nuitka/* linguist-vendored
-console_backend/benches/data/2020-06-25-BayLoop/STEP_v2_SR_L2_OSR_staging_10Hz_GNSSOnly_AmotechL2/25-190704.sbp filter=lfs diff=lfs merge=lfs -text
+console_backend/benches/data/**/**/*.sbp filter=lfs diff=lfs merge=lfs -text

.github/workflows/main.yml

@@ -9,6 +9,65 @@ on:
   pull_request:
 
 jobs:
+  backend_bench:
+    name: Backend Benchmarks
+    strategy:
+      matrix:
+        os:
+          - ubuntu-20.04
+          # - macos-10.15
+          # - windows-2019
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+
+      - name: Checkout source
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+          ssh-key: ${{ secrets.SSH_KEY }}
+
+      - name: Install ${{ runner.os }} Dependencies.
+        run: |
+          if [ "$RUNNER_OS" == "Linux" ]; then
+              sudo apt-get install -y capnproto
+          elif [ "$RUNNER_OS" == "macOS" ]; then
+              brew install capnp
+          elif [ "$RUNNER_OS" == "Windows" ]; then
+              choco install capnproto
+          fi
+
+      - name: Install stable Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          override: true
+      - uses: davidB/rust-cargo-make@v1
+        with:
+          version: '0.32.11'
+      - name: Cache conda
+        uses: actions/cache@v1
+        env:
+          CACHE_NUMBER: 0
+        with:
+          path: ~/conda_pkgs_dir
+          key:
+            ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
+            hashFiles('conda.yml') }}
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: console_pp
+          environment-file: conda.yml
+          use-only-tar-bz2: true
+      - name: Poetry Install
+        run: |
+          conda activate console_pp
+          python -m poetry install
+      - name: Run Benchmarks
+        run: |
+          cargo make benches
   checks:
     runs-on: ubuntu-latest
     defaults:
@@ -21,10 +80,10 @@ jobs:
           submodules: recursive
           ssh-key: ${{ secrets.SSH_KEY }}
 
-      - name: Install stable Rust
+      - name: Install nightly Rust
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: stable
+          toolchain: nightly
           override: true
           components: rustfmt, clippy
       - uses: davidB/rust-cargo-make@v1
@@ -107,10 +166,10 @@ jobs:
               choco install capnproto nsis
           fi
 
-      - name: Install stable Rust
+      - name: Install nightly Rust
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: stable
+          toolchain: nightly
           override: true
           components: rustfmt, clippy
       - uses: davidB/rust-cargo-make@v1

Makefile.toml

@@ -160,3 +160,14 @@ dependencies = ["python-type-check", "rust-type-check"]
 [tasks.tests]
 dependencies = ["rust-tests"]
 
+[tasks.bench-rust]
+dependencies = ["copy-capnp", "generate-resources"]
+command = "cargo"
+args    = ["bench", "--all-features", "--", "--verbose", "--noplot"]
+
+[tasks.bench-rust-validate]
+command = "python"
+args    = ["console_backend/benches/benchmarks.py"]
+
+[tasks.benches]
+dependencies = ["bench-rust", "bench-rust-validate"]

console_backend/Cargo.toml

@@ -12,10 +12,10 @@ sbp = { git = "https://github.com/swift-nav/libsbp.git", rev = "986af3f35033a720
 ordered-float = "2.0"
 ndarray = "0.14.0"
 glob = "0.3.0"
-criterion = "0.3"
+criterion = "0.3.4"
 
 [[bench]]
-name = "tests"
+name = "benches"
 harness = false
 
 
@@ -25,6 +25,7 @@ capnpc = "0.14"
 [lib]
 name = "console_backend"
 crate-type = ["cdylib", "lib"]
+bench = false
 
 [features]
 default = []

console_backend/benches/tests.rs renamed to console_backend/benches/benches.rs

@@ -1,48 +1,42 @@
 #![feature(test)]
+#![allow(unused_imports)]
+use criterion::{criterion_group, criterion_main, Criterion};
 use glob::glob;
-use std::{fs, path::Path, sync::mpsc, time, thread};
+use std::{fs, path::Path, sync::mpsc, thread, time};
 
-
-const TEST_DATA_DIRECTORY: &str = "./benches/data/";
 extern crate console_backend;
 use console_backend::process_messages;
 
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
+const TEST_DATA_DIRECTORY: &str = "./benches/data/";
+const BENCHMARK_TIME_LIMIT: u64 = 15000;
+const BENCHMARK_SAMPLE_SIZE: usize = 50;
 
 pub fn criterion_benchmark(c: &mut Criterion) {
     let glob_pattern = Path::new(&TEST_DATA_DIRECTORY).join("**/**/*.sbp");
     let glob_pattern = glob_pattern.to_str().unwrap();
-    
+
     for ele in glob(glob_pattern).expect("failed to read glob") {
-        
         match ele {
             Ok(filename) => {
                 println!("{:?}", filename.display());
                 let file_in_name = filename.to_str().unwrap();
                 let mut group = c.benchmark_group("proc_messages");
-                group.sample_size(10);
-                group.bench_function("RPM", |b| b.iter(|| run_process_messages(file_in_name.clone())));
-                
+                group.measurement_time(time::Duration::from_millis(BENCHMARK_TIME_LIMIT));
+                group.sample_size(BENCHMARK_SAMPLE_SIZE);
+                group.bench_function("RPM", |b| b.iter(|| run_process_messages(file_in_name)));
             }
             Err(e) => println!("{:?}", e),
         }
     }
-    
 }
 fn run_process_messages(file_in_name: &str) {
     let (client_send, client_recv) = mpsc::channel::<Vec<u8>>();
-    thread::spawn(move|| {
-        loop {
-            if let Err(_) = client_recv.recv() {
-            }
-            thread::sleep(time::Duration::from_millis(1));
-        }
-        
+    thread::spawn(move || loop {
+        if client_recv.recv().is_err() {}
     });
     let messages = sbp::iter_messages(Box::new(fs::File::open(file_in_name).unwrap()));
-    process_messages::process_messages(messages, client_send)
+    process_messages::process_messages(messages, client_send);
 }
-    
 
 #[cfg(feature = "criterion_bench")]
 criterion_group!(benches, criterion_benchmark);

console_backend/benches/benchmarks.py

@@ -0,0 +1,82 @@
+import json
+import sys
+from typing import Any, Optional
+
+WINDOWS = "win32"
+MACOS = "darwin"
+LINUX = "linux"
+NAME = "name"
+FILE_PATH = "file_path"
+KEY_LOCATION = "key_location"
+EXPECTED = "expected"
+ERROR_MARGIN_FRAC = "error_margin_frac"
+
+RUST_BENCHMARKS = {
+    WINDOWS: [
+        {
+            NAME: "2020-09-04-BayLoop/M8L_BDS_ADR431_nativeODO_Swiftlets_OSR_prod_10Hz_AmotechL2/04-155800",
+            FILE_PATH: "target/criterion/proc_messages/RPM/base/estimates.json",
+            KEY_LOCATION: "mean.point_estimate",
+            EXPECTED: 27500000,
+            ERROR_MARGIN_FRAC: 0.05,
+        },
+    ],
+    MACOS: [
+        {
+            NAME: "2020-09-04-BayLoop/M8L_BDS_ADR431_nativeODO_Swiftlets_OSR_prod_10Hz_AmotechL2/04-155800",
+            FILE_PATH: "target/criterion/proc_messages/RPM/base/estimates.json",
+            KEY_LOCATION: "mean.point_estimate",
+            EXPECTED: 27500000,
+            ERROR_MARGIN_FRAC: 0.05,
+        },
+    ],
+    LINUX: [
+        {
+            NAME: "2020-09-04-BayLoop/M8L_BDS_ADR431_nativeODO_Swiftlets_OSR_prod_10Hz_AmotechL2/04-155800",
+            FILE_PATH: "target/criterion/proc_messages/RPM/base/estimates.json",
+            KEY_LOCATION: "mean.point_estimate",
+            EXPECTED: 27500000,
+            ERROR_MARGIN_FRAC: 0.05,
+        },
+    ],
+}
+
+
+def get_nested_key(nested_dict: dict, key_path: str) -> Optional[Any]:
+    """Extract a key in nested dict/json assuming stringified key_path.
+
+    Assuming `key_path` format: <key1>.<key2>.<key3>
+
+    Args:
+        nested_dict (dict): The nested dictionary containing the desired key.
+        key_path (str): The stringified nested dictionary key location.
+
+    Returns:
+        Optional[Any]: A value corresponding to the key_path in the nested dictionary.
+            Otherwise, None if not found.
+    """
+    current_key, *next_keys = key_path.split(".", 1)
+    value = nested_dict.get(current_key, None)
+    return value if not isinstance(value, dict) and len(next_keys) != 1 else get_nested_key(value, next_keys[0])
+
+
+def run_validate_benchmarks():
+    """Runner for a suite of benchmark validations.
+    """
+    for os_, benchmarks in RUST_BENCHMARKS.items():
+        if os_ != sys.platform:
+            continue
+        for bench in benchmarks:
+            with open(bench[FILE_PATH]) as fileo:
+                bench_result = json.load(fileo)
+                bench_value = get_nested_key(bench_result, bench[KEY_LOCATION])
+                assert bench_value is not None, f"Test:{bench[NAME]} retrieved bench value None."
+                assert bench_value - bench[EXPECTED] <= bench[ERROR_MARGIN_FRAC] * bench[EXPECTED], (
+                    f"Test:{bench[NAME]} Bench Value:{bench_value} not within "
+                    f"{bench[ERROR_MARGIN_FRAC]} of {bench[EXPECTED]}."
+                )
+                print(f"PASS - {os_}:{bench[NAME]} MARGIN={bench_value - bench[EXPECTED]}")
+
+
+if __name__ == "__main__":
+    run_validate_benchmarks()

console_backend/benches/data/2020-06-25-BayLoop/STEP_v2_SR_L2_OSR_staging_10Hz_GNSSOnly_AmotechL2/25-190704_chopped.sbp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d30c1c7ee9bcfae65ddf74d86ef4f2b66df08bee054ef9c468ff2b327f6bd36
+size 289

console_backend/benches/data/2020-09-04-BayLoop/M8L_BDS_ADR431_nativeODO_Swiftlets_OSR_prod_10Hz_AmotechL2/04-155800_one_second.sbp

@@ -1,5 +1,5 @@
-pub mod server;
 pub mod process_messages;
+pub mod server;
 pub mod console_backend_capnp {
     include!(concat!(env!("OUT_DIR"), "/console_backend_capnp.rs"));
 }

console_backend/src/lib.rs

@@ -1,9 +1,7 @@
 use capnp::message::Builder;
 use capnp::serialize;
-use ndarray::{s, Array, Array2, Axis};
 use ordered_float::*;
 use sbp::messages::SBP;
-use std::collections::HashMap;
 use std::sync::mpsc;
 
 use crate::console_backend_capnp as m;
@@ -165,4 +163,4 @@ pub fn process_messages(
             }
         }
     }
-}
+}