[CPP-62][CPP-69]Frontend Benchmark CPU Usage - Windows self hosted runner.

.gitattributes

@@ -1,3 +1,4 @@
 src/main/fbs/* linguist-vendored
 src/main/nuitka/* linguist-vendored
-console_backend/benches/data/**/**/*.sbp filter=lfs diff=lfs merge=lfs -text
+"console_backend/benches/data/*" filter=lfs diff=lfs merge=lfs -text
+"src/main/benches/data/*" filter=lfs diff=lfs merge=lfs -text

.github/ci-build.sh

@@ -8,27 +8,47 @@ set -o pipefail
 archive_name="console_pp"
 
 if [[ "$OS_NAME" == "macOS" ]]; then
-    tar -czf ${archive_name}_osx.tar.gz 'Swift Navigation Console.dmg';
+    tar -czf ${archive_name}_osx.tar.gz 'swift_navigation_console.dmg';
     VERSION="$(git describe --always --tags --dirty)";
     BUILD_TRIPLET="$(gcc -dumpmachine)";
     mv ${archive_name}_osx.tar.gz "${archive_name}-${VERSION}-${BUILD_TRIPLET}.tar.gz";
     echo "${archive_name}-${VERSION}-${BUILD_TRIPLET}.tar.gz" >release-archive.filename;
+
+    cp -r src/main/benches 'main.dist';
+    cd 'main.dist';
+    7z a -tzip "../$OS_NAME.zip" *;
+    cd ..;
+    echo "${OS_NAME}.zip" >bench.filename;
     ls -l;
 fi
 
 if [[ "$OS_NAME" == "Linux" ]]; then
-    tar -C "target" -czf ${archive_name}_linux.tar.gz 'Swift Navigation Console.deb';
+    tar -C "target" -czf ${archive_name}_linux.tar.gz 'swift_navigation_console.deb';
     VERSION="$(git describe --always --tags --dirty)";
     BUILD_TRIPLET="$(gcc -dumpmachine)";
     mv ${archive_name}_linux.tar.gz "${archive_name}-${VERSION}-${BUILD_TRIPLET}.tar.gz";
     echo "${archive_name}-${VERSION}-${BUILD_TRIPLET}.tar.gz" >release-archive.filename;
+
+    cp -r src/main/benches 'target/swift_navigation_console';
+    cd 'target/swift_navigation_console';
+    7z a -tzip "../../$OS_NAME.zip" *;
+    cd ../..;
+    echo "${OS_NAME}.zip" >bench.filename;
+
     ls -l;
 fi
 
 if [[ "$OS_NAME" == "Windows" ]]; then
     VERSION="$(git describe --always --tags)";
     BUILD_TRIPLET="x86_64-pc-windows-msvc";
-    mv 'target/Swift Navigation ConsoleSetup.exe' "${archive_name}-${VERSION}-windows-${BUILD_TRIPLET}.exe";
+    mv 'target/swift_navigation_consoleSetup.exe' "${archive_name}-${VERSION}-windows-${BUILD_TRIPLET}.exe";
     echo "${archive_name}-${VERSION}-windows-${BUILD_TRIPLET}.exe" >release-archive.filename;
+
+    cp -r src/main/benches 'target/swift_navigation_console';
+    cd 'target/swift_navigation_console';
+    ls -l;
+    7z a -tzip "../../$OS_NAME.zip" *;
+    cd ../..;
+    echo "${OS_NAME}.zip" >bench.filename;
     ls -l;
 fi

.github/workflows/main.yml

@@ -202,12 +202,59 @@ jobs:
         shell: bash
         run: |
           cargo make prod-installer
+
+      - name: Zip ${{ runner.os }} Binaries.
+        env:
+          OS_NAME: ${{ runner.os }}
+        shell: bash
+        run: |
           bash ./.github/ci-build.sh
           echo "RELEASE_ARCHIVE=$(cat release-archive.filename)" >>$GITHUB_ENV
-
+          echo "BENCH_ARCHIVE=$(cat bench.filename)" >>$GITHUB_ENV
       - uses: actions/upload-artifact@v2
         with:
           name: ${{ runner.os }}-artifacts
           path: |
             ${{ env.RELEASE_ARCHIVE }}
             release-archive.filename
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ runner.os }}-artifacts-bench
+          path: |
+            ${{ env.BENCH_ARCHIVE }}
+            bench.filename
+  frontend_bench:
+    name: Run Frontend Benchmarks
+    continue-on-error: true
+    timeout-minutes: 5
+    needs:
+      - build
+    strategy:
+      matrix:
+        os:
+          - Windows
+    runs-on: [self-hosted]
+    steps:
+      - name: Remove previous build.
+        shell: bash
+        run: |
+          rm -rf output
+      - uses: actions/download-artifact@v2
+        with:
+          name: ${{ runner.os }}-artifacts-bench
+          path: |
+            output
+      - name: Extract binary and data.
+        run: |
+          cd output
+          7z x $(cat bench.filename) -aoa -o${{ runner.os }}
+
+      - name: Run Frontend Benchmark.
+        shell: bash
+        run: |
+          cd output/${{ runner.os }}
+          if [ "$RUNNER_OS" == "Windows" ]; then
+              python benches/frontend_bench.py --binary=swift_navigation_console.exe
+          else
+              python benches/frontend_bench.py --binary=swift_navigation_console
+          fi

Makefile.toml

@@ -26,10 +26,9 @@ conda run -n $CONDA_ENV fbs run
 '''
 
 [tasks.prod-run]
-dependencies = ["copy-capnp", "generate-resources"]
+dependencies = ["copy-capnp", "generate-resources", "prod-install-backend"]
 script_runner = "@shell"
 script = '''
-conda run -n $CONDA_ENV pip install --force ./console_backend
 conda run -n $CONDA_ENV fbs run
 '''
 
@@ -42,7 +41,7 @@ conda run -n $CONDA_ENV python -m pip install setuptools-rust==0.11.6
 [tasks.prod-install-backend]
 script_runner = "@shell"
 script = '''
-conda run -n $CONDA_ENV python -m pip install --force ./console_backend
+conda run -n $CONDA_ENV python -m pip install -t ./console_backend --upgrade --force ./console_backend -v
 '''
 
 [tasks.prod-freeze]
@@ -52,8 +51,20 @@ script = '''
 conda run -n $CONDA_ENV fbs freeze
 '''
 
+[tasks.copy-console-backend-to-fbs]
+script_runner = "@shell"
+script = '''
+mkdir target/swift_navigation_console/console_backend
+cp -r console_backend/console_backend/*  target/swift_navigation_console/console_backend/.
+'''
+
+[tasks.copy-console-backend-to-fbs.mac]
+script_runner = "@shell"
+script = '''
+'''
+
 [tasks.prod-installer]
-dependencies = ["prod-freeze", "remove-negligibles"]
+dependencies = ["prod-freeze", "remove-negligibles", "copy-console-backend-to-fbs"]
 script_runner = "@shell"
 script = '''
 conda run -n $CONDA_ENV fbs installer
@@ -105,7 +116,7 @@ script_runner = "@shell"
 script = '''
 mkdir -p main.app/Contents/MacOS
 cp -r main.dist/* main.app/Contents/MacOS/.
-hdiutil create -volname SwiftNavConsole -srcfolder main.app -ov -format UDZO 'Swift Navigation Console.dmg'
+hdiutil create -volname SwiftNavConsole -srcfolder main.app -ov -format UDZO 'swift_navigation_console.dmg'
 '''
 
 [tasks.rust-format]

console_backend/.gitignore

@@ -2,3 +2,6 @@ build/
 dist/
 *.egg-info
 console_backend.capnp
+console_backend/*.pyd
+console_backend/*.so
+*.dist-info

console_backend/benches/benchmarks.py

@@ -91,26 +91,25 @@ def get_nested_key(nested_dict: dict, key_path: str) -> Optional[Any]:
 def run_validate_benchmarks():
     """Runner for a suite of benchmark validations.
     """
-    for os_, benchmarks in RUST_BENCHMARKS.items():
-        if os_ != sys.platform:
-            continue
-        for bench in benchmarks:
-            with open(bench[FILE_PATH]) as fileo:
-                bench_result = json.load(fileo)
-                bench_value = get_nested_key(bench_result, bench[KEY_LOCATION])
-                assert bench_value is not None, f"Test:{bench[NAME]} retrieved bench value None."
-                if bench[SUCCESS]:
-                    assert bench_value - bench[EXPECTED] <= bench[ERROR_MARGIN_FRAC] * bench[EXPECTED], (
-                        f"Success Test:{bench[NAME]} Bench Value:{bench_value} not within "
-                        f"{bench[ERROR_MARGIN_FRAC]} of {bench[EXPECTED]}."
-                    )
-                    print(f"PASS - {os_}:{bench[NAME]} MARGIN={bench_value - bench[EXPECTED]}")
-                else:
-                    assert bench_value - bench[EXPECTED] > bench[ERROR_MARGIN_FRAC] * bench[EXPECTED], (
-                        f"Failure Test:{bench[NAME]} Bench Value:{bench_value} not outside of "
-                        f"{bench[ERROR_MARGIN_FRAC]} of {bench[EXPECTED]}."
-                    )
-                    print(f"PASS(Fail Test) - {os_}:{bench[NAME]} MARGIN={bench_value - bench[EXPECTED]}")
+    os_ = sys.platform
+    benchmarks = RUST_BENCHMARKS.get(os_, [])
+    for bench in benchmarks:
+        with open(bench[FILE_PATH]) as fileo:
+            bench_result = json.load(fileo)
+            bench_value = get_nested_key(bench_result, bench[KEY_LOCATION])
+            assert bench_value is not None, f"Test:{bench[NAME]} retrieved bench value None."
+            if bench[SUCCESS]:
+                assert bench_value - bench[EXPECTED] <= bench[ERROR_MARGIN_FRAC] * bench[EXPECTED], (
+                    f"Success Test:{bench[NAME]} Bench Value:{bench_value} not within "
+                    f"{bench[ERROR_MARGIN_FRAC]} of {bench[EXPECTED]}."
+                )
+                print(f"PASS - {os_}:{bench[NAME]} MARGIN={bench_value - bench[EXPECTED]}")
+            else:
+                assert bench_value - bench[EXPECTED] > bench[ERROR_MARGIN_FRAC] * bench[EXPECTED], (
+                    f"Failure Test:{bench[NAME]} Bench Value:{bench_value} not outside of "
+                    f"{bench[ERROR_MARGIN_FRAC]} of {bench[EXPECTED]}."
+                )
+                print(f"PASS(Fail Test) - {os_}:{bench[NAME]} MARGIN={bench_value - bench[EXPECTED]}")
 
 
 if __name__ == "__main__":

console_backend/src/server.rs

@@ -17,6 +17,8 @@ use std::thread;
 use crate::console_backend_capnp as m;
 use crate::process_messages::process_messages;
 
+const CLOSE: &str = "CLOSE";
+
 /// The backend server
 #[pyclass]
 struct Server {
@@ -128,7 +130,14 @@ impl Server {
                                 println!("Opened file successfully!");
 
                                 let messages = sbp::iter_messages(stream);
-                                process_messages(messages, client_send_clone);
+                                process_messages(messages, client_send_clone.clone());
+                                let mut builder = Builder::new_default();
+                                let msg = builder.init_root::<m::message::Builder>();
+                                let mut status = msg.init_status();
+                                status.set_text(CLOSE);
+                                let mut msg_bytes: Vec<u8> = vec![];
+                                serialize::write_message(&mut msg_bytes, &builder).unwrap();
+                                client_send_clone.send(msg_bytes).unwrap();
                             } else {
                                 println!("Couldn't open file...");
                             }

src/build/settings/base.json

@@ -1,6 +1,7 @@
 {
-    "app_name": "Swift Navigation Console",
+    "app_name": "swift_navigation_console",
     "author": "Swift Navigation, Inc.",
     "main_module": "src/main/python/main.py",
-    "version": "0.0.0"
+    "version": "0.0.0",
+    "hidden_imports": ["console_backend"]
 }

src/main/benches/frontend_bench.py

@@ -0,0 +1,84 @@
+import argparse
+import json
+import subprocess
+import os
+import sys
+
+WINDOWS = "win32"
+MACOS = "darwin"
+LINUX = "linux"
+RESULTS = "results"
+NAME = "name"
+FILE_PATH = "file_path"
+KEY_LOCATION = "key_location"
+EXPECTED = "expected"
+ERROR_MARGIN_FRAC = "error_margin_frac"
+
+DEFAULT_JSON_FILEPATH = "fileout.json"
+BENCHMARK_COMMAND_ARGS = lambda file_path: f"--file-in={file_path} --connect"
+HYPERFINE_COMMAND = lambda file_out: f"hyperfine --warmup 1 --runs 5 --show-output --export-json {file_out} "
+
+FRONTEND_BENCHMARKS = {
+    WINDOWS: [
+        {
+            NAME: "202010224_192043",
+            FILE_PATH: "benches/data/202010224_192043.sbp",
+            KEY_LOCATION: "mean",
+            EXPECTED: 1.75,
+            ERROR_MARGIN_FRAC: 0.05,
+        },
+    ],
+    MACOS: [
+        {
+            NAME: "202010224_192043",
+            FILE_PATH: "benches/data/202010224_192043.sbp",
+            KEY_LOCATION: "mean",
+            EXPECTED: 1.75,
+            ERROR_MARGIN_FRAC: 0.05,
+        },
+    ],
+    LINUX: [
+        {
+            NAME: "202010224_192043",
+            FILE_PATH: "benches/data/202010224_192043.sbp",
+            KEY_LOCATION: "mean",
+            EXPECTED: 1.75,
+            ERROR_MARGIN_FRAC: 0.05,
+        },
+    ],
+}
+
+
+def run_frontend_benchmark(binary: str):
+    """Runner for a suite of benchmark validations.
+    Args:
+        binary (str): Path to the binary location to run the benchmark on.
+    """
+    prepped_command = f"{binary}"
+    os_ = sys.platform
+    benchmarks = FRONTEND_BENCHMARKS.get(sys.platform, [])
+    for bench in benchmarks:
+        bench_command = (
+            f"{HYPERFINE_COMMAND(DEFAULT_JSON_FILEPATH)} \"{prepped_command} "
+            f"{BENCHMARK_COMMAND_ARGS(bench[FILE_PATH])}\""
+        )
+        subprocess.call(bench_command, shell=True)
+        with open(DEFAULT_JSON_FILEPATH) as fileo:
+            bench_result = json.load(fileo)
+            bench_value = bench_result[RESULTS][0].get(bench[KEY_LOCATION], None)
+            assert bench_value is not None, f"Test:{bench[NAME]} retrieved bench value None."
+            assert bench_value - bench[EXPECTED] <= bench[ERROR_MARGIN_FRAC] * bench[EXPECTED], (  # type: ignore
+                f"Test:{bench[NAME]} Bench Value:{bench_value} not within "
+                f"{bench[ERROR_MARGIN_FRAC]} of {bench[EXPECTED]}."
+            )
+            print(f"PASS - {os_}:{bench[NAME]} MARGIN={bench_value - bench[EXPECTED]}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--binary", required=True, help="The path to the Swift Console binary.")
+
+    args = parser.parse_args()
+
+    assert os.path.isfile(args.binary), f"The path provided {args.binary} is not a file."
+    run_frontend_benchmark(args.binary)