[CPP-33][CPP-70]Backend Memory Benchmark, Refactored Python Bench Runners

.gitattributes

@@ -1,4 +1,3 @@
 src/main/fbs/* linguist-vendored
 src/main/nuitka/* linguist-vendored
-"console_backend/benches/data/*" filter=lfs diff=lfs merge=lfs -text
-"src/main/benches/data/*" filter=lfs diff=lfs merge=lfs -text
+"console_backend/tests/data/*" filter=lfs diff=lfs merge=lfs -text

.github/ci-build.sh

@@ -14,7 +14,7 @@ if [[ "$OS_NAME" == "macOS" ]]; then
     mv ${archive_name}_osx.tar.gz "${archive_name}-${VERSION}-${BUILD_TRIPLET}.tar.gz";
     echo "${archive_name}-${VERSION}-${BUILD_TRIPLET}.tar.gz" >release-archive.filename;
 
-    cp -r src/main/benches 'main.dist';
+    cp -r console_backend/tests/data 'main.dist' && cp utils/bench_runner.py 'main.dist';
     cd 'main.dist';
     7z a -tzip "../$OS_NAME.zip" *;
     cd ..;
@@ -29,7 +29,7 @@ if [[ "$OS_NAME" == "Linux" ]]; then
     mv ${archive_name}_linux.tar.gz "${archive_name}-${VERSION}-${BUILD_TRIPLET}.tar.gz";
     echo "${archive_name}-${VERSION}-${BUILD_TRIPLET}.tar.gz" >release-archive.filename;
 
-    cp -r src/main/benches 'target/swift_navigation_console';
+    cp -r console_backend/tests/data 'target/swift_navigation_console' && cp utils/bench_runner.py 'target/swift_navigation_console';
     cd 'target/swift_navigation_console';
     7z a -tzip "../../$OS_NAME.zip" *;
     cd ../..;
@@ -44,7 +44,7 @@ if [[ "$OS_NAME" == "Windows" ]]; then
     mv 'target/swift_navigation_consoleSetup.exe' "${archive_name}-${VERSION}-windows-${BUILD_TRIPLET}.exe";
     echo "${archive_name}-${VERSION}-windows-${BUILD_TRIPLET}.exe" >release-archive.filename;
 
-    cp -r src/main/benches 'target/swift_navigation_console';
+    cp -r console_backend/tests/data 'target/swift_navigation_console' && cp utils/bench_runner.py 'target/swift_navigation_console';
     cd 'target/swift_navigation_console';
     ls -l;
     7z a -tzip "../../$OS_NAME.zip" *;

.github/workflows/main.yml

@@ -80,10 +80,10 @@ jobs:
         run: |
           conda run -n console_pp poetry install
 
-      - name: Run Benchmarks
+      - name: Run Backend Benchmarks
         shell: bash
         run: |
-          cargo make benches
+          cargo make backend-benches
 
   checks:
 
@@ -277,9 +277,9 @@ jobs:
         run: |
           cd output/${{ runner.os }}
           if [ "$RUNNER_OS" == "Windows" ]; then
-              python benches/frontend_bench.py --binary=swift_navigation_console.exe
+              python bench_runner.py --frontend_cpu --executable=swift_navigation_console.exe
           else
-              python benches/frontend_bench.py --binary=swift_navigation_console
+              python bench_runner.py --frontend_cpu --executable=swift_navigation_console
           fi
 
   release:

Makefile.toml

@@ -189,22 +189,48 @@ dependencies = ["python-type-check", "rust-type-check"]
 [tasks.tests]
 dependencies = ["rust-tests"]
 
-[tasks.bench-rust]
+[tasks.backend-cpu-bench]
 dependencies = ["copy-capnp", "generate-resources"]
 command = "cargo"
 args    = ["bench", "--all-features", "--", "--verbose", "--noplot"]
 
-[tasks.bench-rust-validate]
+[tasks.backend-cpu-bench-validate]
 script_runner = "@shell"
 script = '''
-conda run -n $CONDA_ENV python console_backend/benches/benchmarks.py
+conda run -n $CONDA_ENV python utils/bench_runner.py --backend_cpu
 '''
 
-[tasks.benches]
-dependencies = ["bench-rust", "bench-rust-validate"]
+[tasks.backend-mem-bench]
+dependencies = ["copy-capnp", "generate-resources"]
+command = "cargo"
+args    = ["test", "test_run_process_messages", "--manifest-path=console_backend/Cargo.toml", "--features", "benches", "--", "--nocapture"]
+
+[tasks.backend-benches]
+dependencies = ["backend-cpu-bench", "backend-cpu-bench-validate", "backend-mem-bench"]
 
 [tasks.disk-usage-bench]
 script_runner = "@shell"
 script = '''
-conda run -n $CONDA_ENV python utils/disk_usage_benchmarks.py
+conda run -n $CONDA_ENV python utils/bench_runner.py --disk_usage
+'''
+
+[tasks.frontend-cpu-bench.linux]
+script_runner = "@shell"
+script = '''
+echo 'This is only used to validate locally.'
+cd console_backend/tests && python ../../utils/bench_runner.py --frontend_cpu --executable=../../target/swift_navigation_console/swift_navigation_console && cd -
+'''
+
+[tasks.frontend-cpu-bench.windows]
+script_runner = "@shell"
+script = '''
+echo 'This is only used to validate locally.'
+cd console_backend/tests && python ../../utils/bench_runner.py --frontend_cpu --executable=../../target/swift_navigation_console/swift_navigation_console.exe && cd -
+'''
+
+[tasks.frontend-cpu-bench.mac]
+script_runner = "@shell"
+script = '''
+echo 'This is only used to validate locally.'
+cd console_backend/tests && python ../../utils/bench_runner.py --frontend_cpu --executable=../../main.dist/main && cd -
 '''

console_backend/Cargo.toml

@@ -13,9 +13,10 @@ ordered-float = "2.0"
 ndarray = "0.14.0"
 glob = "0.3.0"
 criterion = "0.3.4"
+sysinfo = "0.16.4"
 
 [[bench]]
-name = "benches"
+name = "cpu_benches"
 harness = false
 
 
@@ -29,4 +30,4 @@ bench = false
 
 [features]
 default = []
-criterion_bench = []
+benches = []