diff --git a/.gitattributes b/.gitattributes index 433852118..560d95036 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ src/main/fbs/* linguist-vendored src/main/nuitka/* linguist-vendored +console_backend/benches/data/**/**/*.sbp filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2adf8efbe..0049a1006 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,12 +9,90 @@ on: pull_request: jobs: + + backend_bench: + + name: Backend Benchmarks + + strategy: + matrix: + os: + - ubuntu-20.04 + - macos-10.15 + - windows-2019 + + runs-on: ${{ matrix.os }} + + steps: + + - name: Checkout source + uses: actions/checkout@v2 + with: + submodules: recursive + ssh-key: ${{ secrets.SSH_KEY }} + ssh-strict: false + + - uses: webfactory/ssh-agent@v0.5.0 + with: + ssh-private-key: ${{ secrets.SSH_KEY }} + + - name: Run ssh-keyscan + run: ssh-keyscan github.com >> ~/.ssh/known_hosts + + - name: Setup SSH for Windows Git LFS + run: | + & "C:\\Program Files\\Git\\bin\\sh.exe" -c "ssh-keyscan github.com >> ~/.ssh/known_hosts" + & "C:\\Program Files\\Git\\bin\\sh.exe" -c "echo '${{ secrets.SSH_KEY }}' >> ~/.ssh/id_rsa" + if: matrix.os == 'windows-2019' + + - name: Install ${{ runner.os }} Dependencies. + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + sudo apt-get install -y capnproto git-lfs + elif [ "$RUNNER_OS" == "macOS" ]; then + brew install capnp git-lfs + elif [ "$RUNNER_OS" == "Windows" ]; then + choco install -y capnproto git-lfs + fi + + - name: Pull Git LFS objects + run: git lfs pull + env: + GIT_SSH_COMMAND: ssh -o StrictHostKeyChecking=no + + - name: Install stable Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + + - uses: davidB/rust-cargo-make@v1 + with: + version: '0.32.11' + + - uses: conda-incubator/setup-miniconda@v2 + with: + activate-environment: console_pp + environment-file: conda.yml + + - name: Poetry Install + run: | + conda run -n console_pp poetry install + + - name: Run Benchmarks + shell: bash + run: | + cargo make benches + checks: + + name: Code Quality Checks + runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} + steps: + - name: Checkout source uses: actions/checkout@v2 with: @@ -27,67 +105,57 @@ jobs: toolchain: stable override: true components: rustfmt, clippy + - uses: davidB/rust-cargo-make@v1 with: version: '0.32.11' - - name: Cache conda - uses: actions/cache@v1 - env: - CACHE_NUMBER: 0 - with: - path: ~/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('conda.yml') }} + - uses: conda-incubator/setup-miniconda@v2 with: activate-environment: console_pp environment-file: conda.yml - use-only-tar-bz2: true + - name: Install Dependencies. run: | sudo apt-get install -y capnproto - shell: bash - name: Poetry Install run: | - conda activate console_pp - python -m poetry install + conda run -n console_pp poetry install - name: Run Format Check run: | - conda activate console_pp cargo make format-check - name: Run Type Check run: | - conda activate console_pp cargo make types - name: Run Lint Check run: | - conda activate console_pp cargo make lint - name: Run Tests run: | - conda activate console_pp cargo make tests build: + name: Build Binaries + needs: - checks + - backend_bench + strategy: matrix: os: - ubuntu-20.04 - macos-10.15 - windows-2019 + runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash -l {0} + steps: - name: Checkout source @@ -97,6 +165,7 @@ jobs: ssh-key: ${{ secrets.SSH_KEY }} - name: Install ${{ runner.os }} Dependencies. + shell: bash run: | if [ "$RUNNER_OS" == "Linux" ]; then sudo apt-get install -y capnproto ruby ruby-dev rubygems build-essential @@ -113,35 +182,25 @@ jobs: toolchain: stable override: true components: rustfmt, clippy + - uses: davidB/rust-cargo-make@v1 with: version: '0.32.11' - - name: Cache conda - uses: actions/cache@v1 - env: - CACHE_NUMBER: 0 - with: - path: ~/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - hashFiles('conda.yml') }} - uses: conda-incubator/setup-miniconda@v2 with: activate-environment: console_pp environment-file: conda.yml - use-only-tar-bz2: true - name: Poetry Install run: | - conda activate console_pp - python -m poetry install + conda run -n console_pp poetry install - name: Build ${{ runner.os }} Binaries. env: OS_NAME: ${{ runner.os }} + shell: bash run: | - conda activate console_pp cargo make prod-installer bash ./.github/ci-build.sh echo "RELEASE_ARCHIVE=$(cat release-archive.filename)" >>$GITHUB_ENV diff --git a/Cargo.lock b/Cargo.lock index c38709f49..7f749949f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6,6 +6,17 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.0.1" @@ -30,6 +41,24 @@ dependencies = [ "wyz", ] +[[package]] +name = "bstr" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" + [[package]] name = "byteorder" version = "1.4.2" @@ -57,6 +86,15 @@ dependencies = [ "capnp", ] +[[package]] +name = "cast" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b9434b9a5aa1450faa3f9cb14ea0e8c53bb5d2b3c1bfd1ab4fc03e9f33fbfb0" +dependencies = [ + "rustc_version", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -69,12 +107,24 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "bitflags", + "textwrap", + "unicode-width", +] + [[package]] name = "console-backend" version = "0.1.0" dependencies = [ "capnp", "capnpc", + "criterion", "glob", "ndarray", "ordered-float", @@ -82,12 +132,122 @@ dependencies = [ "sbp", ] +[[package]] +name = "const_fn" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" + [[package]] name = "crc16" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" +[[package]] +name = "criterion" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab327ed7354547cc2ef43cbe20ef68b988e70b4b593cbd66a2a61733123a3d23" +dependencies = [ + "atty", + "cast", + "clap", + "criterion-plot", + "csv", + "itertools 0.10.0", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e022feadec601fba1649cfa83586381a4ad31c6bf3a9ab7d408118b05dd9889d" +dependencies = [ + "cast", + "itertools 0.9.0", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +dependencies = [ + "cfg-if 1.0.0", + "const_fn", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "csv" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d58633299b24b515ac72a3f869f8b91306a3cec616a602843a383acd6f9e97" +dependencies = [ + "bstr", + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + [[package]] name = "ctor" version = "0.1.18" @@ -107,6 +267,12 @@ dependencies = [ "bytes", ] +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + [[package]] name = "funty" version = "1.1.0" @@ -130,6 +296,21 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "half" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62aca2aba2d62b4a7f5b33f3712cb1b0692779a56fb510499d5c0aa594daeaf3" + +[[package]] +name = "hermit-abi" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +dependencies = [ + "libc", +] + [[package]] name = "indoc" version = "1.0.3" @@ -170,6 +351,45 @@ dependencies = [ "syn", ] +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "js-sys" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cfb73131c35423a367daf8cbd24100af0d077668c8c2943f0e7dd775fef0f65" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "lexical-core" version = "0.7.4" @@ -222,6 +442,15 @@ version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +[[package]] +name = "memoffset" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +dependencies = [ + "autocfg", +] + [[package]] name = "ndarray" version = "0.14.0" @@ -275,6 +504,22 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "ordered-float" version = "2.0.1" @@ -315,6 +560,34 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5d65c4d95931acda4498f675e332fcbdc9a06705cd07086c510e9b6009cd1c1" +[[package]] +name = "plotters" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45ca0ae5f169d0917a7c7f5a9c1a3d3d9598f18f529dd2b8373ed988efea307a" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07fffcddc1cb3a1de753caa4e4df03b79922ba43cf882acc1bdd7e8df9f4590" + +[[package]] +name = "plotters-svg" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b38a02e23bd9604b842a812063aec4ef702b57989c37b655254bb61c471ad211" +dependencies = [ + "plotters-backend", +] + [[package]] name = "proc-macro2" version = "1.0.24" @@ -384,18 +657,85 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" +[[package]] +name = "rayon" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +[[package]] +name = "regex" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" +dependencies = [ + "byteorder", +] + +[[package]] +name = "regex-syntax" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + [[package]] name = "ryu" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "sbp" version = "3.4.5" @@ -416,6 +756,59 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" + +[[package]] +name = "serde_cbor" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e18acfa2f90e8b735b2836ab8d538de304cbb6729a7360729ea5a895d15a622" +dependencies = [ + "half", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3fcab8778dc651bc65cfab2e4eb64996f3c912b74002fb379c94517e1f27c46" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea1c6153794552ea7cf7cf63b1231a25de00ec90db326ba6264440fa08e31486" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "smallvec" version = "1.6.1" @@ -445,6 +838,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36474e732d1affd3a6ed582781b3683df3d0563714c59c39591e8ff707cf078e" +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.23" @@ -465,6 +867,22 @@ dependencies = [ "syn", ] +[[package]] +name = "tinytemplate" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2ada8616fad06a2d0c455adc530de4ef57605a8120cc65da9653e0e9623ca74" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + [[package]] name = "unicode-xid" version = "0.2.1" @@ -483,6 +901,81 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +[[package]] +name = "walkdir" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55c0f7123de74f0dab9b7d00fd614e7b19349cd1e2f5252bbe9b1754b59433be" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc45447f0d4573f3d65720f636bbcc3dd6ce920ed704670118650bcd47764c7" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b8853882eef39593ad4174dd26fc9865a64e84026d223f63bb2c42affcbba2c" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4133b5e7f2a531fa413b3a1695e925038a05a71cf67e87dafa295cb645a01385" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4945e4943ae02d15c13962b38a5b1e81eadd4b71214eee75af64a4d6a4fd64" + +[[package]] +name = "web-sys" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c40dc691fc48003eba817c38da7113c15698142da971298003cac3ef175680b3" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" @@ -499,6 +992,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 9bb6d5e30..d5d85bd61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,2 +1,5 @@ [workspace] -members = ["console_backend"] +members = [ + "console_backend", +] + diff --git a/Makefile.toml b/Makefile.toml index 9633f0605..73580bbef 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -1,5 +1,6 @@ [env] -PYTHON_FILES = { script = ["echo $(find . -path ./target -prune -false -o -name '*.py')"] } +PYTHON_FILES = { script = ["python utils/glob_python_files.py"] } +CONDA_ENV = "console_pp" [config] default_to_workspace = false @@ -7,7 +8,7 @@ default_to_workspace = false [tasks.generate-resources] script_runner = "@shell" script = ''' -pyside2-rcc resources/console_resources.qrc -o src/main/python/console_resources.py +conda run -n $CONDA_ENV pyside2-rcc resources/console_resources.qrc -o src/main/python/console_resources.py ''' [tasks.copy-capnp] @@ -18,35 +19,50 @@ cp src/main/resources/base/console_backend.capnp console_backend/console_backend [tasks.run] dependencies = ["copy-capnp", "generate-resources"] +script_runner = "@shell" script = ''' -pip install -e ./console_backend -fbs run +conda run -n $CONDA_ENV pip install -e ./console_backend +conda run -n $CONDA_ENV fbs run ''' [tasks.prod-run] dependencies = ["copy-capnp", "generate-resources"] +script_runner = "@shell" +script = ''' +conda run -n $CONDA_ENV pip install --force ./console_backend +conda run -n $CONDA_ENV fbs run +''' + +[tasks.setuptools-rust] +script_runner = "@shell" script = ''' -pip install --force ./console_backend -fbs run +conda run -n $CONDA_ENV python -m pip install setuptools-rust==0.11.6 +''' + +[tasks.prod-install-backend] +script_runner = "@shell" +script = ''' +conda run -n $CONDA_ENV python -m pip install --force ./console_backend ''' [tasks.prod-freeze] -dependencies = ["copy-capnp", "generate-resources"] +dependencies = ["copy-capnp", "generate-resources", "setuptools-rust", "prod-install-backend"] +script_runner = "@shell" script = ''' -pip install --force ./console_backend -fbs freeze +conda run -n $CONDA_ENV fbs freeze ''' [tasks.prod-installer] dependencies = ["prod-freeze", "remove-negligibles"] +script_runner = "@shell" script = ''' -fbs installer +conda run -n $CONDA_ENV fbs installer ''' [tasks.create-remove-negligibles-file] script_runner = "@shell" script = ''' -python utils/get_fbs_remove_list.py +conda run -n $CONDA_ENV python utils/get_fbs_remove_list.py ''' [tasks.remove-negligibles.linux] @@ -65,22 +81,22 @@ bash src/main/fbs/windows-remove-negligibles.sh dependencies = ["copy-capnp", "generate-resources"] script_runner = "@shell" script = ''' -pip install --force ./console_backend -bash src/main/nuitka/linux.sh +conda run -n $CONDA_ENV pip install --force ./console_backend +conda run -n $CONDA_ENV bash src/main/nuitka/linux.sh ''' [tasks.prod-nuitka-build.mac] dependencies = ["copy-capnp", "generate-resources"] script_runner = "@shell" script = ''' -pip install --force ./console_backend -bash src/main/nuitka/mac.sh +conda run -n $CONDA_ENV pip install --force ./console_backend +conda run -n $CONDA_ENV bash src/main/nuitka/mac.sh ''' [tasks.create-nuitka-script] script_runner = "@shell" script = ''' -python utils/create_nuitka_script.py +conda run -n $CONDA_ENV python utils/create_nuitka_script.py ''' [tasks.prod-installer.mac] @@ -120,17 +136,19 @@ args = ["check"] [tasks.python-type-check] script_runner = "@shell" script = ''' -mypy $PYTHON_FILES +conda run -n $CONDA_ENV mypy $PYTHON_FILES ''' [tasks.python-format-check] +script_runner = "@shell" script = ''' -black --line-length=120 --check --diff $PYTHON_FILES +conda run -n $CONDA_ENV black --line-length=120 --check --diff $PYTHON_FILES ''' [tasks.python-format-all] +script_runner = "@shell" script = ''' -black --line-length=120 $PYTHON_FILES +conda run -n $CONDA_ENV black --line-length=120 $PYTHON_FILES ''' [tasks.python-format] @@ -139,7 +157,7 @@ alias="python-format-all" [tasks.python-lint] script_runner = "@shell" script = ''' -pylint --output-format=parseable $PYTHON_FILES +conda run -n $CONDA_ENV pylint --output-format=parseable $PYTHON_FILES ''' [tasks.format-all] @@ -160,3 +178,16 @@ dependencies = ["python-type-check", "rust-type-check"] [tasks.tests] dependencies = ["rust-tests"] +[tasks.bench-rust] +dependencies = ["copy-capnp", "generate-resources"] +command = "cargo" +args = ["bench", "--all-features", "--", "--verbose", "--noplot"] + +[tasks.bench-rust-validate] +script_runner = "@shell" +script = ''' +conda run -n $CONDA_ENV python console_backend/benches/benchmarks.py +''' + +[tasks.benches] +dependencies = ["bench-rust", "bench-rust-validate"] diff --git a/console_backend/Cargo.toml b/console_backend/Cargo.toml index 9176c5e6e..709e6267d 100644 --- a/console_backend/Cargo.toml +++ b/console_backend/Cargo.toml @@ -12,6 +12,11 @@ sbp = { git = "https://github.com/swift-nav/libsbp.git", rev = "986af3f35033a720 ordered-float = "2.0" ndarray = "0.14.0" glob = "0.3.0" +criterion = "0.3.4" + +[[bench]] +name = "benches" +harness = false [build-dependencies] @@ -19,4 +24,9 @@ capnpc = "0.14" [lib] name = "console_backend" -crate-type = ["cdylib"] +crate-type = ["cdylib", "lib"] +bench = false + +[features] +default = [] +criterion_bench = [] diff --git a/console_backend/benches/benches.rs b/console_backend/benches/benches.rs new file mode 100644 index 000000000..4c85ae12d --- /dev/null +++ b/console_backend/benches/benches.rs @@ -0,0 +1,74 @@ +#![allow(unused_imports)] +use criterion::{criterion_group, criterion_main, Criterion}; +use glob::glob; +use std::{fs, path::Path, sync::mpsc, thread, time}; + +extern crate console_backend; +use console_backend::process_messages; + +const TEST_DATA_DIRECTORY: &str = "./benches/data/"; +const BENCHMARK_TIME_LIMIT: u64 = 10000; +const BENCHMARK_SAMPLE_SIZE: usize = 50; +const FAILURE_CASE_SLEEP_MILLIS: u64 = 1000; +const BENCH_NAME_FAILURE: &str = "RPM_failure"; +const BENCH_NAME_SUCCESS: &str = "RPM_success"; + +pub fn criterion_benchmark(c: &mut Criterion) { + let glob_pattern = Path::new(&TEST_DATA_DIRECTORY).join("*.sbp"); + let glob_pattern = glob_pattern.to_str().unwrap(); + + for ele in glob(glob_pattern).expect("failed to read glob") { + match ele { + Ok(filename) => { + println!("{:?}", filename.display()); + let file_in_name = filename.to_str().unwrap(); + let mut group = c.benchmark_group("proc_messages"); + group.measurement_time(time::Duration::from_millis(BENCHMARK_TIME_LIMIT)); + group.sample_size(BENCHMARK_SAMPLE_SIZE); + group.bench_function(BENCH_NAME_FAILURE, |b| { + b.iter(|| run_process_messages(file_in_name, true)) + }); + group.bench_function(BENCH_NAME_SUCCESS, |b| { + b.iter(|| run_process_messages(file_in_name, false)) + }); + } + Err(e) => println!("{:?}", e), + } + } +} + +fn run_process_messages(file_in_name: &str, failure: bool) { + use std::sync::mpsc::Receiver; + let (client_recv_tx, client_recv_rx) = mpsc::channel::>>(); + let recv_thread = thread::spawn(move || { + let client_recv = client_recv_rx.recv().unwrap(); + let mut iter_count = 0; + loop { + if client_recv.recv().is_err() { + break; + } + iter_count += 1; + } + assert!(iter_count > 0); + }); + { + let (client_send, client_recv) = mpsc::channel::>(); + client_recv_tx + .send(client_recv) + .expect("sending client recv handle should succeed"); + if failure { + thread::sleep(time::Duration::from_millis(FAILURE_CASE_SLEEP_MILLIS)); + } + let messages = sbp::iter_messages(Box::new(fs::File::open(file_in_name).unwrap())); + process_messages::process_messages(messages, client_send); + } + recv_thread.join().expect("join should succeed"); +} + +#[cfg(feature = "criterion_bench")] +criterion_group!(benches, criterion_benchmark); +#[cfg(feature = "criterion_bench")] +criterion_main!(benches); + +#[cfg(not(feature = "criterion_bench"))] +fn main() {} diff --git a/console_backend/benches/benchmarks.py b/console_backend/benches/benchmarks.py new file mode 100644 index 000000000..66c7a9cb4 --- /dev/null +++ b/console_backend/benches/benchmarks.py @@ -0,0 +1,117 @@ +import json +import sys +from typing import Any, Optional + +WINDOWS = "win32" +MACOS = "darwin" +LINUX = "linux" +NAME = "name" +FILE_PATH = "file_path" +KEY_LOCATION = "key_location" +EXPECTED = "expected" +ERROR_MARGIN_FRAC = "error_margin_frac" +SUCCESS = "success" + +RUST_BENCHMARKS = { + WINDOWS: [ + { + NAME: "piksi-relay.sbp", + FILE_PATH: "target/criterion/proc_messages/RPM_success/base/estimates.json", + KEY_LOCATION: "mean.point_estimate", + EXPECTED: 77500000, + ERROR_MARGIN_FRAC: 0.05, + SUCCESS: True, + }, + { + NAME: "piksi-relay.sbp", + FILE_PATH: "target/criterion/proc_messages/RPM_failure/base/estimates.json", + KEY_LOCATION: "mean.point_estimate", + EXPECTED: 77500000, + ERROR_MARGIN_FRAC: 0.05, + SUCCESS: False, + }, + ], + MACOS: [ + { + NAME: "piksi-relay.sbp", + FILE_PATH: "target/criterion/proc_messages/RPM_success/base/estimates.json", + KEY_LOCATION: "mean.point_estimate", + EXPECTED: 77500000, + ERROR_MARGIN_FRAC: 0.05, + SUCCESS: True, + }, + { + NAME: "piksi-relay.sbp", + FILE_PATH: "target/criterion/proc_messages/RPM_failure/base/estimates.json", + KEY_LOCATION: "mean.point_estimate", + EXPECTED: 77500000, + ERROR_MARGIN_FRAC: 0.05, + SUCCESS: False, + }, + ], + LINUX: [ + { + NAME: "piksi-relay.sbp", + FILE_PATH: "target/criterion/proc_messages/RPM_success/base/estimates.json", + KEY_LOCATION: "mean.point_estimate", + EXPECTED: 77500000, + ERROR_MARGIN_FRAC: 0.05, + SUCCESS: True, + }, + { + NAME: "piksi-relay.sbp", + FILE_PATH: "target/criterion/proc_messages/RPM_failure/base/estimates.json", + KEY_LOCATION: "mean.point_estimate", + EXPECTED: 77500000, + ERROR_MARGIN_FRAC: 0.05, + SUCCESS: False, + }, + ], +} + + +def get_nested_key(nested_dict: dict, key_path: str) -> Optional[Any]: + """Extract a key in nested dict/json assuming stringified key_path. + + Assuming `key_path` format: .. + + Args: + nested_dict (dict): The nested dictionary containing the desired key. + key_path (str): The stringified nested dictionary key location. + + Returns: + Optional[Any]: A value corresponding to the key_path in the nested dictionary. + Otherwise, None if not found. + """ + current_key, *next_keys = key_path.split(".", 1) + value = nested_dict.get(current_key, None) + return value if not isinstance(value, dict) and len(next_keys) != 1 else get_nested_key(value, next_keys[0]) + + +def run_validate_benchmarks(): + """Runner for a suite of benchmark validations. + """ + for os_, benchmarks in RUST_BENCHMARKS.items(): + if os_ != sys.platform: + continue + for bench in benchmarks: + with open(bench[FILE_PATH]) as fileo: + bench_result = json.load(fileo) + bench_value = get_nested_key(bench_result, bench[KEY_LOCATION]) + assert bench_value is not None, f"Test:{bench[NAME]} retrieved bench value None." + if bench[SUCCESS]: + assert bench_value - bench[EXPECTED] <= bench[ERROR_MARGIN_FRAC] * bench[EXPECTED], ( + f"Success Test:{bench[NAME]} Bench Value:{bench_value} not within " + f"{bench[ERROR_MARGIN_FRAC]} of {bench[EXPECTED]}." + ) + print(f"PASS - {os_}:{bench[NAME]} MARGIN={bench_value - bench[EXPECTED]}") + else: + assert bench_value - bench[EXPECTED] > bench[ERROR_MARGIN_FRAC] * bench[EXPECTED], ( + f"Failure Test:{bench[NAME]} Bench Value:{bench_value} not outside of " + f"{bench[ERROR_MARGIN_FRAC]} of {bench[EXPECTED]}." + ) + print(f"PASS(Fail Test) - {os_}:{bench[NAME]} MARGIN={bench_value - bench[EXPECTED]}") + + +if __name__ == "__main__": + run_validate_benchmarks() diff --git a/console_backend/benches/data/piksi-relay.sbp b/console_backend/benches/data/piksi-relay.sbp new file mode 100644 index 000000000..cd0d911bf --- /dev/null +++ b/console_backend/benches/data/piksi-relay.sbp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb289e7f109f8b6d99114e5495a1c549c87281ed205932279fbac70642f2e90e +size 257676 diff --git a/console_backend/src/lib.rs b/console_backend/src/lib.rs index 193bfe094..8a164eda7 100644 --- a/console_backend/src/lib.rs +++ b/console_backend/src/lib.rs @@ -1,5 +1,5 @@ +pub mod process_messages; pub mod server; - pub mod console_backend_capnp { include!(concat!(env!("OUT_DIR"), "/console_backend_capnp.rs")); } diff --git a/console_backend/src/process_messages.rs b/console_backend/src/process_messages.rs new file mode 100644 index 000000000..62b13f9bf --- /dev/null +++ b/console_backend/src/process_messages.rs @@ -0,0 +1,166 @@ +use capnp::message::Builder; +use capnp::serialize; +use ordered_float::*; +use sbp::messages::SBP; +use std::sync::mpsc; + +use crate::console_backend_capnp as m; + +pub fn process_messages( + messages: impl Iterator>, + client_send_clone: mpsc::Sender>, +) { + let mut hpoints: Vec<(f64, OrderedFloat)> = vec![]; + let mut vpoints: Vec<(f64, OrderedFloat)> = vec![]; + let mut sat_headers: Vec = vec![]; + let mut sats: Vec)>> = vec![]; + let mut tow: f64 = 0.0; + for message in messages { + match message { + Ok(SBP::MsgMeasurementState(msg)) => { + for state in msg.states { + if state.cn0 != 0 { + let points = + match sat_headers.iter().position(|&ele| ele == state.mesid.sat) { + Some(idx) => sats.get_mut(idx).unwrap(), + _ => { + sat_headers.push(state.mesid.sat); + sats.push(Vec::new()); + sats.last_mut().unwrap() + } + }; + if points.len() >= 200 { + points.remove(0); + } + points.push((tow, OrderedFloat(state.cn0 as f64 / 4.0))); + } + } + let mut builder = Builder::new_default(); + let msg = builder.init_root::(); + + let mut tracking_status = msg.init_tracking_status(); + tracking_status.set_min(0_f64); + tracking_status.set_max(60_f64); + let mut tracking_headers = tracking_status + .reborrow() + .init_headers(sat_headers.len() as u32); + + for (i, header) in sat_headers.iter().enumerate() { + tracking_headers.set(i as u32, *header); + } + + let mut tracking_points = tracking_status + .reborrow() + .init_data(sat_headers.len() as u32); + { + for idx in 0..sat_headers.len() { + let points = sats.get_mut(idx).unwrap(); + let mut point_val_idx = tracking_points + .reborrow() + .init(idx as u32, points.len() as u32); + for (i, (x, OrderedFloat(y))) in points.iter().enumerate() { + let mut point_val = point_val_idx.reborrow().get(i as u32); + point_val.set_x(*x); + point_val.set_y(*y); + } + } + } + let mut msg_bytes: Vec = vec![]; + serialize::write_message(&mut msg_bytes, &builder).unwrap(); + + client_send_clone.send(msg_bytes).unwrap(); + } + Ok(SBP::MsgTrackingState(_msg)) => {} + Ok(SBP::MsgObs(_msg)) => {} + + Ok(SBP::MsgVelNED(velocity_ned)) => { + let n = velocity_ned.n as f64; + let e = velocity_ned.e as f64; + let d = velocity_ned.d as f64; + + let h_vel = f64::sqrt(f64::powi(n, 2) + f64::powi(e, 2)) / 1000.0; + let v_vel = (-1.0 * d) / 1000.0; + + tow = velocity_ned.tow as f64 / 1000.0; + + let mut _min = 0.0; + let mut _max = 1.0; + { + let vmin = vpoints + .iter() + .min_by_key(|i| i.1) + .unwrap_or(&(0.0, OrderedFloat(0.0))); + let vmax = vpoints + .iter() + .max_by_key(|i| i.1) + .unwrap_or(&(1.0, OrderedFloat(0.0))); + let hmin = hpoints + .iter() + .min_by_key(|i| i.1) + .unwrap_or(&(0.0, OrderedFloat(0.0))); + let hmax = hpoints + .iter() + .max_by_key(|i| i.1) + .unwrap_or(&(1.0, OrderedFloat(0.0))); + + if vmin.1.into_inner() < hmin.1.into_inner() { + _min = vmin.1.into_inner(); + } else { + _min = hmin.1.into_inner(); + } + if vmax.1.into_inner() > hmax.1.into_inner() { + _max = vmax.1.into_inner(); + } else { + _max = hmax.1.into_inner(); + } + } + + if hpoints.len() >= 200 { + hpoints.remove(0); + } + if vpoints.len() >= 200 { + vpoints.remove(0); + } + hpoints.push((tow, OrderedFloat(h_vel))); + vpoints.push((tow, OrderedFloat(v_vel))); + + let mut builder = Builder::new_default(); + let msg = builder.init_root::(); + + let mut velocity_status = msg.init_velocity_status(); + + velocity_status.set_min(_min); + velocity_status.set_max(_max); + + { + let mut hvel_points = velocity_status + .reborrow() + .init_hpoints(hpoints.len() as u32); + for (i, (x, OrderedFloat(y))) in hpoints.iter().enumerate() { + let mut point_val = hvel_points.reborrow().get(i as u32); + point_val.set_x(*x); + point_val.set_y(*y); + } + } + { + let mut vvel_points = velocity_status + .reborrow() + .init_vpoints(vpoints.len() as u32); + for (i, (x, OrderedFloat(y))) in vpoints.iter().enumerate() { + let mut point_val = vvel_points.reborrow().get(i as u32); + point_val.set_x(*x); + point_val.set_y(*y); + } + } + + let mut msg_bytes: Vec = vec![]; + serialize::write_message(&mut msg_bytes, &builder).unwrap(); + + client_send_clone.send(msg_bytes).unwrap(); + } + _ => { + // no-op + } + } + } +} diff --git a/console_backend/src/server.rs b/console_backend/src/server.rs index dda0e3cda..76e938c69 100644 --- a/console_backend/src/server.rs +++ b/console_backend/src/server.rs @@ -1,29 +1,21 @@ #![allow(dead_code)] #![allow(unused_imports)] +#![cfg(not(feature = "criterion_bench"))] use capnp::message::Builder; use capnp::serialize; -use ordered_float::*; + use pyo3::exceptions; use pyo3::prelude::*; use pyo3::types::PyBytes; -use sbp::messages::SBP; -use std::collections::HashMap; + use std::fs; use std::io::{BufReader, Cursor}; use std::net::TcpStream; use std::sync::mpsc; use std::thread; -///For tests. -use glob::glob; -use ndarray::{s, Array, Array2, Axis}; -use std::{io::Write, path::Path}; - use crate::console_backend_capnp as m; - -const TEST_DATA_DIRECTORY: &str = "src/test_data/"; -const CSV_EXTENSION: &str = ".csv"; -const ICBINS_POSTFIX: &str = "-icbins"; +use crate::process_messages::process_messages; /// The backend server #[pyclass] @@ -59,165 +51,6 @@ impl ServerEndpoint { } } -pub fn process_messages( - messages: impl Iterator>, - client_send_clone: mpsc::Sender>, -) { - let mut hpoints: Vec<(f64, OrderedFloat)> = vec![]; - let mut vpoints: Vec<(f64, OrderedFloat)> = vec![]; - let mut sat_headers: Vec = vec![]; - let mut sats: Vec)>> = vec![]; - let mut tow: f64 = 0.0; - for message in messages { - match message { - Ok(SBP::MsgMeasurementState(msg)) => { - for state in msg.states { - if state.cn0 != 0 { - let points = - match sat_headers.iter().position(|&ele| ele == state.mesid.sat) { - Some(idx) => sats.get_mut(idx).unwrap(), - _ => { - sat_headers.push(state.mesid.sat); - sats.push(Vec::new()); - sats.last_mut().unwrap() - } - }; - if points.len() >= 200 { - points.remove(0); - } - points.push((tow, OrderedFloat(state.cn0 as f64 / 4.0))); - } - } - let mut builder = Builder::new_default(); - let msg = builder.init_root::(); - - let mut tracking_status = msg.init_tracking_status(); - tracking_status.set_min(0_f64); - tracking_status.set_max(60_f64); - let mut tracking_headers = tracking_status - .reborrow() - .init_headers(sat_headers.len() as u32); - - for (i, header) in sat_headers.iter().enumerate() { - tracking_headers.set(i as u32, *header); - } - - let mut tracking_points = tracking_status - .reborrow() - .init_data(sat_headers.len() as u32); - { - for idx in 0..sat_headers.len() { - let points = sats.get_mut(idx).unwrap(); - let mut point_val_idx = tracking_points - .reborrow() - .init(idx as u32, points.len() as u32); - for (i, (x, OrderedFloat(y))) in points.iter().enumerate() { - let mut point_val = point_val_idx.reborrow().get(i as u32); - point_val.set_x(*x); - point_val.set_y(*y); - } - } - } - let mut msg_bytes: Vec = vec![]; - serialize::write_message(&mut msg_bytes, &builder).unwrap(); - - client_send_clone.send(msg_bytes).unwrap(); - } - Ok(SBP::MsgTrackingState(_msg)) => {} - Ok(SBP::MsgObs(_msg)) => {} - - Ok(SBP::MsgVelNED(velocity_ned)) => { - let n = velocity_ned.n as f64; - let e = velocity_ned.e as f64; - let d = velocity_ned.d as f64; - - let h_vel = f64::sqrt(f64::powi(n, 2) + f64::powi(e, 2)) / 1000.0; - let v_vel = (-1.0 * d) / 1000.0; - - tow = velocity_ned.tow as f64 / 1000.0; - - let mut _min = 0.0; - let mut _max = 1.0; - { - let vmin = vpoints - .iter() - .min_by_key(|i| i.1) - .unwrap_or(&(0.0, OrderedFloat(0.0))); - let vmax = vpoints - .iter() - .max_by_key(|i| i.1) - .unwrap_or(&(1.0, OrderedFloat(0.0))); - let hmin = hpoints - .iter() - .min_by_key(|i| i.1) - .unwrap_or(&(0.0, OrderedFloat(0.0))); - let hmax = hpoints - .iter() - .max_by_key(|i| i.1) - .unwrap_or(&(1.0, OrderedFloat(0.0))); - - if vmin.1.into_inner() < hmin.1.into_inner() { - _min = vmin.1.into_inner(); - } else { - _min = hmin.1.into_inner(); - } - if vmax.1.into_inner() > hmax.1.into_inner() { - _max = vmax.1.into_inner(); - } else { - _max = hmax.1.into_inner(); - } - } - - if hpoints.len() >= 200 { - hpoints.remove(0); - } - if vpoints.len() >= 200 { - vpoints.remove(0); - } - hpoints.push((tow, OrderedFloat(h_vel))); - vpoints.push((tow, OrderedFloat(v_vel))); - - let mut builder = Builder::new_default(); - let msg = builder.init_root::(); - - let mut velocity_status = msg.init_velocity_status(); - - velocity_status.set_min(_min); - velocity_status.set_max(_max); - - { - let mut hvel_points = velocity_status - .reborrow() - .init_hpoints(hpoints.len() as u32); - for (i, (x, OrderedFloat(y))) in hpoints.iter().enumerate() { - let mut point_val = hvel_points.reborrow().get(i as u32); - point_val.set_x(*x); - point_val.set_y(*y); - } - } - { - let mut vvel_points = velocity_status - .reborrow() - .init_vpoints(vpoints.len() as u32); - for (i, (x, OrderedFloat(y))) in vpoints.iter().enumerate() { - let mut point_val = vvel_points.reborrow().get(i as u32); - point_val.set_x(*x); - point_val.set_y(*y); - } - } - - let mut msg_bytes: Vec = vec![]; - serialize::write_message(&mut msg_bytes, &builder).unwrap(); - - client_send_clone.send(msg_bytes).unwrap(); - } - _ => { - // no-op - } - } - } -} - #[cfg(not(test))] #[pymethods] impl Server { @@ -316,39 +149,10 @@ impl Server { Ok(server_endpoint) } } -#[cfg(not(test))] + #[pymodule] pub fn server(_py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; Ok(()) } - -#[test] -fn test_readfile() { - let glob_pattern = Path::new(&TEST_DATA_DIRECTORY).join("**/*.sbp"); - let glob_pattern = match glob_pattern.to_str() { - Some(i) => i, - _ => "", - }; - for ele in glob(glob_pattern).expect("failed to read glob") { - match ele { - Ok(filename) => { - println!("{:?}", filename.display()); - let file_in_name = &filename; - let file_out_name = file_in_name - .parent() - .unwrap() - .join(file_in_name.file_stem().unwrap()); - let file_out_name = file_out_name.to_str().unwrap(); - let _file_out_orig_name = format!("{}{}", file_out_name, &CSV_EXTENSION); - let _file_out_name = - format!("{}{}{}", file_out_name, &ICBINS_POSTFIX, &CSV_EXTENSION); - let (client_send, _) = mpsc::channel::>(); - let messages = sbp::iter_messages(Box::new(fs::File::open(file_in_name).unwrap())); - process_messages(messages, client_send); - } - Err(e) => println!("{:?}", e), - } - } -} diff --git a/utils/glob_python_files.py b/utils/glob_python_files.py new file mode 100644 index 000000000..11e251a66 --- /dev/null +++ b/utils/glob_python_files.py @@ -0,0 +1,3 @@ +import glob + +print(str.join(" ", glob.glob("**/*.py")))