From 07492ba32d516510e7733d1ac83370441bce1021 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 11 Sep 2025 12:43:48 -0400 Subject: [PATCH 01/11] feat(deps): FIPS --- Cargo.lock | 196 ++++++++++++++++-- Cargo.toml | 4 +- scripts/environment/bootstrap-macos.sh | 4 + scripts/environment/bootstrap-ubuntu-24.04.sh | 3 +- .../environment/bootstrap-windows-2025.ps1 | 5 + 5 files changed, 195 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8248a5f51f919..264d3c9b6bbe3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -628,6 +628,7 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08f6da6d49a956424ca4e28fe93656f790d748b469eaccbc7488fec545315180" dependencies = [ + "aws-lc-rs", "base64 0.22.1", "bytes 1.10.1", "futures 0.3.31", @@ -639,7 +640,6 @@ dependencies = [ "portable-atomic", "rand 0.8.5", "regex", - "ring", "rustls-native-certs 0.7.0", "rustls-pemfile 2.1.0", "rustls-webpki 0.102.8", @@ -829,6 +829,45 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-lc-fips-sys" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2608e5a7965cc9d58c56234d346c9c89b824c4c8652b6f047b3bd0a777c0644f" +dependencies = [ + "bindgen 0.69.5", + "cc", + "cmake", + "dunce", + "fs_extra", + "regex", +] + +[[package]] +name = "aws-lc-rs" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94b8ff6c09cd57b16da53641caa860168b88c172a5ee163b0288d3d6eea12786" +dependencies = [ + "aws-lc-fips-sys", + "aws-lc-sys", + "untrusted 0.7.1", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e44d16778acaf6a9ec9899b92cebd65580b83f685446bf2e1f5d3d732f99dcd" +dependencies = [ + "bindgen 0.72.1", + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "aws-runtime" version = "1.5.10" @@ -1721,6 +1760,69 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.9.0", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "log", + "prettyplease 0.2.15", + "proc-macro2 1.0.101", + "quote 1.0.40", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.106", + "which 4.4.2", +] + +[[package]] +name = "bindgen" +version = "0.71.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" +dependencies = [ + "bitflags 2.9.0", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease 0.2.15", + "proc-macro2 1.0.101", + "quote 1.0.40", + "regex", + "rustc-hash 2.1.1", + "shlex", + "syn 2.0.106", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.9.0", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease 0.2.15", + "proc-macro2 1.0.101", + "quote 1.0.40", + "regex", + "rustc-hash 2.1.1", + "shlex", + "syn 2.0.106", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -2064,10 +2166,11 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.15" +version = "1.2.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c736e259eea577f443d5c86c304f9f4ae0295c43f3ba05c21f1d66b5f06001af" +checksum = "5252b3d2648e5eedbc1a6f501e3c795e07025c1e93bbf8bbdd6eef7f447a6d54" dependencies = [ + "find-msvc-tools", "jobserver", "libc", "shlex", @@ -2079,6 +2182,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "cfb-mode" version = "0.8.2" @@ -2204,6 +2316,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.47" @@ -2286,9 +2409,9 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.50" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" dependencies = [ "cc", ] @@ -3836,6 +3959,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" + [[package]] name = "finl_unicode" version = "1.2.0" @@ -3951,6 +4080,12 @@ dependencies = [ "num", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -5876,6 +6011,12 @@ dependencies = [ "spin 0.5.2", ] +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.175" @@ -5920,6 +6061,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libloading" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" +dependencies = [ + "cfg-if", + "windows-targets 0.53.2", +] + [[package]] name = "libm" version = "0.2.8" @@ -6368,7 +6519,7 @@ dependencies = [ "mlua_derive", "num-traits", "parking_lot 0.12.4", - "rustc-hash", + "rustc-hash 2.1.1", "rustversion", ] @@ -8321,7 +8472,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.1", "rustls 0.23.23", "socket2 0.5.10", "thiserror 2.0.3", @@ -8339,7 +8490,7 @@ dependencies = [ "getrandom 0.2.15", "rand 0.8.5", "ring", - "rustc-hash", + "rustc-hash 2.1.1", "rustls 0.23.23", "rustls-pki-types", "slab", @@ -8984,7 +9135,7 @@ dependencies = [ "cfg-if", "getrandom 0.2.15", "libc", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -9158,9 +9309,15 @@ checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" [[package]] name = "rustc-hash" -version = "2.0.0" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hash" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" @@ -9262,6 +9419,7 @@ version = "0.23.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" dependencies = [ + "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", @@ -9342,7 +9500,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ "ring", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -9351,9 +9509,10 @@ version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -9491,7 +9650,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ "ring", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -11054,6 +11213,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f591660438b3038dd04d16c938271c79e7e06260ad2ea2885a4861bfb238605d" dependencies = [ + "aws-lc-rs", "base64 0.22.1", "bytes 1.10.1", "futures-core", @@ -11061,7 +11221,6 @@ dependencies = [ "http 1.3.1", "httparse", "rand 0.8.5", - "ring", "rustls-pki-types", "tokio", "tokio-rustls 0.26.2", @@ -11870,6 +12029,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -12048,6 +12213,7 @@ dependencies = [ "azure_storage", "azure_storage_blobs", "base64 0.22.1", + "bindgen 0.71.1", "bloomy", "bollard", "byteorder", diff --git a/Cargo.toml b/Cargo.toml index 2ec3bff5fb8cc..9a59aab9253af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -138,6 +138,7 @@ members = [ anyhow = { version = "1.0.99", default-features = false, features = ["std"] } async-stream = { version = "0.3.6", default-features = false } async-trait = { version = "0.1.89", default-features = false } +aws-lc-sys = { version = "0.31.0", features = ["bindgen"] } # Required for FIPS enabled cross compilation bytes = { version = "1.10.1", default-features = false, features = ["serde"] } base64 = { version = "0.22.1", default-features = false } cfg-if = { version = "1.0.3", default-features = false } @@ -386,7 +387,7 @@ lru = { version = "0.16.0", default-features = false } maxminddb = { version = "0.26.0", default-features = false, optional = true, features = ["simdutf8"] } md-5 = { version = "0.10", default-features = false, optional = true } mongodb = { version = "2.8.2", default-features = false, features = ["tokio-runtime"], optional = true } -async-nats = { version = "0.42.0", default-features = false, optional = true, features = ["ring"] } +async-nats = { version = "0.42.0", default-features = false, optional = true, features = ["fips"] } nkeys = { version = "0.4.5", default-features = false, optional = true } nom = { workspace = true, optional = true } notify = { version = "8.1.0", default-features = false, features = ["macos_fsevent"] } @@ -447,6 +448,7 @@ netlink-packet-core = "0.7.0" netlink-sys = { version = "0.8.7", features = ["tokio_socket"] } [build-dependencies] +bindgen = { version = "0.71.1" } prost-build = { workspace = true, optional = true } tonic-build = { workspace = true, optional = true } # update 'openssl_version' in website/config.toml whenever version changes diff --git a/scripts/environment/bootstrap-macos.sh b/scripts/environment/bootstrap-macos.sh index 5cf014b31546c..b5c7e2f1b4834 100755 --- a/scripts/environment/bootstrap-macos.sh +++ b/scripts/environment/bootstrap-macos.sh @@ -4,6 +4,10 @@ set -e -o verbose brew update brew install ruby@3 coreutils cue-lang/tap/cue protobuf +# Required for building aws-lc-rs +# https://github.com/aws/aws-lc/issues/2129 +brew install go + gem install bundler echo "export PATH=\"/usr/local/opt/ruby/bin:\$PATH\"" >> "$HOME/.bash_profile" diff --git a/scripts/environment/bootstrap-ubuntu-24.04.sh b/scripts/environment/bootstrap-ubuntu-24.04.sh index 664efef7b68ab..fa887011d148c 100755 --- a/scripts/environment/bootstrap-ubuntu-24.04.sh +++ b/scripts/environment/bootstrap-ubuntu-24.04.sh @@ -47,7 +47,8 @@ apt-get install --yes --no-install-recommends \ shellcheck \ sudo \ unzip \ - wget + wget \ + golang-go # required by aws-lc-rs - # https://github.com/aws/aws-lc/issues/2129 # Cue TEMP=$(mktemp -d) diff --git a/scripts/environment/bootstrap-windows-2025.ps1 b/scripts/environment/bootstrap-windows-2025.ps1 index 7af1123b2df35..d7ac01f3a0b42 100644 --- a/scripts/environment/bootstrap-windows-2025.ps1 +++ b/scripts/environment/bootstrap-windows-2025.ps1 @@ -19,6 +19,11 @@ $env:NUGET_ENABLE_ENHANCED_HTTP_RETRY = "true" choco install make choco install protoc +# required by aws-lc-rs +# https://github.com/aws/aws-lc/issues/2129 +choco install ninja +choco install nasm + # Set a specific override path for libclang. echo "LIBCLANG_PATH=$( (gcm clang).source -replace "clang.exe" )" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append From 4feebf0a025bf42f2e2b33872a9fa89c1f86b85b Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 11 Sep 2025 12:44:52 -0400 Subject: [PATCH 02/11] chore(dev): cargo vdev build licenses --- LICENSE-3rdparty.csv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 36943f20cb2e0..06a91844da60b 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -49,6 +49,9 @@ atoi,https://github.com/pacman82/atoi-rs,MIT,Markus Klein atomic-waker,https://github.com/smol-rs/atomic-waker,Apache-2.0 OR MIT,"Stjepan Glavina , Contributors to futures-rs" aws-config,https://github.com/smithy-lang/smithy-rs,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-credential-types,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rust SDK Team +aws-lc-fips-sys,https://github.com/aws/aws-lc-rs,ISC AND (Apache-2.0 OR ISC) AND OpenSSL,AWS-LC +aws-lc-rs,https://github.com/aws/aws-lc-rs,ISC AND (Apache-2.0 OR ISC),AWS-LibCrypto +aws-lc-sys,https://github.com/aws/aws-lc-rs,ISC AND (Apache-2.0 OR ISC) AND OpenSSL,AWS-LC aws-runtime,https://github.com/smithy-lang/smithy-rs,Apache-2.0,AWS Rust SDK Team aws-sdk-cloudwatch,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " aws-sdk-cloudwatchlogs,https://github.com/awslabs/aws-sdk-rust,Apache-2.0,"AWS Rust SDK Team , Russell Cohen " From 0ce09dfc6c2e99e0c3c179c1ca2b754801e8417d Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 11 Sep 2025 12:55:18 -0400 Subject: [PATCH 03/11] major additions to scripts/cross/bootstrap-ubuntu.sh --- scripts/cross/bootstrap-ubuntu.sh | 41 +++++++++++++++++++++++++++---- scripts/environment/prepare.sh | 8 ++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/scripts/cross/bootstrap-ubuntu.sh b/scripts/cross/bootstrap-ubuntu.sh index cf053aa365b20..219c133302af2 100755 --- a/scripts/cross/bootstrap-ubuntu.sh +++ b/scripts/cross/bootstrap-ubuntu.sh @@ -18,10 +18,41 @@ EOF wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key| apt-key add - +# onig_sys and aws-lc-rs dependencies apt-get update - -# needed by onig_sys apt-get install -y \ - libclang1-9 \ - llvm-9 \ - unzip + gcc-arm-linux-gnueabihf \ + g++-arm-linux-gnueabihf \ + gcc-aarch64-linux-gnu \ + g++-aarch64-linux-gnu \ + libc6-dev-armhf-cross \ + libc6-dev-arm64-cross \ + clang \ + cmake \ + libssl-dev \ + libclang-dev \ + libsasl2-dev \ + llvm \ + unzip + +# Required by the `rdkafka-sys` Rust dependency +ZLIB_VERSION=1.3.1 +wget https://www.zlib.net/zlib-${ZLIB_VERSION}.tar.gz +tar xzvf zlib-${ZLIB_VERSION}.tar.gz +cd zlib-${ZLIB_VERSION} +./configure +make +make install + +# Go installation is required for building aws-lc-rs +# https://github.com/aws/aws-lc/issues/2129 +GO_VERSION="1.24.0" +GO_TAR_FILE="go${GO_VERSION}.linux-amd64.tar.gz" +wget https://go.dev/dl/${GO_TAR_FILE} +tar -C /usr/local -xzf ${GO_TAR_FILE} +rm ${GO_TAR_FILE} +ln -s /usr/local/go/bin/go /usr/local/bin/go + +scripts/environment/prepare.sh --modules=rustup,bindgen +ln -s "$(dirname "$(which cargo)")/"* /usr/local/bin/ +./pre diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index 2acae5859dfbc..ada9f857e5ee9 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -14,6 +14,7 @@ ALL_MODULES=( datadog-ci release-flags vdev + bindgen ) # By default, install everything @@ -51,6 +52,7 @@ Modules: markdownlint datadog-ci vdev + bindgen If a module requires rust then rustup will be automatically installed. By default, all modules are installed. To install only a subset: @@ -177,3 +179,9 @@ if contains_module vdev; then rustup run stable cargo "${install[@]}" vdev --version 0.1.0 --force --locked fi fi + +if contains_module bindgen; then + if ! bindgen --version 2>/dev/null | grep -q '^bindgen v0.72.1'; then + rustup run stable cargo "${install[@]}" bindgen-cli --version v0.72.1 --force --locked + fi +fi From 28bbbad4976e344304f610055299e577e9cb4a8d Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 11 Sep 2025 13:02:13 -0400 Subject: [PATCH 04/11] bump bindgen --- Cargo.lock | 22 +--------------------- Cargo.toml | 2 +- scripts/cross/bootstrap-ubuntu.sh | 3 +-- 3 files changed, 3 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 264d3c9b6bbe3..879b7a29da718 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1783,26 +1783,6 @@ dependencies = [ "which 4.4.2", ] -[[package]] -name = "bindgen" -version = "0.71.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" -dependencies = [ - "bitflags 2.9.0", - "cexpr", - "clang-sys", - "itertools 0.13.0", - "log", - "prettyplease 0.2.15", - "proc-macro2 1.0.101", - "quote 1.0.40", - "regex", - "rustc-hash 2.1.1", - "shlex", - "syn 2.0.106", -] - [[package]] name = "bindgen" version = "0.72.1" @@ -12213,7 +12193,7 @@ dependencies = [ "azure_storage", "azure_storage_blobs", "base64 0.22.1", - "bindgen 0.71.1", + "bindgen 0.72.1", "bloomy", "bollard", "byteorder", diff --git a/Cargo.toml b/Cargo.toml index 9a59aab9253af..a41ad8b275dcd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -448,7 +448,7 @@ netlink-packet-core = "0.7.0" netlink-sys = { version = "0.8.7", features = ["tokio_socket"] } [build-dependencies] -bindgen = { version = "0.71.1" } +bindgen = { version = "0.72.1" } prost-build = { workspace = true, optional = true } tonic-build = { workspace = true, optional = true } # update 'openssl_version' in website/config.toml whenever version changes diff --git a/scripts/cross/bootstrap-ubuntu.sh b/scripts/cross/bootstrap-ubuntu.sh index 219c133302af2..1b9aec423f89a 100755 --- a/scripts/cross/bootstrap-ubuntu.sh +++ b/scripts/cross/bootstrap-ubuntu.sh @@ -53,6 +53,5 @@ tar -C /usr/local -xzf ${GO_TAR_FILE} rm ${GO_TAR_FILE} ln -s /usr/local/go/bin/go /usr/local/bin/go -scripts/environment/prepare.sh --modules=rustup,bindgen +../environment/prepare.sh --modules=rustup,bindgen ln -s "$(dirname "$(which cargo)")/"* /usr/local/bin/ -./pre From 9bd05c0dbf4824ee29b12a3eeb73d4f9c4daeb12 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 11 Sep 2025 15:46:07 -0400 Subject: [PATCH 05/11] copy env scripts --- scripts/cross/Dockerfile | 2 +- scripts/cross/bootstrap-ubuntu.sh | 2 +- scripts/environment/prepare.sh | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/cross/Dockerfile b/scripts/cross/Dockerfile index 80a75caba683c..2da07d2f59362 100644 --- a/scripts/cross/Dockerfile +++ b/scripts/cross/Dockerfile @@ -5,7 +5,7 @@ FROM ghcr.io/cross-rs/${TARGET}:${CROSS_VERSION} # Common steps for all targets COPY scripts/cross/bootstrap-ubuntu.sh / -COPY scripts/environment/install-protoc.sh / +COPY scripts/environment/ /scripts/environment/ RUN /bootstrap-ubuntu.sh && bash /install-protoc.sh # Relocate libstdc++ for musl targets that need it (TODO: investigate if still required) diff --git a/scripts/cross/bootstrap-ubuntu.sh b/scripts/cross/bootstrap-ubuntu.sh index 1b9aec423f89a..54f7d9d19fb3e 100755 --- a/scripts/cross/bootstrap-ubuntu.sh +++ b/scripts/cross/bootstrap-ubuntu.sh @@ -53,5 +53,5 @@ tar -C /usr/local -xzf ${GO_TAR_FILE} rm ${GO_TAR_FILE} ln -s /usr/local/go/bin/go /usr/local/bin/go -../environment/prepare.sh --modules=rustup,bindgen +/scripts/environment/prepare.sh --modules=rustup,bindgen ln -s "$(dirname "$(which cargo)")/"* /usr/local/bin/ diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index ada9f857e5ee9..8cf814d392931 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -euo pipefail +SCRIPT_DIR=$(realpath "$(dirname "${BASH_SOURCE[0]}")") + ALL_MODULES=( rustup cargo-deb @@ -106,13 +108,13 @@ fi install=(install) if contains_module rustup; then - . scripts/environment/release-flags.sh + . "${SCRIPT_DIR}"/release-flags.sh rustup show active-toolchain || rustup toolchain install stable rustup show if [ "${require_binstall}" = "true" ]; then - if cargo binstall -V &>/dev/null || ./scripts/environment/binstall.sh; then + if cargo binstall -V &>/dev/null || "${SCRIPT_DIR}"/binstall.sh; then install=(binstall -y) else echo "Failed to install cargo binstall, defaulting to cargo install" From fdf02b543e2e0d529a1f4312555ae0d8ef88851e Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 11 Sep 2025 16:26:08 -0400 Subject: [PATCH 06/11] install rustup --- scripts/environment/prepare.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index 8cf814d392931..eb5b92dd12e09 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -108,6 +108,11 @@ fi install=(install) if contains_module rustup; then + if ! command -v rustup >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + source "$HOME/.cargo/env" + fi + . "${SCRIPT_DIR}"/release-flags.sh rustup show active-toolchain || rustup toolchain install stable From 1813bee82796efbccf1686fb0bb5b81fb27fabf2 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 12 Sep 2025 12:39:33 -0400 Subject: [PATCH 07/11] update go and test SHA - also use prepare in integration/Dockerfile --- scripts/cross/bootstrap-ubuntu.sh | 4 +++- scripts/integration/Dockerfile | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/cross/bootstrap-ubuntu.sh b/scripts/cross/bootstrap-ubuntu.sh index 54f7d9d19fb3e..8ca07680ba61b 100755 --- a/scripts/cross/bootstrap-ubuntu.sh +++ b/scripts/cross/bootstrap-ubuntu.sh @@ -46,9 +46,11 @@ make install # Go installation is required for building aws-lc-rs # https://github.com/aws/aws-lc/issues/2129 -GO_VERSION="1.24.0" +GO_VERSION="1.24.7" +SHA="da18191ddb7db8a9339816f3e2b54bdded8047cdc2a5d67059478f8d1595c43f" GO_TAR_FILE="go${GO_VERSION}.linux-amd64.tar.gz" wget https://go.dev/dl/${GO_TAR_FILE} +echo "${SHA} ${GO_TAR_FILE}" | sha256sum -c - tar -C /usr/local -xzf ${GO_TAR_FILE} rm ${GO_TAR_FILE} ln -s /usr/local/go/bin/go /usr/local/bin/go diff --git a/scripts/integration/Dockerfile b/scripts/integration/Dockerfile index fdf6188178801..187f6437e301f 100644 --- a/scripts/integration/Dockerfile +++ b/scripts/integration/Dockerfile @@ -6,17 +6,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ cmake \ curl \ g++ \ + git \ + golang-go \ libclang1 \ libsasl2-dev \ libssl-dev \ llvm \ pkg-config \ - zlib1g-dev \ unzip \ - git \ + zlib1g-dev \ && rm -rf /var/lib/apt/lists/* -RUN cargo install cargo-nextest --version 0.9.95 --locked +COPY scripts/environment/ /scripts/environment/ +RUN bash /scripts/environment/prepare.sh --modules=cargo-nextest,bindgen COPY scripts/environment/install-protoc.sh / COPY tests/data/ca/certs /certs From cf2b103fab6c3e101dd464efa9cf8f870e1e1071 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 12 Sep 2025 12:40:43 -0400 Subject: [PATCH 08/11] fix bindgen version --- scripts/environment/prepare.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index eb5b92dd12e09..d14c9ab3a2a46 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -188,7 +188,7 @@ if contains_module vdev; then fi if contains_module bindgen; then - if ! bindgen --version 2>/dev/null | grep -q '^bindgen v0.72.1'; then - rustup run stable cargo "${install[@]}" bindgen-cli --version v0.72.1 --force --locked + if ! bindgen --version 2>/dev/null | grep -q '^bindgen 0.72.1'; then + rustup run stable cargo "${install[@]}" bindgen-cli --version 0.72.1 --force --locked fi fi From 3f0d1152e42677aaa0d3a3dbd38b3898356e2223 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 12 Sep 2025 13:14:42 -0400 Subject: [PATCH 09/11] fixing prepare.sh --- scripts/environment/prepare.sh | 53 ++++++++++++++++++++++++---------- scripts/integration/Dockerfile | 2 +- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index d14c9ab3a2a46..2f8d8dbd1310b 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -1,6 +1,33 @@ #!/usr/bin/env bash set -euo pipefail +ensure_active_toolchain_is_installed() { + if ! command -v rustup >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + fi + + # Ensure cargo/rustup are on PATH even if rustup was preinstalled in the image + if [ -f "${HOME}/.cargo/env" ]; then + source "${HOME}/.cargo/env" + fi + + # Determine desired toolchain and ensure it's installed (simple & readable). + ACTIVE_TOOLCHAIN="$(rustup show active-toolchain 2>/dev/null || true)" + ACTIVE_TOOLCHAIN="${ACTIVE_TOOLCHAIN%% *}" # keep only the first token + + if [ -z "${ACTIVE_TOOLCHAIN}" ]; then + # No active toolchain yet: fall back to env override or stable, then make it default. + ACTIVE_TOOLCHAIN="${RUSTUP_TOOLCHAIN:-stable}" + rustup toolchain install "${ACTIVE_TOOLCHAIN}" + rustup default "${ACTIVE_TOOLCHAIN}" + else + # Ensure the active (possibly from rust-toolchain.toml) exists. Idempotent. + rustup toolchain install "${ACTIVE_TOOLCHAIN}" || rustup toolchain install "${ACTIVE_TOOLCHAIN%%-*}" + fi + + rustup show +} + SCRIPT_DIR=$(realpath "$(dirname "${BASH_SOURCE[0]}")") ALL_MODULES=( @@ -108,15 +135,9 @@ fi install=(install) if contains_module rustup; then - if ! command -v rustup >/dev/null 2>&1; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - source "$HOME/.cargo/env" - fi - . "${SCRIPT_DIR}"/release-flags.sh - rustup show active-toolchain || rustup toolchain install stable - rustup show + ensure_active_toolchain_is_installed if [ "${require_binstall}" = "true" ]; then if cargo binstall -V &>/dev/null || "${SCRIPT_DIR}"/binstall.sh; then @@ -129,43 +150,43 @@ fi set -e -o verbose if contains_module cargo-deb; then if [[ "$(cargo-deb --version 2>/dev/null)" != "2.9.3" ]]; then - rustup run stable cargo "${install[@]}" cargo-deb --version 2.9.3 --force --locked + cargo "${install[@]}" cargo-deb --version 2.9.3 --force --locked fi fi if contains_module cross; then if ! cross --version 2>/dev/null | grep -q '^cross 0.2.5'; then - rustup run stable cargo "${install[@]}" cross --version 0.2.5 --force --locked + cargo "${install[@]}" cross --version 0.2.5 --force --locked fi fi if contains_module cargo-nextest; then if ! cargo-nextest --version 2>/dev/null | grep -q '^cargo-nextest 0.9.95'; then - rustup run stable cargo "${install[@]}" cargo-nextest --version 0.9.95 --force --locked + cargo "${install[@]}" cargo-nextest --version 0.9.95 --force --locked fi fi if contains_module cargo-deny; then if ! cargo-deny --version 2>/dev/null | grep -q '^cargo-deny 0.16.2'; then - rustup run stable cargo "${install[@]}" cargo-deny --version 0.16.2 --force --locked + cargo "${install[@]}" cargo-deny --version 0.16.2 --force --locked fi fi if contains_module cargo-msrv; then if ! cargo-msrv --version 2>/dev/null | grep -q '^cargo-msrv 0.18.4'; then - rustup run stable cargo "${install[@]}" cargo-msrv --version 0.18.4 --force --locked + cargo "${install[@]}" cargo-msrv --version 0.18.4 --force --locked fi fi if contains_module dd-rust-license-tool; then if ! dd-rust-license-tool --help &>/dev/null; then - rustup run stable cargo install dd-rust-license-tool --version 1.0.2 --force --locked + cargo install dd-rust-license-tool --version 1.0.2 --force --locked fi fi if contains_module wasm-pack; then if ! wasm-pack --version | grep -q '^wasm-pack 0.13.1'; then - rustup run stable cargo "${install[@]}" --locked --version 0.13.1 wasm-pack + cargo "${install[@]}" --locked --version 0.13.1 wasm-pack fi fi @@ -183,12 +204,12 @@ fi if contains_module vdev; then if [[ "$(vdev --version 2>/dev/null)" != "vdev 0.1.0" ]]; then - rustup run stable cargo "${install[@]}" vdev --version 0.1.0 --force --locked + cargo "${install[@]}" vdev --version 0.1.0 --force --locked fi fi if contains_module bindgen; then if ! bindgen --version 2>/dev/null | grep -q '^bindgen 0.72.1'; then - rustup run stable cargo "${install[@]}" bindgen-cli --version 0.72.1 --force --locked + cargo "${install[@]}" bindgen-cli --version 0.72.1 --force --locked fi fi diff --git a/scripts/integration/Dockerfile b/scripts/integration/Dockerfile index 187f6437e301f..c9c204423d794 100644 --- a/scripts/integration/Dockerfile +++ b/scripts/integration/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* COPY scripts/environment/ /scripts/environment/ -RUN bash /scripts/environment/prepare.sh --modules=cargo-nextest,bindgen +RUN bash /scripts/environment/prepare.sh --modules=rustup,cargo-nextest,bindgen COPY scripts/environment/install-protoc.sh / COPY tests/data/ca/certs /certs From 0c3b4eaa53cc52f8eff89d6866dff191aa12ef20 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 12 Sep 2025 16:18:57 -0400 Subject: [PATCH 10/11] fix script path --- scripts/cross/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cross/Dockerfile b/scripts/cross/Dockerfile index 2da07d2f59362..3e2fa10d9e9bd 100644 --- a/scripts/cross/Dockerfile +++ b/scripts/cross/Dockerfile @@ -6,7 +6,7 @@ FROM ghcr.io/cross-rs/${TARGET}:${CROSS_VERSION} # Common steps for all targets COPY scripts/cross/bootstrap-ubuntu.sh / COPY scripts/environment/ /scripts/environment/ -RUN /bootstrap-ubuntu.sh && bash /install-protoc.sh +RUN /bootstrap-ubuntu.sh && bash /scripts/environment/install-protoc.sh # Relocate libstdc++ for musl targets that need it (TODO: investigate if still required) RUN if [ "$TARGET" = "arm-unknown-linux-musleabi" ]; then \ From 8655ec7b43b85347bc2e4ee4d527871ca0dcd90f Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 15 Sep 2025 10:11:45 -0400 Subject: [PATCH 11/11] stared work to split into two builds --- Cargo.lock | 2 ++ Cargo.toml | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 879b7a29da718..65bc47c4a3dd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12167,6 +12167,8 @@ dependencies = [ "async-trait", "aws-config", "aws-credential-types", + "aws-lc-rs", + "aws-lc-sys", "aws-runtime", "aws-sdk-cloudwatch", "aws-sdk-cloudwatchlogs", diff --git a/Cargo.toml b/Cargo.toml index 34448e41b0e8a..b8dda883e92cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -138,6 +138,7 @@ members = [ anyhow = { version = "1.0.99", default-features = false, features = ["std"] } async-stream = { version = "0.3.6", default-features = false } async-trait = { version = "0.1.89", default-features = false } +aws-lc-rs = { version = "1.14.0", features = ["bindgen"] } aws-lc-sys = { version = "0.31.0", features = ["bindgen"] } # Required for FIPS enabled cross compilation bytes = { version = "1.10.1", default-features = false, features = ["serde"] } base64 = { version = "0.22.1", default-features = false } @@ -199,6 +200,9 @@ vector-config-macros = { path = "lib/vector-config-macros" } vrl = { git = "https://github.com/vectordotdev/vrl.git", branch = "main", features = ["arbitrary", "cli", "test", "test_framework"] } [dependencies] + +aws-lc-rs = { workspace = true, optional = true } +aws-lc-sys = { workspace = true, optional = true } cfg-if.workspace = true clap.workspace = true indoc.workspace = true @@ -506,6 +510,9 @@ default-no-api-client = ["api", "enrichment-tables", "sinks", "sources", "source default-no-vrl-cli = ["api", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "secrets"] tokio-console = ["dep:console-subscriber", "tokio/tracing"] +# Enable the FIPS-only crypto backend +crypto-aws-lc-fips = ["dep:aws-lc-rs", "dep:aws-lc-sys"] + # Enables the binary secret-backend-example secret-backend-example = ["transforms"]