From 139e6bc52aef3f0326c47155d7f0ca76e3016275 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Wed, 19 Mar 2025 12:59:59 -0700 Subject: [PATCH 1/2] Conditionally install bcfips jars when building for observabilitySRE This commit implements a pattern for performing specific gradle tasks based on a newly named "fedrampHighMode" option. This option is used to configure tests to run with additional configuration specific to the observabilitySRE use case. Similarly the additional jar dependencies for bouncycastle fips providers are conditionally installed gated on the "fedrampHighMode" option. In order to ensure the the "fedrampHighMode" option persists through the layers of sub-processes spawned between gradle and rake we store and respect an environment variable FEDRAMP_HIGH_MODE. This may be useful generally in building the docker image. Try codereview suggestion --- .buildkite/pull_request_pipeline.yml | 6 ++-- ci/run-fips-integration-tests.sh | 2 +- logstash-core/build.gradle | 5 --- rubyUtils.gradle | 4 +++ x-pack/ci/integration_tests.sh | 2 +- x-pack/ci/unit_tests.sh | 2 +- .../observabilitySRE/build-ext.gradle | 36 ++++++++++++++++--- .../observabilitySRE/docker/Dockerfile | 4 +-- 8 files changed, 44 insertions(+), 17 deletions(-) diff --git a/.buildkite/pull_request_pipeline.yml b/.buildkite/pull_request_pipeline.yml index f9a349e60ba..5d2952526df 100644 --- a/.buildkite/pull_request_pipeline.yml +++ b/.buildkite/pull_request_pipeline.yml @@ -54,7 +54,7 @@ steps: set -euo pipefail docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile . - docker run test-runner-image ./gradlew --info --stacktrace -PrunTestsInFIPSMode=true rubyTests + docker run test-runner-image ./gradlew --info --stacktrace -PfedrampHighMode=true rubyTests artifact_paths: - "coverage/coverage.json" @@ -96,7 +96,7 @@ steps: set -euo pipefail docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile . - docker run test-runner-image ./gradlew --info --stacktrace -PrunTestsInFIPSMode=true javaTests + docker run test-runner-image ./gradlew --info --stacktrace -PfedrampHighMode=true javaTests artifact_paths: - "**/build/test-results/javaTests/TEST-*.xml" - "**/jacocoTestReport.xml" @@ -139,7 +139,7 @@ steps: source .buildkite/scripts/common/vm-agent.sh # TODO: Use https://github.com/elastic/logstash/pull/17311 to compute QUALIFIED_VERSION once merged QUALIFIED_VERSION="8.19.0-SNAPSHOT" - ./gradlew --stacktrace artifactDockerObservabilitySRE + ./gradlew --stacktrace artifactDockerObservabilitySRE -PfedrampHighMode=true docker run docker.elastic.co/logstash/logstash-observability-sre:$${QUALIFIED_VERSION} \ logstash -e 'input { generator { count => 3 } } output { stdout { codec => rubydebug } }' diff --git a/ci/run-fips-integration-tests.sh b/ci/run-fips-integration-tests.sh index 6bff29c111b..7ed6e9c348a 100755 --- a/ci/run-fips-integration-tests.sh +++ b/ci/run-fips-integration-tests.sh @@ -6,4 +6,4 @@ half_number=$1 source ci/get-test-half.sh specs=$(get_test_half "$half_number") -./gradlew --info --stacktrace -PrunTestsInFIPSMode=true runIntegrationTests -PrubyIntegrationSpecs="$specs" \ No newline at end of file +./gradlew --info --stacktrace -PfedrampHighMode=true runIntegrationTests -PrubyIntegrationSpecs="$specs" \ No newline at end of file diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle index 0ce2ff2f2dd..831f0fefde5 100644 --- a/logstash-core/build.gradle +++ b/logstash-core/build.gradle @@ -235,11 +235,6 @@ dependencies { runtimeOnly 'commons-logging:commons-logging:1.3.1' // also handle libraries relying on log4j 1.x to redirect their logs runtimeOnly "org.apache.logging.log4j:log4j-1.2-api:${log4jVersion}" - // FIPS deps. TODO: figure out how to actually manage these - runtimeOnly("org.bouncycastle:bc-fips:2.0.0") - runtimeOnly("org.bouncycastle:bcpkix-fips:2.0.7") - runtimeOnly("org.bouncycastle:bctls-fips:2.0.19") - runtimeOnly("org.bouncycastle:bcutil-fips:2.0.3") implementation('org.reflections:reflections:0.10.2') { exclude group: 'com.google.guava', module: 'guava' } diff --git a/rubyUtils.gradle b/rubyUtils.gradle index 94d020543c2..9e3203c2f01 100644 --- a/rubyUtils.gradle +++ b/rubyUtils.gradle @@ -191,6 +191,10 @@ Object executeJruby(File projectDir, File buildDir, Closure /* Object*/ block env.put "GEM_HOME", gemDir env.put "GEM_SPEC_CACHE", "${buildDir}/cache".toString() env.put "GEM_PATH", gemDir + // Pass through FEDRAMP_HIGH_MODE if it exists in the project properties + if (project.hasProperty('fedrampHighMode') && project.property('fedrampHighMode').toBoolean()) { + env.put "FEDRAMP_HIGH_MODE", "true" + } try { block(jruby) } finally { diff --git a/x-pack/ci/integration_tests.sh b/x-pack/ci/integration_tests.sh index db88dc6e946..beca4f87def 100755 --- a/x-pack/ci/integration_tests.sh +++ b/x-pack/ci/integration_tests.sh @@ -18,7 +18,7 @@ if [ -n "$BUILD_JAVA_HOME" ]; then fi if [ -n "$FIPS_MODE" ]; then - ./gradlew runXPackIntegrationTests -PrunTestsInFIPSMode=true + ./gradlew runXPackIntegrationTests -PfedrampHighMode=true else ./gradlew runXPackIntegrationTests fi diff --git a/x-pack/ci/unit_tests.sh b/x-pack/ci/unit_tests.sh index d402478442f..86fb9cd8e9a 100755 --- a/x-pack/ci/unit_tests.sh +++ b/x-pack/ci/unit_tests.sh @@ -17,7 +17,7 @@ if [ -n "$BUILD_JAVA_HOME" ]; then fi if [ -n "$FIPS_MODE" ]; then - ./gradlew runXPackUnitTests -PrunTestsInFIPSMode=true + ./gradlew runXPackUnitTests -PfedrampHighMode=true else ./gradlew runXPackUnitTests fi diff --git a/x-pack/distributions/internal/observabilitySRE/build-ext.gradle b/x-pack/distributions/internal/observabilitySRE/build-ext.gradle index b5c5b87c75d..b5dc732cbe5 100644 --- a/x-pack/distributions/internal/observabilitySRE/build-ext.gradle +++ b/x-pack/distributions/internal/observabilitySRE/build-ext.gradle @@ -1,17 +1,29 @@ ext { - runTestsInFIPSMode = project.hasProperty('runTestsInFIPSMode') ? project.property('runTestsInFIPSMode').toBoolean() : false + fedrampHighMode = System.getenv('FEDRAMP_HIGH_MODE') == 'true' || + (project.hasProperty('fedrampHighMode') ? project.property('fedrampHighMode').toBoolean() : false) } subprojects { ext { - runTestsInFIPSMode = rootProject.runTestsInFIPSMode + fedrampHighMode = rootProject.fedrampHighMode } } allprojects { afterEvaluate { + // Preserve fedrampHighMode option across subprocesses + if (rootProject.fedrampHighMode) { + tasks.withType(JavaExec).configureEach { + environment("FEDRAMP_HIGH_MODE", "true") + } + + tasks.withType(Exec).configureEach { + environment("FEDRAMP_HIGH_MODE", "true") + } + } + tasks.withType(Test) { - if (runTestsInFIPSMode) { + if (rootProject.fedrampHighMode) { logger.debug("configuring ${it} to run in FIPSMode ") systemProperty "java.security.properties", System.getenv("JAVA_SECURITY_PROPERTIES") systemProperty "javax.net.ssl.keyStore", "/etc/java/security/keystore.bcfks" @@ -28,4 +40,20 @@ allprojects { } } } -} \ No newline at end of file +} + +project(':logstash-core') { + afterEvaluate { + if (rootProject.fedrampHighMode) { + logger.lifecycle("Adding BouncyCastle FIPS dependencies to logstash-core") + dependencies { + // Add FIPS dependencies to the runtimeOnly configuration + // This ensures they'll be included by the existing copyRuntimeLibs task + runtimeOnly "org.bouncycastle:bc-fips:2.0.0" + runtimeOnly "org.bouncycastle:bcpkix-fips:2.0.7" + runtimeOnly "org.bouncycastle:bctls-fips:2.0.19" + runtimeOnly "org.bouncycastle:bcutil-fips:2.0.3" + } + } + } +} diff --git a/x-pack/distributions/internal/observabilitySRE/docker/Dockerfile b/x-pack/distributions/internal/observabilitySRE/docker/Dockerfile index 3645b93d475..317ee95ee22 100644 --- a/x-pack/distributions/internal/observabilitySRE/docker/Dockerfile +++ b/x-pack/distributions/internal/observabilitySRE/docker/Dockerfile @@ -43,7 +43,7 @@ ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk ENV PATH="${JAVA_HOME}/bin:${PATH}" # Initial build using JKS truststore -RUN ./gradlew clean bootstrap assemble installDefaultGems +RUN ./gradlew clean bootstrap assemble installDefaultGems -PfedrampHighMode=true # Convert JKS to BCFKS for truststore and keystore RUN keytool -importkeystore \ @@ -87,4 +87,4 @@ ENV LS_JAVA_OPTS="\ -Dorg.bouncycastle.fips.approved_only=true" # Example test run, most use cases will override this -CMD ["./gradlew", "--info", "--stacktrace", "-PrunTestsInFIPSMode=true", "runIntegrationTests"] \ No newline at end of file +CMD ["./gradlew", "--info", "--stacktrace", "-PfedrampHighMode=true", "runIntegrationTests"] \ No newline at end of file From 6478278bb4ce73931d2e580c5cf5972d74989bdc Mon Sep 17 00:00:00 2001 From: donoghuc Date: Thu, 20 Mar 2025 15:46:27 -0700 Subject: [PATCH 2/2] Use gradle pattern for setting properties with env vars Gradle has a mechanism for setting properties with environment variables prefixed with `ORG_GRADLE_PROJECT`. This commit updates the gradle tasks to use that pattern. See https://docs.gradle.org/current/userguide/build_environment.html#setting_a_project_property for details. --- rubyUtils.gradle | 6 ++++-- .../internal/observabilitySRE/build-ext.gradle | 8 +++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/rubyUtils.gradle b/rubyUtils.gradle index 9e3203c2f01..9ba2ae7f042 100644 --- a/rubyUtils.gradle +++ b/rubyUtils.gradle @@ -191,9 +191,11 @@ Object executeJruby(File projectDir, File buildDir, Closure /* Object*/ block env.put "GEM_HOME", gemDir env.put "GEM_SPEC_CACHE", "${buildDir}/cache".toString() env.put "GEM_PATH", gemDir - // Pass through FEDRAMP_HIGH_MODE if it exists in the project properties + // Pass through ORG_GRADLE_PROJECT_fedrampHighMode if it exists in the project properties + // See https://docs.gradle.org/current/userguide/build_environment.html#setting_a_project_property + // For more information about setting properties via env vars prefixed with ORG_GRADLE_PROJECT if (project.hasProperty('fedrampHighMode') && project.property('fedrampHighMode').toBoolean()) { - env.put "FEDRAMP_HIGH_MODE", "true" + env.put "ORG_GRADLE_PROJECT_fedrampHighMode", "true" } try { block(jruby) diff --git a/x-pack/distributions/internal/observabilitySRE/build-ext.gradle b/x-pack/distributions/internal/observabilitySRE/build-ext.gradle index b5dc732cbe5..5b97a00d28b 100644 --- a/x-pack/distributions/internal/observabilitySRE/build-ext.gradle +++ b/x-pack/distributions/internal/observabilitySRE/build-ext.gradle @@ -1,6 +1,5 @@ ext { - fedrampHighMode = System.getenv('FEDRAMP_HIGH_MODE') == 'true' || - (project.hasProperty('fedrampHighMode') ? project.property('fedrampHighMode').toBoolean() : false) + fedrampHighMode = Objects.requireNonNullElse(project.findProperty('fedrampHighMode'), false).toBoolean() } subprojects { @@ -14,14 +13,13 @@ allprojects { // Preserve fedrampHighMode option across subprocesses if (rootProject.fedrampHighMode) { tasks.withType(JavaExec).configureEach { - environment("FEDRAMP_HIGH_MODE", "true") + environment("ORG_GRADLE_PROJECT_fedrampHighMode", "true") } tasks.withType(Exec).configureEach { - environment("FEDRAMP_HIGH_MODE", "true") + environment("ORG_GRADLE_PROJECT_fedrampHighMode", "true") } } - tasks.withType(Test) { if (rootProject.fedrampHighMode) { logger.debug("configuring ${it} to run in FIPSMode ")