Merge branch 'main' into feature/source/trino

Author: nightkr

CHANGELOG.md

@@ -14,10 +14,15 @@ All notable changes to this project will be documented in this file.
   image to the HBase image. The script `export-snapshot-to-s3` makes
   exporting easier ([#621]).
 - kafka: Build from source ([#659], [#661]).
+- kafka: Add jmx broker config to builder image ([#703]).
+- nifi: Build from source ([#678]).
 - omid: Include Apache Omid in all workflows such as building and releasing images ([#635]).
 - java-devel: New image to serve as base layer for builder stages ([#665]).
 - hdfs: Exclude YARN and Mapreduce projects from build ([#667]).
 - stackable-base: Mitigate CVE-2023-37920 by removing e-Tugra root certificates ([#673]).
+- hdfs: Exclude unused jars and mitigate snappy-java CVEs by bumping dependency ([#682]).
+- druid: Build from source ([#684], [#696]).
+- opa: Add log processing script to opa for decision logging ([#695], [#704]).
 
 ### Changed
 
@@ -28,6 +33,8 @@ All notable changes to this project will be documented in this file.
 - ubi8-rust-builder: Bump `protoc` from `21.5` to `26.1` ([#624]).
 - pass platform argument to preflight check ([#626]).
 - nifi: provision stackable-bcrypt from Maven ([#663])
+- nifi: move /bin/stackable-bcrypt to /stackable/stackable-bcrypt and added softlink for backwards compatibility ([#678]).
+- nifi: patch nifi-assembly pom file to not zip binaries after the build to save disk space ([#685]).
 - hadoop: use java-devel as base layer for the builder stage ([#665])
 - hive: use java-devel as base layer for the builder stage ([#665])
 - zookeeper: use java-devel as base layer for the builder stage ([#665])
@@ -37,6 +44,7 @@ All notable changes to this project will be documented in this file.
 - opa-bundle-builder: Bump image to 1.1.2 ([#666])
 - opa: Build from source ([#676])
 - trino: Build from source ([#687]).
+- spark: Build from source ([#679])
 
 ### Fixed
 
@@ -47,6 +55,7 @@ All notable changes to this project will be documented in this file.
 - hive: Fix compilation on ARM by back-porting [HIVE-21939](https://issues.apache.org/jira/browse/HIVE-21939) from [this](https://github.com/apache/hive/commit/2baf21bb55fcf33d8522444c78a8d8cab60e7415) commit ([#617]).
 - hive: Fix compilation on ARM in CI as well ([#619]).
 - hive: Fix compilation of x86 in CI due to lower disk usage to prevent disk running full ([#619]).
+- hive: Provide logging dependency previously bundled with the hadoop yarn client ([#688]).
 
 ### Removed
 
@@ -74,9 +83,19 @@ All notable changes to this project will be documented in this file.
 [#665]: https://github.com/stackabletech/docker-images/pull/665
 [#666]: https://github.com/stackabletech/docker-images/pull/666
 [#667]: https://github.com/stackabletech/docker-images/pull/667
-[#676]: https://github.com/stackabletech/docker-images/pull/676
 [#673]: https://github.com/stackabletech/docker-images/pull/673
+[#676]: https://github.com/stackabletech/docker-images/pull/676
+[#678]: https://github.com/stackabletech/docker-images/pull/678
+[#679]: https://github.com/stackabletech/docker-images/pull/679
+[#682]: https://github.com/stackabletech/docker-images/pull/682
+[#684]: https://github.com/stackabletech/docker-images/pull/684
+[#685]: https://github.com/stackabletech/docker-images/pull/685
 [#687]: https://github.com/stackabletech/docker-images/pull/687
+[#688]: https://github.com/stackabletech/docker-images/pull/688
+[#696]: https://github.com/stackabletech/docker-images/pull/696
+[#695]: https://github.com/stackabletech/docker-images/pull/695
+[#703]: https://github.com/stackabletech/docker-images/pull/703
+[#704]: https://github.com/stackabletech/docker-images/pull/704
 
 ## [24.3.0] - 2024-03-20
 

airflow/Dockerfile

@@ -41,7 +41,7 @@ RUN microdnf update && \
         python${PYTHON}-devel \
         python${PYTHON}-pip \
         python${PYTHON}-wheel \
-        # the unixODBC-devel package is not available for ubi9
+        # The airflow odbc provider can compile without the development files (headers and libraries) (see https://github.com/stackabletech/docker-images/pull/683)
         unixODBC && \
     microdnf clean all && \
     rm -rf /var/cache/yum

conf.py

@@ -56,6 +56,7 @@
             {
                 "product": "26.0.0",
                 "java-base": "11",
+                "java-devel": "11",
                 "jackson_dataformat_xml": "2.10.5",
                 "stax2_api": "4.2.1",
                 "woodstox_core": "6.2.1",
@@ -64,6 +65,7 @@
             {
                 "product": "27.0.0",
                 "java-base": "11",
+                "java-devel": "11",
                 "jackson_dataformat_xml": "2.10.5",
                 "stax2_api": "4.2.1",
                 "woodstox_core": "6.2.1",
@@ -76,6 +78,7 @@
                 # Caused by: java.lang.reflect.InaccessibleObjectException: Unable to make protected final java.lang.Class
                 # java.lang.ClassLoader.defineClass(java.lang.String,byte[],int,int,java.security.ProtectionDomain) throws java.lang.ClassFormatError
                 "java-base": "11",
+                "java-devel": "11",
                 "jackson_dataformat_xml": "2.12.7",  # from https://github.com/apache/druid/blob/b8201e31aa6b124049a61764309145baaad78db7/pom.xml#L100
                 "stax2_api": "4.2.2",
                 "woodstox_core": "6.6.0",
@@ -120,7 +123,7 @@
                 "java-base": "11",
                 "java-devel": "11",
                 "async_profiler": "2.9",
-                "phoenix": "5.1.3",
+                "phoenix": "5.2.0",
                 "hbase_profile": "2.4",
                 "hadoop": "3.3.6",
                 "jmx_exporter": "0.20.0",
@@ -147,6 +150,8 @@
                 "java-devel": "1.8.0",
                 "hadoop": "3.3.4",
                 "jackson_dataformat_xml": "2.12.3",
+                # No longer bundled with the hadoop-yarn/mapreduce libraries (2.12.7 corresponds to the hadoop build for 3.3.4).
+                "jackson_jaxb_annotations": "2.12.7",
                 # Normally Hive 3.1.3 ships with "postgresql-9.4.1208.jre7.jar", but as this is old enough it does only support
                 # MD5 based authentication. Because of this, it does not work against more recent PostgresQL versions.
                 # See https://github.com/stackabletech/hive-operator/issues/170 for details.
@@ -249,9 +254,21 @@
     {
         "name": "nifi",
         "versions": [
-            {"product": "1.21.0", "java-base": "11"},
-            {"product": "1.23.2", "java-base": "11"},
-            {"product": "1.25.0", "java-base": "21"},
+            {
+                "product": "1.21.0",
+                "java-base": "11",
+                "java-devel": "11"
+            },
+            {
+                "product": "1.23.2",
+                "java-base": "11",
+                "java-devel": "11"
+            },
+            {
+                "product": "1.25.0",
+                "java-base": "21",
+                "java-devel": "11"
+            },
         ],
     },
     {
@@ -287,10 +304,9 @@
         "versions": [
             {
                 "product": "3.4.1",
-                "spark": "3.4.1",
                 "java-base": "11",
+                "java-devel": "11",
                 "python": "3.11",
-                "hadoop_short_version": "3",
                 "hadoop_long_version": "3.3.4",  # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122
                 "aws_java_sdk_bundle": "1.12.262",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4
                 "azure_storage": "7.0.1",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4
@@ -304,10 +320,9 @@
             },
             {
                 "product": "3.4.2",
-                "spark": "3.4.2",
                 "java-base": "11",
+                "java-devel": "11",
                 "python": "3.11",
-                "hadoop_short_version": "3",
                 "hadoop_long_version": "3.3.4",  # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122
                 "aws_java_sdk_bundle": "1.12.262",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4
                 "azure_storage": "7.0.1",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4
@@ -321,10 +336,9 @@
             },
             {
                 "product": "3.5.0",
-                "spark": "3.5.0",
                 "java-base": "11",
+                "java-devel": "11",
                 "python": "3.11",
-                "hadoop_short_version": "3",
                 "hadoop_long_version": "3.3.4",  # https://github.com/apache/spark/blob/6a5747d66e53ed0d934cdd9ca5c9bd9fde6868e6/pom.xml#L125
                 "aws_java_sdk_bundle": "1.12.262",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4
                 "azure_storage": "7.0.1",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4
@@ -340,8 +354,8 @@
                 "product": "3.5.1",
                 "spark": "3.5.1",
                 "java-base": "11",
+                "java-devel": "11",
                 "python": "3.11",
-                "hadoop_short_version": "3",
                 "hadoop_long_version": "3.3.4",  # https://github.com/apache/spark/blob/6a5747d66e53ed0d934cdd9ca5c9bd9fde6868e6/pom.xml#L125
                 "aws_java_sdk_bundle": "1.12.262",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4
                 "azure_storage": "7.0.1",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4

druid/Dockerfile

@@ -4,13 +4,76 @@
 # Ignoring DL4006 globally because we inherit the SHELL from our base image
 # hadolint global ignore=DL3038,DL4006
 
-FROM stackable/image/java-base
+# hadolint ignore=DL3006
+FROM stackable/image/java-devel as druid-builder
 
 ARG PRODUCT
 ARG JACKSON_DATAFORMAT_XML
 ARG STAX2_API
 ARG WOODSTOX_CORE
 ARG AUTHORIZER
+
+RUN microdnf update && \
+    microdnf install \
+    # Required to install pyyaml
+    python-pip \
+    # Required to patch druid
+    patch && \
+    microdnf clean all && \
+    rm -rf /var/cache/yum && \
+    # pyyaml is required for the compile Druid
+    pip install --no-cache-dir pyyaml==6.0.1
+
+USER stackable
+WORKDIR /stackable
+
+COPY --chown=stackable:stackable druid/stackable/patches/apply_patches.sh /stackable/apache-druid-${PRODUCT}-src/patches/apply_patches.sh
+COPY --chown=stackable:stackable druid/stackable/patches/${PRODUCT} /stackable/apache-druid-${PRODUCT}-src/patches/${PRODUCT}
+
+RUN curl --fail -L "https://repo.stackable.tech/repository/packages/druid/apache-druid-${PRODUCT}-src.tar.gz" | tar -xzC . && \
+    cd apache-druid-${PRODUCT}-src && \
+    ./patches/apply_patches.sh ${PRODUCT} && \
+    mvn clean install -Pdist -pl '!extensions-core/druid-ranger-security' -DskipTests -Dmaven.javadoc.skip=true && \
+    tar -xzf /stackable/apache-druid-${PRODUCT}-src/distribution/target/apache-druid-${PRODUCT}-bin.tar.gz && \
+    mv /stackable/apache-druid-${PRODUCT}-src/apache-druid-${PRODUCT} /stackable/apache-druid-${PRODUCT} && \
+    rm -rf /stackable/apache-druid-${PRODUCT}-src
+    #  Do not remove the /stackable/apache-druid-${PRODUCT}/quickstart folder, it is needed for loading the Wikipedia
+    # testdata in kuttl tests and the getting started guide.
+
+    # Install the Prometheus emitter extension. This bundle contains the emitter and all jar dependencies.
+RUN curl --fail "https://repo.stackable.tech/repository/packages/druid/druid-prometheus-emitter-${PRODUCT}.tar.gz" | tar -xzC /stackable/apache-druid-${PRODUCT}/extensions && \
+    # Install OPA authorizer extension.
+    curl --fail "https://repo.stackable.tech/repository/packages/druid/druid-opa-authorizer-${AUTHORIZER}.tar.gz" | tar -xzC /stackable/apache-druid-${PRODUCT}/extensions && \
+    # Install jackson-dataformat-xml, stax2-api, and woodstox-core which are required for logging, and remove stax-ex.
+    rm /stackable/apache-druid-${PRODUCT}/lib/stax-ex-*.jar && \
+    curl --fail -L -o /stackable/apache-druid-${PRODUCT}/lib/jackson-dataformat-xml-${JACKSON_DATAFORMAT_XML}.jar \
+        "https://repo.stackable.tech/repository/packages/jackson-dataformat-xml/jackson-dataformat-xml-${JACKSON_DATAFORMAT_XML}.jar" && \
+    curl --fail -L -o /stackable/apache-druid-${PRODUCT}/lib/stax2-api-${STAX2_API}.jar \
+        "https://repo.stackable.tech/repository/packages/stax2-api/stax2-api-${STAX2_API}.jar" && \
+    curl --fail -L -o /stackable/apache-druid-${PRODUCT}/lib/woodstox-core-${WOODSTOX_CORE}.jar \
+        "https://repo.stackable.tech/repository/packages/woodstox-core/woodstox-core-${WOODSTOX_CORE}.jar"
+
+# For earlier versions this script removes the .class file that contains the
+# vulnerable code.
+# TODO: This can be restricted to target only versions which do not honor the environment
+#   varible that has been set above but this has not currently been implemented
+COPY shared/log4shell.sh /bin
+RUN /bin/log4shell.sh "/stackable/apache-druid-${PRODUCT}"
+
+# Ensure no vulnerable files are left over
+# This will currently report vulnerable files being present, as it also alerts on
+# SocketNode.class, which we do not remove with our scripts.
+# Further investigation will be needed whether this should also be removed.
+COPY shared/log4shell_1.6.1-log4shell_Linux_x86_64 /bin/log4shell_scanner_x86_64
+COPY shared/log4shell_1.6.1-log4shell_Linux_aarch64 /bin/log4shell_scanner_aarch64
+COPY shared/log4shell_scanner /bin/log4shell_scanner
+RUN /bin/log4shell_scanner s "/stackable/apache-druid-${PRODUCT}"
+# ===
+
+# hadolint ignore=DL3006
+FROM stackable/image/java-base as final
+
+ARG PRODUCT
 ARG RELEASE
 
 LABEL name="Apache Druid" \
@@ -22,36 +85,27 @@ LABEL name="Apache Druid" \
       description="This image is deployed by the Stackable Operator for Apache Druid."
 
 RUN microdnf update && \
-    microdnf install \
-    findutils \
-    gzip \
-    tar \
-    zip && \
     microdnf clean all && \
     rm -rf /var/cache/yum
 
 USER stackable
 WORKDIR /stackable
 
+COPY --from=druid-builder /stackable /stackable
 COPY --chown=stackable:stackable druid/stackable /stackable
 COPY --chown=stackable:stackable druid/licenses /licenses
 
-RUN curl --fail -L https://repo.stackable.tech/repository/packages/druid/apache-druid-${PRODUCT}-bin.tar.gz | tar -xzC . && \
-    ln -s /stackable/apache-druid-${PRODUCT} /stackable/druid && \
+RUN ln -s /stackable/apache-druid-${PRODUCT} /stackable/druid && \
     # Force to overwrite the existing 'run-druid'
-    ln -sf /stackable/bin/run-druid /stackable/druid/bin/run-druid && \
-    # Install the Prometheus emitter extension. This bundle contains the emitter and all jar dependencies.
-    curl --fail https://repo.stackable.tech/repository/packages/druid/druid-prometheus-emitter-${PRODUCT}.tar.gz | tar -xzC /stackable/druid/extensions && \
-    # Install OPA authorizer extension.
-    curl --fail https://repo.stackable.tech/repository/packages/druid/druid-opa-authorizer-${AUTHORIZER}.tar.gz | tar -xzC /stackable/druid/extensions && \
-    # Install jackson-dataformat-xml, stax2-api, and woodstox-core which are required for logging, and remove stax-ex.
-    rm /stackable/druid/lib/stax-ex-*.jar && \
-    curl --fail -L -o /stackable/druid/lib/jackson-dataformat-xml-${JACKSON_DATAFORMAT_XML}.jar \
-        https://repo.stackable.tech/repository/packages/jackson-dataformat-xml/jackson-dataformat-xml-${JACKSON_DATAFORMAT_XML}.jar && \
-    curl --fail -L -o /stackable/druid/lib/stax2-api-${STAX2_API}.jar \
-        https://repo.stackable.tech/repository/packages/stax2-api/stax2-api-${STAX2_API}.jar && \
-    curl --fail -L -o /stackable/druid/lib/woodstox-core-${WOODSTOX_CORE}.jar \
-        https://repo.stackable.tech/repository/packages/woodstox-core/woodstox-core-${WOODSTOX_CORE}.jar
+    ln -sf /stackable/bin/run-druid /stackable/druid/bin/run-druid
+
+ENV PATH="${PATH}":/stackable/druid/bin
+
+# ===
+# Mitigation for CVE-2021-44228 (Log4Shell)
+# This variable is supported as of Log4j version 2.10 and
+# disables the vulnerable feature
+ENV LOG4J_FORMAT_MSG_NO_LOOKUPS=true
 
 WORKDIR /stackable/druid
 CMD ["bin/run-druid", "coordinator", "conf/druid/cluster/master/coordinator-overlord/"]

druid/stackable/patches/26.0.0/001-remove-druid-ranger-security.patch

@@ -0,0 +1,13 @@
+diff --git a/distribution/pom.xml b/distribution/pom.xml
+index 14013a7a5f..58fd00009e 100644
+--- a/distribution/pom.xml
++++ b/distribution/pom.xml
+@@ -252,8 +252,6 @@
+                                         <argument>-c</argument>
+                                         <argument>org.apache.druid.extensions:druid-pac4j</argument>
+                                         <argument>-c</argument>
+-                                        <argument>org.apache.druid.extensions:druid-ranger-security</argument>
+-                                        <argument>-c</argument>
+                                         <argument>org.apache.druid.extensions:druid-kubernetes-extensions</argument>
+                                         <argument>-c</argument>
+                                         <argument>org.apache.druid.extensions:druid-catalog</argument>

druid/stackable/patches/27.0.0/001-remove-druid-ranger-security.patch

@@ -0,0 +1,13 @@
+diff --git a/distribution/pom.xml b/distribution/pom.xml
+index 14013a7a5f..58fd00009e 100644
+--- a/distribution/pom.xml
++++ b/distribution/pom.xml
+@@ -252,8 +252,6 @@
+                                         <argument>-c</argument>
+                                         <argument>org.apache.druid.extensions:druid-pac4j</argument>
+                                         <argument>-c</argument>
+-                                        <argument>org.apache.druid.extensions:druid-ranger-security</argument>
+-                                        <argument>-c</argument>
+                                         <argument>org.apache.druid.extensions:druid-kubernetes-extensions</argument>
+                                         <argument>-c</argument>
+                                         <argument>org.apache.druid.extensions:druid-catalog</argument>

druid/stackable/patches/28.0.1/001-remove-druid-ranger-security.patch

@@ -0,0 +1,13 @@
+diff --git a/distribution/pom.xml b/distribution/pom.xml
+index 14013a7a5f..58fd00009e 100644
+--- a/distribution/pom.xml
++++ b/distribution/pom.xml
+@@ -252,8 +252,6 @@
+                                         <argument>-c</argument>
+                                         <argument>org.apache.druid.extensions:druid-pac4j</argument>
+                                         <argument>-c</argument>
+-                                        <argument>org.apache.druid.extensions:druid-ranger-security</argument>
+-                                        <argument>-c</argument>
+                                         <argument>org.apache.druid.extensions:druid-kubernetes-extensions</argument>
+                                         <argument>-c</argument>
+                                         <argument>org.apache.druid.extensions:druid-catalog</argument>

druid/stackable/patches/apply_patches.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Enable error handling and unset variable checking
+set -eu
+set -o pipefail
+
+# Check if $1 (VERSION) is provided
+if [ -z "${1-}" ]; then
+  echo "Please provide a value for VERSION as the first argument."
+  exit 1
+fi
+
+VERSION="$1"
+PATCH_DIR="patches/$VERSION"
+
+# Check if version-specific patches directory exists
+if [ ! -d "$PATCH_DIR" ]; then
+  echo "Patches directory '$PATCH_DIR' does not exist."
+  exit 1
+fi
+
+# Create an array to hold the patches in sorted order
+declare -a patch_files=()
+
+echo "Applying patches from ${PATCH_DIR}" now
+
+# Read the patch files into the array
+while IFS= read -r -d $'\0' file; do
+  patch_files+=("$file")
+done < <(find "$PATCH_DIR" -name "*.patch" -print0 | sort -zV)
+
+echo "Found ${#patch_files[@]} patches, applying now"
+
+# Iterate through sorted patch files
+for patch_file in "${patch_files[@]}"; do
+  echo "Applying $patch_file"
+  # We can not use Git here, as we are not within a Git repo
+  patch --directory "." --strip=1 < "$patch_file" || {
+    echo "Failed to apply $patch_file"
+    exit 1
+  }
+done
+
+echo "All patches applied successfully."