feat: Update demos for Release 24.11 (#116)

NickLarsenNZ · xeniape · razvan · web-flow · commit a7d34d2f7d2f · 2024-11-11T13:23:08.000Z
* chore(opa): bump to 0.67.0 Depends on: stackabletech/docker-images#797 * chore(opa): bump 0.67.1 * chore(opa): bump to 0.67.0 Depends on: stackabletech/docker-images#797 * chore: Bump Vector to 0.40.0 * chore(airflow): bump version to 2.9.3 * chore(trino): bump version to 455 * chore(spark): version 3.5.2 * Update create-spark-ingestion-job.yaml * chore: bump hdfs version to 3.4.0 * chore: bump nifi for 24.11 * chore: Upgrade the Vector aggregator * chore(druid): Bump to 30.0.0 * chore(pre-commit): Ignore braces lint for templates yaml files * chore: lint * Apply suggestions from code review Use images that exist. We can't use hdfs from 24.7 because there is a tool needed. For consistency, we use the same image for consistency * Apply suggestions from code review * chore: fix lint issue * Apply suggestions from code review * chore: fix lint issue * chore: disable the brace lint --------- Co-authored-by: xeniape <xenia.fischer@stackable.tech> Co-authored-by: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Co-authored-by: Siegfried Weber <mail@siegfriedweber.net>
diff --git a/.yamllint.yaml b/.yamllint.yaml
@@ -7,7 +7,4 @@ rules:
     check-keys: false
   comments:
     min-spaces-from-content: 1  # Needed due to https://github.com/adrienverge/yamllint/issues/443
-  braces:
-    ignore: |
-      stacks/signal-processing/*
-      stacks/_templates/*
+  braces: disable # because the yaml files are templates which can have {{ ... }}
diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml
@@ -43,13 +43,13 @@ data:
         stackable.tech/vendor: Stackable
     spec:
       sparkImage:
-        productVersion: 3.5.1
+        productVersion: 3.5.2
       mode: cluster
       mainApplicationFile: local:///stackable/spark/jobs/spark-ingest-into-lakehouse.py
       deps:
         packages:
-          - org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.5.0
-          - org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.1
+          - org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.6.1
+          - org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.2
       s3connection:
         reference: minio
       sparkConf:
diff --git a/demos/end-to-end-security/create-spark-report.yaml b/demos/end-to-end-security/create-spark-report.yaml
@@ -55,7 +55,7 @@ data:
       name: spark-report
     spec:
       sparkImage:
-        productVersion: 3.5.1
+        productVersion: 3.5.2
       mode: cluster
       mainApplicationFile: local:///stackable/spark/jobs/spark-report.py
       deps:
diff --git a/demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml b/demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml
@@ -11,14 +11,15 @@ spec:
           # We use 24.3.0 here which contains the distcp MapReduce components
           # This is not included in the 24.7 images and will fail.
           # See: https://github.com/stackabletech/docker-images/issues/793
-          image: docker.stackable.tech/stackable/hadoop:3.3.4-stackable24.3.0
+          image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable0.0.0-dev
           env:
             - name: HADOOP_USER_NAME
               value: stackable
             - name: HADOOP_CONF_DIR
               value: "/stackable/conf/hdfs"
             - name: HADOOP_CLASSPATH
               value: "/stackable/hadoop/share/hadoop/tools/lib/*.jar"
+          # yamllint disable-line rule:line-length
           command: ["bash", "-c", "bin/hdfs dfs -mkdir -p /data/raw && bin/hadoop distcp -D fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider s3a://public-backup-nyc-tlc/cycling-tripdata/demo-cycling-tripdata.csv.gz hdfs://hdfs/data/raw"]
           volumeMounts:
             - name: config-volume-hdfs
diff --git a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml
@@ -8,13 +8,15 @@ spec:
     spec:
       containers:
         - name: load-ny-taxi-data
-          image: docker.stackable.tech/stackable/hadoop:3.3.4-stackable24.7.0
+          image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable0.0.0-dev
+          # yamllint disable rule:line-length
           command: ["bash", "-c", "/stackable/hadoop/bin/hdfs dfs -mkdir -p /ny-taxi-data/raw \
           && cd /tmp \
           && for month in 2020-09; do \
             curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet \
             && /stackable/hadoop/bin/hdfs dfs -put fhvhv_tripdata_$month.parquet /ny-taxi-data/raw/; \
           done"]
+          # yamllint enable rule:line-length
           volumeMounts:
             - name: hdfs-discovery-configmap
               mountPath: /hdfs
diff --git a/demos/signal-processing/DownloadAndWriteToDB.xml b/demos/signal-processing/DownloadAndWriteToDB.xml
@@ -61,7 +61,7 @@
             <bundle>
                 <artifact>nifi-dbcp-service-nar</artifact>
                 <group>org.apache.nifi</group>
-                <version>1.21.0</version>
+                <version>1.27.0</version>
             </bundle>
             <comments></comments>
             <descriptors>
@@ -258,7 +258,7 @@
             <bundle>
                 <artifact>nifi-record-serialization-services-nar</artifact>
                 <group>org.apache.nifi</group>
-                <version>1.21.0</version>
+                <version>1.27.0</version>
             </bundle>
             <comments></comments>
             <descriptors>
@@ -561,7 +561,7 @@
             </position>
             <height>88.0</height>
             <label>This flow downloads a dataset, writing it to a temporary table in TimescaleDB.
-This data is then written to the target table with the time offsets preserved, 
+This data is then written to the target table with the time offsets preserved,
 but re-based to the current time. This means that the data can be displayed
 in Grafana as if it were being streamed, whereas in fact the dashboard moves
 through "future" data that has already been persisted.</label>
@@ -584,7 +584,7 @@ through "future" data that has already been persisted.</label>
             <bundle>
                 <artifact>nifi-standard-nar</artifact>
                 <group>org.apache.nifi</group>
-                <version>1.21.0</version>
+                <version>1.27.0</version>
             </bundle>
             <config>
                 <backoffMechanism>PENALIZE_FLOWFILE</backoffMechanism>
@@ -1069,7 +1069,7 @@ through "future" data that has already been persisted.</label>
             <bundle>
                 <artifact>nifi-standard-nar</artifact>
                 <group>org.apache.nifi</group>
-                <version>1.21.0</version>
+                <version>1.27.0</version>
             </bundle>
             <config>
                 <backoffMechanism>PENALIZE_FLOWFILE</backoffMechanism>
@@ -1223,7 +1223,7 @@ from conditions_temp;</value>
             <bundle>
                 <artifact>nifi-standard-nar</artifact>
                 <group>org.apache.nifi</group>
-                <version>1.21.0</version>
+                <version>1.27.0</version>
             </bundle>
             <config>
                 <backoffMechanism>PENALIZE_FLOWFILE</backoffMechanism>
diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml
@@ -37,7 +37,7 @@ data:
       name: spark-ad
     spec:
       sparkImage:
-        productVersion: 3.5.1
+        productVersion: 3.5.2
       mode: cluster
       mainApplicationFile: local:///spark-scripts/spark-ad.py
       deps:
diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml
@@ -9,6 +9,7 @@ spec:
       containers:
         - name: load-ny-taxi-data
           image: "bitnami/minio:2022-debian-10"
+          # yamllint disable-line rule:line-length
           command: ["bash", "-c", "cd /tmp && for month in 2020-09 2020-10 2020-11 2020-12; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet && mc --insecure alias set minio http://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) && mc cp fhvhv_tripdata_$month.parquet minio/demo/ny-taxi-data/raw/ && mc mb --ignore-existing minio/prediction; done"]
           volumeMounts:
             - name: minio-s3-credentials
diff --git a/demos/trino-taxi-data/load-test-data.yaml b/demos/trino-taxi-data/load-test-data.yaml
@@ -9,6 +9,7 @@ spec:
       containers:
         - name: load-ny-taxi-data
           image: "bitnami/minio:2024-debian-12"
+          # yamllint disable-line rule:line-length
           command: ["bash", "-c", "cd /tmp && for month in 2020-01 2020-02 2020-03 2020-04 2020-05 2020-06 2020-07 2020-08 2020-09 2020-10 2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/yellow_tripdata_$month.parquet && mc --insecure alias set minio http://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) && mc cp yellow_tripdata_$month.parquet minio/demo/ny-taxi-data/raw/; done"]
           volumeMounts:
             - name: minio-s3-credentials
diff --git a/docs/modules/demos/pages/trino-iceberg.adoc b/docs/modules/demos/pages/trino-iceberg.adoc
@@ -78,7 +78,7 @@ As an alternative, you can use https://trino.io/download.html[trino-cli] by runn
 
 [source,console]
 ----
-$ java -jar ~/Downloads/trino-cli-451-executable.jar --user admin --insecure --password --server https://172.18.0.2:30856
+$ java -jar ~/Downloads/trino-cli-455-executable.jar --user admin --insecure --password --server https://172.18.0.2:30856
 ----
 
 Make sure to replace the server endpoint with the endpoint listed in the `stackablectl stacklet list` output.
diff --git a/stacks/_templates/vector-aggregator.yaml b/stacks/_templates/vector-aggregator.yaml
@@ -4,11 +4,11 @@ name: vector
 repo:
   name: vector
   url: https://helm.vector.dev
-version: 0.37.0
+version: 0.36.1 # app version 0.41.1
 options:
   commonLabels:
     stackable.tech/vendor: Stackable
-  podLabels: # Doesn't seem to work?
+  podLabels:
     stackable.tech/vendor: Stackable
   role: Aggregator
   customConfig:
@@ -26,7 +26,7 @@ options:
           - https://opensearch-cluster-master.default.svc.cluster.local:9200
         mode: bulk
         # The auto-detection of the API version does not work in Vector
-        # 0.39.0 for OpenSearch, so the version must be set explicitly
+        # 0.41.1 for OpenSearch, so the version must be set explicitly
         # (see https://github.com/vectordotdev/vector/issues/17690).
         api_version: v8
         tls:
diff --git a/stacks/airflow/airflow.yaml b/stacks/airflow/airflow.yaml
@@ -1,12 +1,12 @@
-{% raw %}
 ---
+# {% raw %}
 apiVersion: airflow.stackable.tech/v1alpha1
 kind: AirflowCluster
 metadata:
   name: airflow
 spec:
   image:
-    productVersion: 2.9.2
+    productVersion: 2.9.3
   clusterConfig:
     listenerClass: external-unstable
     loadExamples: false
@@ -272,7 +272,7 @@ data:
     spec:
       version: "1.0"
       sparkImage:
-        productVersion: 3.5.1
+        productVersion: 3.5.2
       mode: cluster
       mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py
       job:
@@ -285,22 +285,22 @@ data:
               limit: 512Mi
       driver:
         config:
-          resources: 
+          resources:
             cpu:
               min: 1000m
               max: 1200m
             memory:
               limit: 1024Mi
       executor:
         config:
-          resources: 
+          resources:
             cpu:
               min: 500m
               max: 1000m
             memory:
               limit: 1024Mi
         replicas: 3
-{% endraw %}
+# {% endraw %}
 ---
 apiVersion: v1
 kind: Secret
diff --git a/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml b/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml
@@ -5,7 +5,7 @@ metadata:
   name: nifi
 spec:
   image:
-    productVersion: 1.25.0
+    productVersion: 1.27.0
   clusterConfig:
     authentication:
       - authenticationClass: nifi-admin-credentials
diff --git a/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml b/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml
@@ -5,7 +5,7 @@ metadata:
   name: trino
 spec:
   image:
-    productVersion: "451"
+    productVersion: "455"
   clusterConfig:
     listenerClass: external-unstable
     catalogLabelSelector:
@@ -114,7 +114,7 @@ metadata:
   name: opa
 spec:
   image:
-    productVersion: 0.66.0
+    productVersion: 0.67.1
   servers:
     roleGroups:
       default: {}
diff --git a/stacks/dual-hive-hdfs-s3/hdfs.yaml b/stacks/dual-hive-hdfs-s3/hdfs.yaml
@@ -25,7 +25,7 @@ metadata:
   name: hdfs
 spec:
   image:
-    productVersion: 3.3.4
+    productVersion: 3.4.0
   clusterConfig:
     listenerClass: external-unstable
     dfsReplication: 1
diff --git a/stacks/dual-hive-hdfs-s3/trino.yaml b/stacks/dual-hive-hdfs-s3/trino.yaml
@@ -61,7 +61,7 @@ metadata:
   name: trino
 spec:
   image:
-    productVersion: "451"
+    productVersion: "455"
   clusterConfig:
     authorization:
       opa:
diff --git a/stacks/end-to-end-security/hdfs.yaml b/stacks/end-to-end-security/hdfs.yaml
@@ -1,10 +1,11 @@
+---
 apiVersion: hdfs.stackable.tech/v1alpha1
 kind: HdfsCluster
 metadata:
   name: hdfs
 spec:
   image:
-    productVersion: 3.3.4
+    productVersion: 3.4.0
   clusterConfig:
     zookeeperConfigMapName: hdfs-znode
     authentication:
@@ -29,8 +30,9 @@ spec:
                 level: DEBUG
     configOverrides: &configOverrides
       core-site.xml:
-        # The idea is that the user "hive" can't do anything in hdfs, *but* it can impersonate other users
-        # (such as trino), that have the needed permissions
+        # The idea is that the user "hive" can't do anything in hdfs,
+        # *but* it can impersonate other users (such as trino),
+        # that have the needed permissions
         hadoop.proxyuser.hive.users: "*"
         hadoop.proxyuser.hive.hosts: "*"
     roleGroups:
diff --git a/stacks/end-to-end-security/opa.yaml b/stacks/end-to-end-security/opa.yaml
@@ -5,7 +5,7 @@ metadata:
   name: opa
 spec:
   image:
-    productVersion: 0.66.0
+    productVersion: 0.67.1
   clusterConfig:
     userInfo:
       backend:
diff --git a/stacks/end-to-end-security/trino-regorules.yaml b/stacks/end-to-end-security/trino-regorules.yaml
@@ -1,5 +1,5 @@
-{% raw %}
 ---
+# {% raw %}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -1460,7 +1460,7 @@ data:
     #         "groups": ["group1", ...],
     #         "user": "username",
     #       },
-    #       "softwareStack": {"trinoVersion": "442"},
+    #       "softwareStack": {"trinoVersion": "455"},
     #     }
     #   }
     #
@@ -1610,7 +1610,7 @@ data:
     #         "groups": ["group1", ...],
     #         "user": "username",
     #       },
-    #       "softwareStack": {"trinoVersion": "442"},
+    #       "softwareStack": {"trinoVersion": "455"},
     #     }
     #   }
     #
@@ -1671,7 +1671,7 @@ data:
     #         "groups": ["group1", ...],
     #         "user": "username",
     #       },
-    #       "softwareStack": {"trinoVersion": "442"},
+    #       "softwareStack": {"trinoVersion": "455"},
     #     }
     #   }
     #
@@ -1736,7 +1736,7 @@ data:
     #         "groups": ["group1", ...],
     #         "user": "username",
     #       },
-    #       "softwareStack": {"trinoVersion": "442"},
+    #       "softwareStack": {"trinoVersion": "455"},
     #     }
     #   }
     #
@@ -1797,4 +1797,4 @@ data:
 
       regex.match(pattern_with_anchors, value)
     }
-{% endraw %}
+# {% endraw %}
diff --git a/stacks/end-to-end-security/trino.yaml b/stacks/end-to-end-security/trino.yaml
@@ -5,7 +5,7 @@ metadata:
   name: trino
 spec:
   image:
-    productVersion: "451"
+    productVersion: "455"
   clusterConfig:
     listenerClass: external-unstable
     tls:
diff --git a/stacks/hdfs-hbase/hdfs.yaml b/stacks/hdfs-hbase/hdfs.yaml
@@ -1,10 +1,11 @@
+---
 apiVersion: hdfs.stackable.tech/v1alpha1
 kind: HdfsCluster
 metadata:
   name: hdfs
 spec:
   image:
-    productVersion: 3.3.4
+    productVersion: 3.4.0
   clusterConfig:
     dfsReplication: 1
     zookeeperConfigMapName: hdfs-znode
diff --git a/stacks/jupyterhub-pyspark-hdfs/hdfs.yaml b/stacks/jupyterhub-pyspark-hdfs/hdfs.yaml
@@ -13,7 +13,7 @@ metadata:
   name: hdfs
 spec:
   image:
-    productVersion: 3.3.4
+    productVersion: 3.4.0
   clusterConfig:
     dfsReplication: 1
     zookeeperConfigMapName: hdfs-znode
diff --git a/stacks/keycloak-opa-poc/druid.yaml b/stacks/keycloak-opa-poc/druid.yaml
diff --git a/stacks/keycloak-opa-poc/hdfs.yaml b/stacks/keycloak-opa-poc/hdfs.yaml
diff --git a/stacks/keycloak-opa-poc/opa.yaml b/stacks/keycloak-opa-poc/opa.yaml
diff --git a/stacks/keycloak-opa-poc/trino.yaml b/stacks/keycloak-opa-poc/trino.yaml
diff --git a/stacks/monitoring/grafana-dashboards.yaml b/stacks/monitoring/grafana-dashboards.yaml
diff --git a/stacks/nifi-kafka-druid-superset-s3/druid.yaml b/stacks/nifi-kafka-druid-superset-s3/druid.yaml
diff --git a/stacks/signal-processing/grafana-dashboards.yaml b/stacks/signal-processing/grafana-dashboards.yaml
diff --git a/stacks/trino-iceberg/trino.yaml b/stacks/trino-iceberg/trino.yaml
diff --git a/stacks/trino-superset-s3/trino.yaml b/stacks/trino-superset-s3/trino.yaml
