diff --git a/CHANGELOG.md b/CHANGELOG.md index 7880e08..8a2fce9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ All notable changes to this project will be documented in this file. images can be used which have a different directory structure than the Stackable image ([#18]). - Add Prometheus labels and annotations to role-group services ([#26]). - Helm: Allow Pod `priorityClassName` to be configured ([#34]). +- Support log configuration and log aggregation ([#40]). [#10]: https://github.com/stackabletech/opensearch-operator/pull/10 [#17]: https://github.com/stackabletech/opensearch-operator/pull/17 @@ -32,3 +33,4 @@ All notable changes to this project will be documented in this file. [#26]: https://github.com/stackabletech/opensearch-operator/pull/26 [#34]: https://github.com/stackabletech/opensearch-operator/pull/34 [#38]: https://github.com/stackabletech/opensearch-operator/pull/38 +[#40]: https://github.com/stackabletech/opensearch-operator/pull/40 diff --git a/Cargo.lock b/Cargo.lock index 18b1a51..4ab3341 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -553,6 +553,12 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -1904,6 +1910,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -2036,9 +2052,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.2" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c" dependencies = [ "aho-corasick", "memchr", @@ -2048,9 +2064,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad" dependencies = [ "aho-corasick", "memchr", @@ -2511,6 +2527,8 @@ dependencies = [ "built", "clap", "futures 0.3.31", + "pretty_assertions", + "regex", "rstest", "schemars", "serde", @@ -3582,6 +3600,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72e6e0a83ae73d886ab66fc2f82b598fbbb8f373357d5f2f9f783e50e4d06435" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.8.0" diff --git a/Cargo.nix b/Cargo.nix index 90abfeb..75dcbd5 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -1741,6 +1741,16 @@ rec { }; resolvedDefaultFeatures = [ "default" "from" ]; }; + "diff" = rec { + crateName = "diff"; + version = "0.1.13"; + edition = "2015"; + sha256 = "1j0nzjxci2zqx63hdcihkp0a4dkdmzxd7my4m7zk6cjyfy34j9an"; + authors = [ + "Utkarsh Kukreti " + ]; + + }; "digest" = rec { crateName = "digest"; version = "0.10.7"; @@ -6209,6 +6219,31 @@ rec { }; resolvedDefaultFeatures = [ "simd" "std" ]; }; + "pretty_assertions" = rec { + crateName = "pretty_assertions"; + version = "1.4.1"; + edition = "2018"; + sha256 = "0v8iq35ca4rw3rza5is3wjxwsf88303ivys07anc5yviybi31q9s"; + authors = [ + "Colin Kiegel " + "Florent Fayolle " + "Tom Milligan " + ]; + dependencies = [ + { + name = "diff"; + packageId = "diff"; + } + { + name = "yansi"; + packageId = "yansi"; + } + ]; + features = { + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "proc-macro-crate" = rec { crateName = "proc-macro-crate"; version = "3.4.0"; @@ -6564,9 +6599,9 @@ rec { }; "regex" = rec { crateName = "regex"; - version = "1.11.2"; + version = "1.11.3"; edition = "2021"; - sha256 = "04k9rzxd11hcahpyihlswy6f1zqw7lspirv4imm4h0lcdl8gvmr3"; + sha256 = "0b58ya98c4i5cjjiwhpcnjr61cv9g143qhdwhsryggj09098hllb"; authors = [ "The Rust Project Developers" "Andrew Gallant " @@ -6622,9 +6657,9 @@ rec { }; "regex-automata" = rec { crateName = "regex-automata"; - version = "0.4.10"; + version = "0.4.11"; edition = "2021"; - sha256 = "1mllcfmgjcl6d52d5k09lwwq9wj5mwxccix4bhmw5spy1gx5i53b"; + sha256 = "1bawj908pxixpggcnma3xazw53mwyz68lv9hn4yg63nlhv7bjgl3"; libName = "regex_automata"; authors = [ "The Rust Project Developers" @@ -8177,6 +8212,10 @@ rec { packageId = "futures 0.3.31"; features = [ "compat" ]; } + { + name = "regex"; + packageId = "regex"; + } { name = "schemars"; packageId = "schemars"; @@ -8227,6 +8266,10 @@ rec { } ]; devDependencies = [ + { + name = "pretty_assertions"; + packageId = "pretty_assertions"; + } { name = "rstest"; packageId = "rstest"; @@ -12992,6 +13035,24 @@ rec { ]; }; + "yansi" = rec { + crateName = "yansi"; + version = "1.0.1"; + edition = "2021"; + sha256 = "0jdh55jyv0dpd38ij4qh60zglbw9aa8wafqai6m0wa7xaxk3mrfg"; + authors = [ + "Sergio Benitez " + ]; + features = { + "default" = [ "std" ]; + "detect-env" = [ "std" ]; + "detect-tty" = [ "is-terminal" "std" ]; + "hyperlink" = [ "std" ]; + "is-terminal" = [ "dep:is-terminal" ]; + "std" = [ "alloc" ]; + }; + resolvedDefaultFeatures = [ "alloc" "default" "std" ]; + }; "yoke" = rec { crateName = "yoke"; version = "0.8.0"; diff --git a/Cargo.toml b/Cargo.toml index a180799..68c16e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,8 @@ stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", built = { version = "0.8.0", features = ["chrono", "git2"] } clap = "4.5" futures = { version = "0.3", features = ["compat"] } +pretty_assertions = "1.4" +regex = "1.11" rstest = "0.26" schemars = { version = "1.0.0", features = ["url2"] } # same as in operator-rs serde = { version = "1.0", features = ["derive"] } diff --git a/deploy/helm/opensearch-operator/crds/crds.yaml b/deploy/helm/opensearch-operator/crds/crds.yaml index 3443dda..12f3da0 100644 --- a/deploy/helm/opensearch-operator/crds/crds.yaml +++ b/deploy/helm/opensearch-operator/crds/crds.yaml @@ -28,6 +28,21 @@ spec: OpenSearch. Find more information on how to use it and the resources that the operator generates in the [operator documentation](https://docs.stackable.tech/home/nightly/opensearch/). properties: + clusterConfig: + default: {} + description: Configuration that applies to all roles and role groups + properties: + vectorAggregatorConfigMapName: + description: |- + Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). + It must contain the key `ADDRESS` with the address of the Vector aggregator. + Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) + to learn how to configure log aggregation with Vector. + maxLength: 253 + minLength: 1 + nullable: true + type: string + type: object clusterOperation: default: reconciliationPaused: false @@ -169,9 +184,107 @@ spec: nullable: true type: string listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. + description: |- + This field controls which + [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) + is used to expose the HTTP communication. + maxLength: 253 + minLength: 1 nullable: true type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object nodeRoles: description: |- Roles of the OpenSearch node. @@ -419,9 +532,107 @@ spec: nullable: true type: string listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. + description: |- + This field controls which + [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) + is used to expose the HTTP communication. + maxLength: 253 + minLength: 1 nullable: true type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object nodeRoles: description: |- Roles of the OpenSearch node. diff --git a/docs/modules/opensearch/pages/usage-guide/logging.adoc b/docs/modules/opensearch/pages/usage-guide/logging.adoc new file mode 100644 index 0000000..d9d4594 --- /dev/null +++ b/docs/modules/opensearch/pages/usage-guide/logging.adoc @@ -0,0 +1,34 @@ += Log aggregation +:description: The logs can be forwarded to a Vector log aggregator by providing a discovery ConfigMap for the aggregator and by enabling the log agent. + +The logs can be forwarded to a Vector log aggregator by providing a discovery ConfigMap for the aggregator and by enabling the log agent: + +[source,yaml] +---- +spec: + clusterConfig: + vectorAggregatorConfigMapName: vector-aggregator-discovery + nodes: + config: + logging: + enableVectorAgent: true + containers: + opensearch: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO + vector: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO +---- + +Further information on how to configure logging, can be found in xref:concepts:logging.adoc[]. diff --git a/docs/modules/opensearch/partials/nav.adoc b/docs/modules/opensearch/partials/nav.adoc index 42649b5..159b56d 100644 --- a/docs/modules/opensearch/partials/nav.adoc +++ b/docs/modules/opensearch/partials/nav.adoc @@ -7,6 +7,7 @@ ** xref:opensearch:usage-guide/storage-resource-configuration.adoc[] ** xref:opensearch:usage-guide/configuration-environment-overrides.adoc[] ** xref:opensearch:usage-guide/monitoring.adoc[] +** xref:opensearch:usage-guide/logging.adoc[] ** xref:opensearch:usage-guide/operations/index.adoc[] *** xref:opensearch:usage-guide/operations/cluster-operations.adoc[] *** xref:opensearch:usage-guide/operations/pod-placement.adoc[] diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index c5a9213..e93fbdc 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -14,6 +14,7 @@ stackable-operator.workspace = true clap.workspace = true futures.workspace = true +regex.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true @@ -27,4 +28,5 @@ uuid.workspace = true built.workspace = true [dev-dependencies] +pretty_assertions.workspace = true rstest.workspace = true diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 6add4d7..832e488 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -33,8 +33,9 @@ use crate::{ v1alpha1::{self}, }, framework::{ - ClusterName, ControllerName, HasName, HasUid, NameIsValidLabelValue, NamespaceName, - OperatorName, ProductName, ProductVersion, RoleGroupName, RoleName, Uid, + ClusterName, ControllerName, HasName, HasUid, ListenerClassName, NameIsValidLabelValue, + NamespaceName, OperatorName, ProductName, ProductVersion, RoleGroupName, RoleName, Uid, + product_logging::framework::{ValidatedContainerLogConfigChoice, VectorContainerLogConfig}, role_utils::{GenericProductSpecificCommonConfig, RoleGroupConfig}, }, }; @@ -123,14 +124,28 @@ type OpenSearchRoleGroupConfig = type OpenSearchNodeResources = stackable_operator::commons::resources::Resources; -/// The validated [`v1alpha1::OpenSearchConfig`] +/// Validated [`v1alpha1::OpenSearchConfig`] #[derive(Clone, Debug, PartialEq)] pub struct ValidatedOpenSearchConfig { pub affinity: StackableAffinity, + pub listener_class: ListenerClassName, + pub logging: ValidatedLogging, pub node_roles: NodeRoles, pub resources: OpenSearchNodeResources, pub termination_grace_period_seconds: i64, - pub listener_class: String, +} + +/// Validated log configuration per container +#[derive(Clone, Debug, PartialEq)] +pub struct ValidatedLogging { + pub opensearch_container: ValidatedContainerLogConfigChoice, + pub vector_container: Option, +} + +impl ValidatedLogging { + pub fn is_vector_agent_enabled(&self) -> bool { + self.vector_container.is_some() + } } /// The validated [`v1alpha1::OpenSearchCluster`] @@ -355,17 +370,20 @@ mod tests { commons::{affinity::StackableAffinity, product_image_selection::ResolvedProductImage}, k8s_openapi::api::core::v1::PodTemplateSpec, kvp::LabelValue, + product_logging::spec::AutomaticContainerLogConfig, role_utils::GenericRoleConfig, }; use uuid::uuid; - use super::{Context, OpenSearchRoleGroupConfig, ValidatedCluster}; + use super::{Context, OpenSearchRoleGroupConfig, ValidatedCluster, ValidatedLogging}; use crate::{ controller::{OpenSearchNodeResources, ValidatedOpenSearchConfig}, crd::{NodeRoles, v1alpha1}, framework::{ - ClusterName, NamespaceName, OperatorName, ProductVersion, RoleGroupName, - builder::pod::container::EnvVarSet, role_utils::GenericProductSpecificCommonConfig, + ClusterName, ListenerClassName, NamespaceName, OperatorName, ProductVersion, + RoleGroupName, builder::pod::container::EnvVarSet, + product_logging::framework::ValidatedContainerLogConfigChoice, + role_utils::GenericProductSpecificCommonConfig, }, }; @@ -487,10 +505,16 @@ mod tests { replicas, config: ValidatedOpenSearchConfig { affinity: StackableAffinity::default(), + listener_class: ListenerClassName::from_str_unsafe("external-stable"), + logging: ValidatedLogging { + opensearch_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + vector_container: None, + }, node_roles: NodeRoles(node_roles.to_vec()), resources: OpenSearchNodeResources::default(), termination_grace_period_seconds: 120, - listener_class: "external-stable".to_owned(), }, config_overrides: HashMap::default(), env_overrides: EnvVarSet::default(), diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index 8764751..43c3bda 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -7,6 +7,7 @@ use role_builder::RoleBuilder; use super::{ContextNames, KubernetesResources, Prepared, ValidatedCluster}; pub mod node_config; +pub mod product_logging; pub mod role_builder; pub mod role_group_builder; @@ -65,6 +66,7 @@ mod tests { k8s_openapi::api::core::v1::PodTemplateSpec, kube::Resource, kvp::LabelValue, + product_logging::spec::AutomaticContainerLogConfig, role_utils::GenericRoleConfig, }; use uuid::uuid; @@ -73,12 +75,12 @@ mod tests { use crate::{ controller::{ ContextNames, OpenSearchNodeResources, OpenSearchRoleGroupConfig, ValidatedCluster, - ValidatedOpenSearchConfig, + ValidatedContainerLogConfigChoice, ValidatedLogging, ValidatedOpenSearchConfig, }, crd::{NodeRoles, v1alpha1}, framework::{ - ClusterName, ControllerName, NamespaceName, OperatorName, ProductName, ProductVersion, - RoleGroupName, builder::pod::container::EnvVarSet, + ClusterName, ControllerName, ListenerClassName, NamespaceName, OperatorName, + ProductName, ProductVersion, RoleGroupName, builder::pod::container::EnvVarSet, role_utils::GenericProductSpecificCommonConfig, }, }; @@ -200,10 +202,16 @@ mod tests { replicas, config: ValidatedOpenSearchConfig { affinity: StackableAffinity::default(), + listener_class: ListenerClassName::from_str_unsafe("external-stable"), + logging: ValidatedLogging { + opensearch_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + vector_container: None, + }, node_roles: NodeRoles(node_roles.to_vec()), resources: OpenSearchNodeResources::default(), termination_grace_period_seconds: 120, - listener_class: "external-stable".to_owned(), }, config_overrides: HashMap::default(), env_overrides: EnvVarSet::default(), diff --git a/rust/operator-binary/src/controller/build/node_config.rs b/rust/operator-binary/src/controller/build/node_config.rs index 8a18ddc..db795f7 100644 --- a/rust/operator-binary/src/controller/build/node_config.rs +++ b/rust/operator-binary/src/controller/build/node_config.rs @@ -81,7 +81,7 @@ impl NodeConfig { } /// Creates the main OpenSearch configuration file in YAML format - pub fn static_opensearch_config_file(&self) -> String { + pub fn static_opensearch_config_file_content(&self) -> String { Self::to_yaml(self.static_opensearch_config()) } @@ -156,19 +156,19 @@ impl NodeConfig { // Set the OpenSearch node name to the Pod name. // The node name is used e.g. for INITIAL_CLUSTER_MANAGER_NODES. .with_field_path( - EnvVarName::from_str_unsafe(CONFIG_OPTION_NODE_NAME), + &EnvVarName::from_str_unsafe(CONFIG_OPTION_NODE_NAME), FieldPathEnvVar::Name, ) .with_value( - EnvVarName::from_str_unsafe(CONFIG_OPTION_DISCOVERY_SEED_HOSTS), + &EnvVarName::from_str_unsafe(CONFIG_OPTION_DISCOVERY_SEED_HOSTS), &self.discovery_service_name, ) .with_value( - EnvVarName::from_str_unsafe(CONFIG_OPTION_INITIAL_CLUSTER_MANAGER_NODES), + &EnvVarName::from_str_unsafe(CONFIG_OPTION_INITIAL_CLUSTER_MANAGER_NODES), self.initial_cluster_manager_nodes(), ) .with_value( - EnvVarName::from_str_unsafe(CONFIG_OPTION_NODE_ROLES), + &EnvVarName::from_str_unsafe(CONFIG_OPTION_NODE_ROLES), self.role_group_config .config .node_roles @@ -275,32 +275,53 @@ mod tests { }, k8s_openapi::api::core::v1::PodTemplateSpec, kvp::LabelValue, + product_logging::spec::AutomaticContainerLogConfig, role_utils::GenericRoleConfig, }; use uuid::uuid; use super::*; use crate::{ - controller::ValidatedOpenSearchConfig, + controller::{ValidatedLogging, ValidatedOpenSearchConfig}, crd::NodeRoles, framework::{ - ClusterName, NamespaceName, ProductVersion, RoleGroupName, + ClusterName, ListenerClassName, NamespaceName, ProductVersion, RoleGroupName, + product_logging::framework::ValidatedContainerLogConfigChoice, role_utils::GenericProductSpecificCommonConfig, }, }; - pub fn node_config( + struct TestConfig { replicas: u16, - config_settings: &[(&str, &str)], - env_vars: &[(&str, &str)], - ) -> NodeConfig { + config_settings: &'static [(&'static str, &'static str)], + env_vars: &'static [(&'static str, &'static str)], + } + + impl Default for TestConfig { + fn default() -> Self { + Self { + replicas: 3, + config_settings: &[], + env_vars: &[], + } + } + } + + fn node_config(test_config: TestConfig) -> NodeConfig { let image: ProductImage = serde_json::from_str(r#"{"productVersion": "3.1.0"}"#) .expect("should be a valid ProductImage"); let role_group_config = OpenSearchRoleGroupConfig { - replicas, + replicas: test_config.replicas, config: ValidatedOpenSearchConfig { affinity: StackableAffinity::default(), + listener_class: ListenerClassName::from_str_unsafe("cluster-internal"), + logging: ValidatedLogging { + opensearch_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + vector_container: None, + }, node_roles: NodeRoles(vec![ v1alpha1::NodeRole::ClusterManager, v1alpha1::NodeRole::Data, @@ -309,18 +330,19 @@ mod tests { ]), resources: Resources::default(), termination_grace_period_seconds: 30, - listener_class: "cluster-internal".to_string(), }, config_overrides: [( CONFIGURATION_FILE_OPENSEARCH_YML.to_owned(), - config_settings + test_config + .config_settings .iter() .map(|(k, v)| (k.to_string(), v.to_string())) .collect(), )] .into(), env_overrides: EnvVarSet::new().with_values( - env_vars + test_config + .env_vars .iter() .map(|(k, v)| (EnvVarName::from_str_unsafe(k), *v)), ), @@ -359,7 +381,10 @@ mod tests { #[test] pub fn test_static_opensearch_config_file() { - let node_config = node_config(2, &[("test", "value")], &[]); + let node_config = node_config(TestConfig { + config_settings: &[("test", "value")], + ..TestConfig::default() + }); assert_eq!( concat!( @@ -370,17 +395,23 @@ mod tests { "test: \"value\"" ) .to_owned(), - node_config.static_opensearch_config_file() + node_config.static_opensearch_config_file_content() ); } #[test] pub fn test_tls_on_http_port_enabled() { - let node_config_tls_undefined = node_config(2, &[], &[]); - let node_config_tls_enabled = - node_config(2, &[("plugins.security.ssl.http.enabled", "true")], &[]); - let node_config_tls_disabled = - node_config(2, &[("plugins.security.ssl.http.enabled", "false")], &[]); + let node_config_tls_undefined = node_config(TestConfig::default()); + + let node_config_tls_enabled = node_config(TestConfig { + config_settings: &[("plugins.security.ssl.http.enabled", "true")], + ..TestConfig::default() + }); + + let node_config_tls_disabled = node_config(TestConfig { + config_settings: &[("plugins.security.ssl.http.enabled", "false")], + ..TestConfig::default() + }); assert!(!node_config_tls_undefined.tls_on_http_port_enabled()); assert!(node_config_tls_enabled.tls_on_http_port_enabled()); @@ -426,25 +457,29 @@ mod tests { #[test] pub fn test_environment_variables() { - let node_config = node_config(2, &[], &[("TEST", "value")]); + let node_config = node_config(TestConfig { + replicas: 2, + env_vars: &[("TEST", "value")], + ..TestConfig::default() + }); assert_eq!( EnvVarSet::new() - .with_value(EnvVarName::from_str_unsafe("TEST"), "value",) + .with_value(&EnvVarName::from_str_unsafe("TEST"), "value") .with_value( - EnvVarName::from_str_unsafe("cluster.initial_cluster_manager_nodes"), + &EnvVarName::from_str_unsafe("cluster.initial_cluster_manager_nodes"), "my-opensearch-cluster-nodes-default-0,my-opensearch-cluster-nodes-default-1", ) .with_value( - EnvVarName::from_str_unsafe("discovery.seed_hosts"), + &EnvVarName::from_str_unsafe("discovery.seed_hosts"), "my-opensearch-cluster-manager", ) .with_field_path( - EnvVarName::from_str_unsafe("node.name"), + &EnvVarName::from_str_unsafe("node.name"), FieldPathEnvVar::Name ) .with_value( - EnvVarName::from_str_unsafe("node.roles"), + &EnvVarName::from_str_unsafe("node.roles"), "cluster_manager,data,ingest,remote_cluster_client" ), node_config.environment_variables() @@ -453,8 +488,15 @@ mod tests { #[test] pub fn test_discovery_type() { - let node_config_single_node = node_config(1, &[], &[]); - let node_config_multiple_nodes = node_config(2, &[], &[]); + let node_config_single_node = node_config(TestConfig { + replicas: 1, + ..TestConfig::default() + }); + + let node_config_multiple_nodes = node_config(TestConfig { + replicas: 2, + ..TestConfig::default() + }); assert_eq!( "single-node".to_owned(), @@ -468,8 +510,15 @@ mod tests { #[test] pub fn test_initial_cluster_manager_nodes() { - let node_config_single_node = node_config(1, &[], &[]); - let node_config_multiple_nodes = node_config(3, &[], &[]); + let node_config_single_node = node_config(TestConfig { + replicas: 1, + ..TestConfig::default() + }); + + let node_config_multiple_nodes = node_config(TestConfig { + replicas: 3, + ..TestConfig::default() + }); assert_eq!( "".to_owned(), diff --git a/rust/operator-binary/src/controller/build/product_logging.rs b/rust/operator-binary/src/controller/build/product_logging.rs new file mode 100644 index 0000000..ef68c36 --- /dev/null +++ b/rust/operator-binary/src/controller/build/product_logging.rs @@ -0,0 +1 @@ +pub mod config; diff --git a/rust/operator-binary/src/controller/build/product_logging/config.rs b/rust/operator-binary/src/controller/build/product_logging/config.rs new file mode 100644 index 0000000..c70694c --- /dev/null +++ b/rust/operator-binary/src/controller/build/product_logging/config.rs @@ -0,0 +1,271 @@ +//! OpenSearch specific log configuration + +use std::{cmp, collections::BTreeMap}; + +use stackable_operator::{ + memory::{BinaryMultiple, MemoryQuantity}, + product_logging::spec::{AppenderConfig, AutomaticContainerLogConfig, LogLevel, LoggerConfig}, +}; + +use crate::{ + crd::v1alpha1::{self}, + framework::{ + builder::pod::container::{EnvVarName, EnvVarSet}, + product_logging::framework::STACKABLE_LOG_DIR, + }, +}; + +/// OpenSearch log configuration file +pub const CONFIGURATION_FILE_LOG4J2_PROPERTIES: &str = "log4j2.properties"; + +const OPENSEARCH_SERVER_LOG_FILE: &str = "opensearch_server.json"; + +pub const MAX_OPENSEARCH_SERVER_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { + value: 10.0, + unit: BinaryMultiple::Mebi, +}; + +/// Create a log4j2 configuration from the given automatic log configuration +pub fn create_log4j2_config(config: &AutomaticContainerLogConfig) -> String { + [ + log4j2_root_logger_config(&config.root_log_level()), + log4j2_loggers_config(&config.loggers), + log4j2_console_appender_config(&config.console), + log4j2_file_appender_config(&config.file), + ] + .iter() + .flatten() + .map(|(key, value)| format!("{key} = {value}\n")) + .collect() +} + +fn log4j2_root_logger_config(root_log_level: &LogLevel) -> Vec<(String, String)> { + vec![ + ( + "rootLogger.level".to_owned(), + root_log_level.to_log4j2_literal(), + ), + ( + "rootLogger.appenderRef.CONSOLE.ref".to_owned(), + "CONSOLE".to_owned(), + ), + ( + "rootLogger.appenderRef.FILE.ref".to_owned(), + "FILE".to_owned(), + ), + ] +} + +fn log4j2_loggers_config(loggers_config: &BTreeMap) -> Vec<(String, String)> { + loggers_config + .iter() + .filter(|(name, _)| name.as_str() != AutomaticContainerLogConfig::ROOT_LOGGER) + .enumerate() + .flat_map(|(index, (name, logger_config))| { + [ + ( + format!("logger.{index}.name"), + name.escape_default().to_string(), + ), + ( + format!("logger.{index}.level"), + logger_config.level.to_log4j_literal(), + ), + ] + }) + .collect::>() +} + +fn log4j2_console_appender_config( + console_appender_config: &Option, +) -> Vec<(String, String)> { + vec![ + ("appender.CONSOLE.type".to_owned(), "Console".to_owned()), + ("appender.CONSOLE.name".to_owned(), "CONSOLE".to_owned()), + ( + "appender.CONSOLE.target".to_owned(), + "SYSTEM_ERR".to_owned(), + ), + ( + "appender.CONSOLE.layout.type".to_owned(), + "PatternLayout".to_owned(), + ), + // Same as the default layout pattern of the console appender + // see https://github.com/opensearch-project/OpenSearch/blob/3.1.0/distribution/src/config/log4j2.properties#L17 + ( + "appender.CONSOLE.layout.pattern".to_owned(), + "[%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n".to_owned(), + ), + ( + "appender.CONSOLE.filter.threshold.type".to_owned(), + "ThresholdFilter".to_owned(), + ), + ( + "appender.CONSOLE.filter.threshold.level".to_owned(), + console_appender_config + .as_ref() + .and_then(|console| console.level) + .unwrap_or_default() + .to_log4j2_literal(), + ), + ] +} + +fn log4j2_file_appender_config( + file_appender_config: &Option, +) -> Vec<(String, String)> { + let log_path = format!( + "{STACKABLE_LOG_DIR}/{container}/{OPENSEARCH_SERVER_LOG_FILE}", + container = v1alpha1::Container::OpenSearch.to_container_name() + ); + + let number_of_archived_log_files = 1; + let max_log_files_size_in_mib = MAX_OPENSEARCH_SERVER_LOG_FILES_SIZE + .scale_to(BinaryMultiple::Mebi) + .floor() + .value as u32; + let max_log_file_size_in_mib = cmp::max( + 1, + max_log_files_size_in_mib / (1 + number_of_archived_log_files), + ); + + vec![ + ("appender.FILE.type".to_owned(), "RollingFile".to_owned()), + ("appender.FILE.name".to_owned(), "FILE".to_owned()), + ("appender.FILE.fileName".to_owned(), log_path.to_owned()), + ( + "appender.FILE.filePattern".to_owned(), + format!("{log_path}.%i"), + ), + ( + "appender.FILE.layout.type".to_owned(), + "OpenSearchJsonLayout".to_owned(), + ), + ( + "appender.FILE.layout.type_name".to_owned(), + "server".to_owned(), + ), + ( + "appender.FILE.policies.type".to_owned(), + "Policies".to_owned(), + ), + ( + "appender.FILE.policies.size.type".to_owned(), + "SizeBasedTriggeringPolicy".to_owned(), + ), + ( + "appender.FILE.policies.size.size".to_owned(), + format!("{max_log_file_size_in_mib}MB"), + ), + ( + "appender.FILE.strategy.type".to_owned(), + "DefaultRolloverStrategy".to_owned(), + ), + ( + "appender.FILE.strategy.max".to_owned(), + number_of_archived_log_files.to_string(), + ), + ( + "appender.FILE.filter.threshold.type".to_owned(), + "ThresholdFilter".to_owned(), + ), + ( + "appender.FILE.filter.threshold.level".to_owned(), + file_appender_config + .as_ref() + .and_then(|file| file.level) + .unwrap_or_default() + .to_log4j2_literal(), + ), + ] +} + +/// Returns the Vector configuration file content as YAML +pub fn vector_config_file_content() -> String { + include_str!("vector.yaml").to_owned() +} + +/// Returns the OpenSearch specific environment variables used in the Vector configuration file +/// +/// The common environment variables are already set in +/// [`crate::framework::product_logging::framework::vector_container`]. +pub fn vector_config_file_extra_env_vars() -> EnvVarSet { + EnvVarSet::new().with_value( + &EnvVarName::from_str_unsafe("OPENSEARCH_SERVER_LOG_FILE"), + "opensearch_server.json", + ) +} + +#[cfg(test)] +mod tests { + use stackable_operator::product_logging::spec::{ + AppenderConfig, AutomaticContainerLogConfig, LogLevel, LoggerConfig, + }; + + use super::{create_log4j2_config, vector_config_file_extra_env_vars}; + + #[test] + pub fn test_create_log4j2_config() { + let log4j2_config = create_log4j2_config(&AutomaticContainerLogConfig { + loggers: [ + ( + "org.opensearch.index.reindex".to_owned(), + LoggerConfig { + level: LogLevel::DEBUG, + }, + ), + ( + "org.opensearch.indices.recovery".to_owned(), + LoggerConfig { + level: LogLevel::TRACE, + }, + ), + ] + .into(), + console: Some(AppenderConfig { + level: Some(LogLevel::WARN), + }), + file: Some(AppenderConfig { + level: Some(LogLevel::DEBUG), + }), + }); + + let expected_config = concat!( + "rootLogger.level = INFO\n", + "rootLogger.appenderRef.CONSOLE.ref = CONSOLE\n", + "rootLogger.appenderRef.FILE.ref = FILE\n", + "logger.0.name = org.opensearch.index.reindex\n", + "logger.0.level = DEBUG\n", + "logger.1.name = org.opensearch.indices.recovery\n", + "logger.1.level = TRACE\n", + "appender.CONSOLE.type = Console\n", + "appender.CONSOLE.name = CONSOLE\n", + "appender.CONSOLE.target = SYSTEM_ERR\n", + "appender.CONSOLE.layout.type = PatternLayout\n", + "appender.CONSOLE.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n\n", + "appender.CONSOLE.filter.threshold.type = ThresholdFilter\n", + "appender.CONSOLE.filter.threshold.level = WARN\n", + "appender.FILE.type = RollingFile\n", + "appender.FILE.name = FILE\n", + "appender.FILE.fileName = /stackable/log/opensearch/opensearch_server.json\n", + "appender.FILE.filePattern = /stackable/log/opensearch/opensearch_server.json.%i\n", + "appender.FILE.layout.type = OpenSearchJsonLayout\n", + "appender.FILE.layout.type_name = server\n", + "appender.FILE.policies.type = Policies\n", + "appender.FILE.policies.size.type = SizeBasedTriggeringPolicy\n", + "appender.FILE.policies.size.size = 5MB\n", + "appender.FILE.strategy.type = DefaultRolloverStrategy\n", + "appender.FILE.strategy.max = 1\n", + "appender.FILE.filter.threshold.type = ThresholdFilter\n", + "appender.FILE.filter.threshold.level = DEBUG\n", + ).to_owned(); + + assert_eq!(expected_config, log4j2_config); + } + + #[test] + pub fn test_vector_config_file_extra_env_vars() { + // Test that the function does not panic + vector_config_file_extra_env_vars(); + } +} diff --git a/rust/operator-binary/src/controller/build/product_logging/test-vector.sh b/rust/operator-binary/src/controller/build/product_logging/test-vector.sh new file mode 100755 index 0000000..cf43eab --- /dev/null +++ b/rust/operator-binary/src/controller/build/product_logging/test-vector.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env sh + +LOG_DIR=/stackable/log \ +OPENSEARCH_SERVER_LOG_FILE=opensearch_server.json \ +NAMESPACE=default \ +CLUSTER_NAME=opensearch \ +ROLE_NAME=nodes \ +ROLE_GROUP_NAME=cluster-manager \ +VECTOR_AGGREGATOR_ADDRESS=vector-aggregator \ +VECTOR_FILE_LOG_LEVEL=info \ +vector test vector.yaml vector-test.yaml diff --git a/rust/operator-binary/src/controller/build/product_logging/vector-test.yaml b/rust/operator-binary/src/controller/build/product_logging/vector-test.yaml new file mode 100644 index 0000000..7ffa2cd --- /dev/null +++ b/rust/operator-binary/src/controller/build/product_logging/vector-test.yaml @@ -0,0 +1,463 @@ +# Run tests with `./test-vector.sh` +# +# A downside of these test cases is that they compare the whole event and that the message can +# contain source code positions in vector.yaml, e.g. "function call error for \"parse_timestamp\" at (584:643)". Please adapt the tests if you change VRL code in vector.yaml. +--- +tests: + - name: Test opensearch_server log entry without stacktrace + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "2025-10-01T12:47:28,582Z", "level": "INFO", "component": "o.o.n.Node", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "started", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "file": "opensearch_server.json", + "level": "INFO", + "logger": "o.o.n.Node", + "message": "started", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": t'2025-10-01T10:47:28.582Z' + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry with stacktrace + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "2025-10-01T12:47:28,363Z", "level": "INFO", "component": "o.o.c.c.JoinHelper", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "failed to join {opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true} with JoinRequest{sourceNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}, minimumTerm=0, optionalJoin=Optional[Join{term=1, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}, targetNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}}]}", + "stacktrace": ["org.opensearch.transport.RemoteTransportException: [opensearch-nodes-cluster-manager-0][10.244.0.20:9300][internal:cluster/coordination/join]", + "Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: became follower", + "at org.opensearch.cluster.coordination.JoinHelper$$CandidateJoinAccumulator.lambda$$close$$3(JoinHelper.java:648) ~[opensearch-3.1.0.jar:3.1.0]", + "at java.base/java.util.HashMap$$Values.forEach(HashMap.java:1073) ~[?:?]", + "at org.opensearch.cluster.coordination.JoinHelper$$CandidateJoinAccumulator.close(JoinHelper.java:648) ~[opensearch-3.1.0.jar:3.1.0]", + "at org.opensearch.cluster.coordination.Coordinator.becomeFollower(Coordinator.java:829) ~[opensearch-3.1.0.jar:3.1.0]", + "at org.opensearch.cluster.coordination.Coordinator.onFollowerCheckRequest(Coordinator.java:405) ~[opensearch-3.1.0.jar:3.1.0]", + "at org.opensearch.cluster.coordination.FollowersChecker$$2.doRun(FollowersChecker.java:250) ~[opensearch-3.1.0.jar:3.1.0]", + "at org.opensearch.common.util.concurrent.ThreadContext$$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:975) ~[opensearch-3.1.0.jar:3.1.0]", + "at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) ~[opensearch-3.1.0.jar:3.1.0]", + "at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]", + "at java.base/java.util.concurrent.ThreadPoolExecutor$$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]", + "at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]"] } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473451193Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "file": "opensearch_server.json", + "level": "INFO", + "logger": "o.o.c.c.JoinHelper", + "message": "\ + failed to join {opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true} with JoinRequest{sourceNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}, minimumTerm=0, optionalJoin=Optional[Join{term=1, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}, targetNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}}]}\n\ + \n\ + org.opensearch.transport.RemoteTransportException: [opensearch-nodes-cluster-manager-0][10.244.0.20:9300][internal:cluster/coordination/join]\n\ + Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: became follower\n\ + at org.opensearch.cluster.coordination.JoinHelper$$CandidateJoinAccumulator.lambda$$close$$3(JoinHelper.java:648) ~[opensearch-3.1.0.jar:3.1.0]\n\ + at java.base/java.util.HashMap$$Values.forEach(HashMap.java:1073) ~[?:?]\n\ + at org.opensearch.cluster.coordination.JoinHelper$$CandidateJoinAccumulator.close(JoinHelper.java:648) ~[opensearch-3.1.0.jar:3.1.0]\n\ + at org.opensearch.cluster.coordination.Coordinator.becomeFollower(Coordinator.java:829) ~[opensearch-3.1.0.jar:3.1.0]\n\ + at org.opensearch.cluster.coordination.Coordinator.onFollowerCheckRequest(Coordinator.java:405) ~[opensearch-3.1.0.jar:3.1.0]\n\ + at org.opensearch.cluster.coordination.FollowersChecker$$2.doRun(FollowersChecker.java:250) ~[opensearch-3.1.0.jar:3.1.0]\n\ + at org.opensearch.common.util.concurrent.ThreadContext$$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:975) ~[opensearch-3.1.0.jar:3.1.0]\n\ + at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) ~[opensearch-3.1.0.jar:3.1.0]\n\ + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]\n\ + at java.base/java.util.concurrent.ThreadPoolExecutor$$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]\n\ + at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": t'2025-10-01T10:47:28.363Z' + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry with unparsable message + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "2025-10-01T12:47:28,582Z", "level": "INFO", "component": "o.o.n.Node", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "started", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "errors": [ + "JSON not parsable: function call error for \"parse_json\" at (110:133): unable to parse json: EOF while parsing an object at line 2 column 0" + ], + "file": "opensearch_server.json", + "level": "INFO", + "logger": "", + "message": "{\"type\": \"server\", \"timestamp\": \"2025-10-01T12:47:28,582Z\", \"level\": \"INFO\", \"component\": \"o.o.n.Node\", \"cluster.name\": \"opensearch\", \"node.name\": \"opensearch-nodes-cluster-manager-0\", \"message\": \"started\", \"cluster.uuid\": \"Jh1D6YAhTmyzkHI7vM1WWw\", \"node.id\": \"sk-r0P_TTYuPqaamTFbjKg\"\n", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": "2025-10-01T12:47:29.473487331Z" + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry without a JSON object in the message field + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + false + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "errors": [ + "Parsed event is not a JSON object." + ], + "file": "opensearch_server.json", + "level": "INFO", + "logger": "", + "message": "false\n", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": "2025-10-01T12:47:29.473487331Z" + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry with unparsable timestamp + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "unparsable", "level": "INFO", "component": "o.o.n.Node", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "started", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "errors": [ + "Timestamp not parsable, using current time instead: function call error for \"parse_timestamp\" at (584:643): Invalid timestamp \"unparsable\": input contains invalid characters" + ], + "file": "opensearch_server.json", + "level": "INFO", + "logger": "o.o.n.Node", + "message": "started", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": "2025-10-01T12:47:29.473487331Z" + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry without timestamp + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "level": "INFO", "component": "o.o.n.Node", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "started", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "errors": [ + "Timestamp not found, using current time instead." + ], + "file": "opensearch_server.json", + "level": "INFO", + "logger": "o.o.n.Node", + "message": "started", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": "2025-10-01T12:47:29.473487331Z" + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry without logger + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "2025-10-01T12:47:28,582Z", "level": "INFO", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "started", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "errors": [ + "Logger not found." + ], + "file": "opensearch_server.json", + "level": "INFO", + "logger": "", + "message": "started", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": t'2025-10-01T10:47:28.582Z' + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry without level + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "2025-10-01T12:47:28,582Z", "component": "o.o.n.Node", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "started", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "file": "opensearch_server.json", + "errors": [ + "Level not found, using \"INFO\" instead." + ], + "level": "INFO", + "logger": "o.o.n.Node", + "message": "started", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": t'2025-10-01T10:47:28.582Z' + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry with unknown level + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "2025-10-01T12:47:28,582Z", "level": "CRITICAL", "component": "o.o.n.Node", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "started", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "errors": [ + "Level \"CRITICAL\" unknown, using \"INFO\" instead." + ], + "file": "opensearch_server.json", + "level": "INFO", + "logger": "o.o.n.Node", + "message": "started", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": t'2025-10-01T10:47:28.582Z' + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry without message + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "2025-10-01T12:47:28,582Z", "level": "INFO", "component": "o.o.n.Node", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "errors": [ + "Message not found." + ], + "file": "opensearch_server.json", + "level": "INFO", + "logger": "o.o.n.Node", + "message": "", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": t'2025-10-01T10:47:28.582Z' + } + + assert_eq!(expected_log_event, .) + - name: Test Vector internal logs + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + arch: x86_64 + debug: false, + message: Vector has started. + metadata: + kind: event + level: INFO + module_path: vector::internal_events::process + target: vector + pid: 14 + pod: opensearch-nodes-cluster-manager-0 + revision: dc7e792 2025-08-12 13:47:08.632326804 + source_type: internal_logs + timestamp: 2025-10-02T09:46:14.479381097Z + version: 0.49.0 + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "arch": "x86_64", + "cluster": "opensearch", + "container": "vector", + "debug": "false,", + "level": "INFO", + "logger": "vector::internal_events::process", + "message": "Vector has started.", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "revision": "dc7e792 2025-08-12 13:47:08.632326804", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": "2025-10-02T09:46:14.479381097Z", + "version": "0.49.0" + } + + assert_eq!(expected_log_event, .) + - name: Test Vector internal log level filtering - TRACE/DEBUG + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: TRACE + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: DEBUG + no_outputs_from: + - filtered_logs_vector + - name: Test Vector internal log level filtering - INFO + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: INFO + outputs: + - extract_from: filtered_logs_vector + conditions: + - type: vrl + source: | + assert_eq!("INFO", .metadata.level) + - name: Test Vector internal log level filtering - WARN + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: WARN + outputs: + - extract_from: filtered_logs_vector + conditions: + - type: vrl + source: | + assert_eq!("WARN", .metadata.level) + - name: Test Vector internal log level filtering - ERROR + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: ERROR + outputs: + - extract_from: filtered_logs_vector + conditions: + - type: vrl + source: | + assert_eq!("ERROR", .metadata.level) diff --git a/rust/operator-binary/src/controller/build/product_logging/vector.yaml b/rust/operator-binary/src/controller/build/product_logging/vector.yaml new file mode 100644 index 0000000..c01ac7c --- /dev/null +++ b/rust/operator-binary/src/controller/build/product_logging/vector.yaml @@ -0,0 +1,148 @@ +--- +data_dir: /stackable/vector/var + +log_schema: + host_key: pod + +sources: + # Reads logs created with the OpenSearchJsonLayout and the type server + files_opensearch_server: + type: file + include: + - ${LOG_DIR}/*/${OPENSEARCH_SERVER_LOG_FILE} + multiline: + condition_pattern: ^[^{] + mode: continue_through + start_pattern: ^\{ + timeout_ms: 100 + + # Reads the internal Vector logs + vector: + type: internal_logs + +transforms: + # Transforms logs created with the OpenSearchJsonLayout and the type server + processed_files_opensearch_server: + inputs: + - files_opensearch_server + type: remap + source: | + raw_message = string!(.message) + + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + parsed_event, err = parse_json(raw_message) + if err != null { + error = "JSON not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else if !is_object(parsed_event) { + error = "Parsed event is not a JSON object." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event) + + timestamp_string, err = string(event.timestamp) + if err == null { + parsed_timestamp, err = parse_timestamp(timestamp_string, "%Y-%m-%dT%H:%M:%S,%3fZ") + if err == null { + .timestamp = parsed_timestamp + } else { + .errors = push(.errors, "Timestamp not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "Timestamp not found, using current time instead.") + } + + .logger, err = string(event.component) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = level + } + + .message, err = string(event.message) + if err != null || is_empty(.message) { + .errors = push(.errors, "Message not found.") + } + stacktrace = join(event.stacktrace, "\n") ?? "" + .message = join!(compact([.message, stacktrace]), "\n\n") + } + + # Extends the processed files with the fields "container" and "file" + extended_logs_files: + inputs: + - processed_files_* + type: remap + source: | + del(.source_type) + if .errors == [] { + del(.errors) + } + . |= parse_regex!(.file, r'^${LOG_DIR}/(?P.*?)/(?P.*?)$') + + # Filters the logs of the Vector agent according to the defined log level + filtered_logs_vector: + inputs: + - vector + type: filter + condition: > + (.metadata.level == "TRACE" && "${VECTOR_FILE_LOG_LEVEL}" == "trace") || + (.metadata.level == "DEBUG" && includes(["trace", "debug"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "INFO" && includes(["trace", "debug", "info"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "WARN" && includes(["trace", "debug", "info", "warn"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "ERROR" && includes(["trace", "debug", "info", "warn", "error"], "${VECTOR_FILE_LOG_LEVEL}")) + + # Aligns the logs of the Vector agent with the common format + extended_logs_vector: + inputs: + - filtered_logs_vector + type: remap + source: | + .container = "vector" + .level = .metadata.level + .logger = .metadata.module_path + if exists(.file) { + .processed_file = del(.file) + } + del(.metadata) + del(.pid) + del(.source_type) + + # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs + extended_logs: + inputs: + - extended_logs_* + type: remap + source: | + .namespace = "${NAMESPACE}" + .cluster = "${CLUSTER_NAME}" + .role = "${ROLE_NAME}" + .roleGroup = "${ROLE_GROUP_NAME}" + +sinks: + # Forward the logs to the Vector aggregator + aggregator: + inputs: + - extended_logs + type: vector + address: ${VECTOR_AGGREGATOR_ADDRESS} + console: + inputs: + - vector + type: console + encoding: + codec: json diff --git a/rust/operator-binary/src/controller/build/role_builder.rs b/rust/operator-binary/src/controller/build/role_builder.rs index 0ad2bcb..454d7d5 100644 --- a/rust/operator-binary/src/controller/build/role_builder.rs +++ b/rust/operator-binary/src/controller/build/role_builder.rs @@ -218,6 +218,7 @@ mod tests { str::FromStr, }; + use pretty_assertions::assert_eq; use serde_json::json; use stackable_operator::{ commons::{ @@ -227,6 +228,7 @@ mod tests { }, k8s_openapi::api::core::v1::PodTemplateSpec, kvp::LabelValue, + product_logging::spec::AutomaticContainerLogConfig, role_utils::GenericRoleConfig, }; use uuid::uuid; @@ -234,12 +236,13 @@ mod tests { use super::RoleBuilder; use crate::{ controller::{ - ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster, ValidatedOpenSearchConfig, + ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster, + ValidatedContainerLogConfigChoice, ValidatedLogging, ValidatedOpenSearchConfig, }, crd::{NodeRoles, v1alpha1}, framework::{ - ClusterName, ControllerName, NamespaceName, OperatorName, ProductName, ProductVersion, - RoleGroupName, builder::pod::container::EnvVarSet, + ClusterName, ControllerName, ListenerClassName, NamespaceName, OperatorName, + ProductName, ProductVersion, RoleGroupName, builder::pod::container::EnvVarSet, role_utils::GenericProductSpecificCommonConfig, }, }; @@ -260,6 +263,13 @@ mod tests { replicas: 1, config: ValidatedOpenSearchConfig { affinity: StackableAffinity::default(), + listener_class: ListenerClassName::from_str_unsafe("cluster-internal"), + logging: ValidatedLogging { + opensearch_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + vector_container: None, + }, node_roles: NodeRoles(vec![ v1alpha1::NodeRole::ClusterManager, v1alpha1::NodeRole::Data, @@ -268,7 +278,6 @@ mod tests { ]), resources: Resources::default(), termination_grace_period_seconds: 30, - listener_class: "cluster-internal".to_string(), }, config_overrides: HashMap::default(), env_overrides: EnvVarSet::default(), diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index 313371a..4b42052 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -1,9 +1,9 @@ //! Builder for role-group resources -use std::str::FromStr; +use std::{collections::BTreeMap, str::FromStr}; use stackable_operator::{ - builder::{meta::ObjectMetaBuilder, pod::container::ContainerBuilder}, + builder::meta::ObjectMetaBuilder, crd::listener::{self}, k8s_openapi::{ DeepMerge, @@ -11,29 +11,49 @@ use stackable_operator::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ Affinity, ConfigMap, ConfigMapVolumeSource, Container, ContainerPort, - PersistentVolumeClaim, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, - Service, ServicePort, ServiceSpec, TCPSocketAction, Volume, VolumeMount, + EmptyDirVolumeSource, PersistentVolumeClaim, PodSecurityContext, PodSpec, + PodTemplateSpec, Probe, Service, ServicePort, ServiceSpec, TCPSocketAction, Volume, + VolumeMount, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, }, - kvp::{Annotations, Label, Labels}, + kvp::{Annotation, Annotations, Label, Labels}, + product_logging::framework::{ + VECTOR_CONFIG_FILE, calculate_log_volume_size_limit, create_vector_shutdown_file_command, + remove_vector_shutdown_file_command, + }, + utils::COMMON_BASH_TRAP_FUNCTIONS, }; -use super::node_config::{CONFIGURATION_FILE_OPENSEARCH_YML, NodeConfig}; +use super::{ + node_config::{CONFIGURATION_FILE_OPENSEARCH_YML, NodeConfig}, + product_logging::config::{ + CONFIGURATION_FILE_LOG4J2_PROPERTIES, create_log4j2_config, vector_config_file_content, + }, +}; use crate::{ - controller::{ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster}, + constant, + controller::{ + ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster, + build::product_logging::config::{ + MAX_OPENSEARCH_SERVER_LOG_FILES_SIZE, vector_config_file_extra_env_vars, + }, + }, crd::v1alpha1, framework::{ PersistentVolumeClaimName, RoleGroupName, ServiceAccountName, ServiceName, VolumeName, builder::{ meta::ownerreference_from_resource, pod::{ - container::EnvVarName, + container::{EnvVarName, new_container_builder}, volume::{ListenerReference, listener_operator_volume_source_builder_build_pvc}, }, }, kvp::label::{recommended_labels, role_group_selector, role_selector}, + product_logging::framework::{ + STACKABLE_LOG_DIR, ValidatedContainerLogConfigChoice, vector_container, + }, role_group_utils::ResourceNames, }, }; @@ -43,26 +63,18 @@ pub const HTTP_PORT: u16 = 9200; pub const TRANSPORT_PORT_NAME: &str = "transport"; pub const TRANSPORT_PORT: u16 = 9300; -const CONFIG_VOLUME_NAME: &str = "config"; -const DATA_VOLUME_NAME: &str = "data"; - -const LISTENER_VOLUME_NAME: &str = "listener"; -const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; +constant!(CONFIG_VOLUME_NAME: VolumeName = "config"); -const DEFAULT_OPENSEARCH_HOME: &str = "/stackable/opensearch"; +constant!(LOG_CONFIG_VOLUME_NAME: VolumeName = "log-config"); +constant!(DATA_VOLUME_NAME: VolumeName = "data"); -fn config_volume_name() -> VolumeName { - VolumeName::from_str(CONFIG_VOLUME_NAME).expect("should be a valid Volume name") -} +constant!(LISTENER_VOLUME_NAME: PersistentVolumeClaimName = "listener"); +const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; -fn data_volume_name() -> VolumeName { - VolumeName::from_str(DATA_VOLUME_NAME).expect("should be a valid Volume name") -} +constant!(LOG_VOLUME_NAME: VolumeName = "log"); +const LOG_VOLUME_DIR: &str = "/stackable/log"; -fn listener_volume_name() -> PersistentVolumeClaimName { - PersistentVolumeClaimName::from_str(LISTENER_VOLUME_NAME) - .expect("should be a valid PersistentVolumeClaim name") -} +const DEFAULT_OPENSEARCH_HOME: &str = "/stackable/opensearch"; /// Builder for role-group resources pub struct RoleGroupBuilder<'a> { @@ -110,11 +122,30 @@ impl<'a> RoleGroupBuilder<'a> { .common_metadata(self.resource_names.role_group_config_map()) .build(); - let data = [( + let mut data = BTreeMap::new(); + + data.insert( CONFIGURATION_FILE_OPENSEARCH_YML.to_owned(), - self.node_config.static_opensearch_config_file(), - )] - .into(); + self.node_config.static_opensearch_config_file_content(), + ); + + if let ValidatedContainerLogConfigChoice::Automatic(log_config) = + &self.role_group_config.config.logging.opensearch_container + { + data.insert( + CONFIGURATION_FILE_LOG4J2_PROPERTIES.to_owned(), + create_log4j2_config(log_config), + ); + }; + + if self + .role_group_config + .config + .logging + .is_vector_agent_enabled() + { + data.insert(VECTOR_CONFIG_FILE.to_owned(), vector_config_file_content()); + } ConfigMap { metadata, @@ -137,7 +168,7 @@ impl<'a> RoleGroupBuilder<'a> { .resources .storage .data - .build_pvc(data_volume_name().as_ref(), Some(vec!["ReadWriteOnce"])); + .build_pvc(DATA_VOLUME_NAME.as_ref(), Some(vec!["ReadWriteOnce"])); let listener_group_name = self.resource_names.listener_name(); @@ -149,7 +180,7 @@ impl<'a> RoleGroupBuilder<'a> { let listener_volume_claim_template = listener_operator_volume_source_builder_build_pvc( &ListenerReference::Listener(listener_group_name), &self.recommended_labels(), - &listener_volume_name(), + &LISTENER_VOLUME_NAME, ); let pvcs: Option> = Some(vec![ @@ -188,9 +219,71 @@ impl<'a> RoleGroupBuilder<'a> { let metadata = ObjectMetaBuilder::new() .with_labels(self.recommended_labels()) .with_labels(node_role_labels) + .with_annotation( + Annotation::try_from(( + "kubectl.kubernetes.io/default-container".to_owned(), + v1alpha1::Container::OpenSearch.to_container_name(), + )) + .expect("should be a valid annotation"), + ) .build(); - let container = self.build_container(&self.role_group_config); + let opensearch_container = self.build_opensearch_container(); + let vector_container = self + .role_group_config + .config + .logging + .vector_container + .as_ref() + .map(|vector_container_log_config| { + vector_container( + &v1alpha1::Container::Vector.to_container_name(), + &self.cluster.image, + vector_container_log_config, + &self.resource_names, + &CONFIG_VOLUME_NAME, + &LOG_VOLUME_NAME, + vector_config_file_extra_env_vars(), + ) + }); + + let log_config_volume_config_map = + if let ValidatedContainerLogConfigChoice::Custom(config_map_name) = + &self.role_group_config.config.logging.opensearch_container + { + config_map_name.clone() + } else { + self.resource_names.role_group_config_map() + }; + + let volumes = vec![ + Volume { + name: CONFIG_VOLUME_NAME.to_string(), + config_map: Some(ConfigMapVolumeSource { + name: self.resource_names.role_group_config_map().to_string(), + ..Default::default() + }), + ..Volume::default() + }, + Volume { + name: LOG_CONFIG_VOLUME_NAME.to_string(), + config_map: Some(ConfigMapVolumeSource { + name: log_config_volume_config_map.to_string(), + ..Default::default() + }), + ..Volume::default() + }, + Volume { + name: LOG_VOLUME_NAME.to_string(), + empty_dir: Some(EmptyDirVolumeSource { + size_limit: Some(calculate_log_volume_size_limit(&[ + MAX_OPENSEARCH_SERVER_LOG_FILES_SIZE, + ])), + ..EmptyDirVolumeSource::default() + }), + ..Volume::default() + }, + ]; // The PodBuilder is not used because it re-validates the values which are already // validated. For instance, it would be necessary to convert the @@ -209,7 +302,10 @@ impl<'a> RoleGroupBuilder<'a> { .pod_anti_affinity .clone(), }), - containers: vec![container], + containers: [Some(opensearch_container), vector_container] + .into_iter() + .flatten() + .collect(), node_selector: self .role_group_config .config @@ -227,14 +323,7 @@ impl<'a> RoleGroupBuilder<'a> { .config .termination_grace_period_seconds, ), - volumes: Some(vec![Volume { - name: config_volume_name().to_string(), - config_map: Some(ConfigMapVolumeSource { - name: self.resource_names.role_group_config_map().to_string(), - ..Default::default() - }), - ..Volume::default() - }]), + volumes: Some(volumes), ..PodSpec::default() }), }; @@ -277,7 +366,7 @@ impl<'a> RoleGroupBuilder<'a> { } /// Builds the container for the [`PodTemplateSpec`] - fn build_container(&self, role_group_config: &OpenSearchRoleGroupConfig) -> Container { + fn build_opensearch_container(&self) -> Container { // Probe values taken from the official Helm chart let startup_probe = Probe { failure_threshold: Some(30), @@ -305,45 +394,79 @@ impl<'a> RoleGroupBuilder<'a> { // Use `OPENSEARCH_HOME` from envOverrides or default to `DEFAULT_OPENSEARCH_HOME`. let opensearch_home = env_vars - .get(EnvVarName::from_str_unsafe("OPENSEARCH_HOME")) + .get(&EnvVarName::from_str_unsafe("OPENSEARCH_HOME")) .and_then(|env_var| env_var.value.clone()) .unwrap_or(DEFAULT_OPENSEARCH_HOME.to_owned()); // Use `OPENSEARCH_PATH_CONF` from envOverrides or default to `OPENSEARCH_HOME/config`, // i.e. depend on `OPENSEARCH_HOME`. let opensearch_path_conf = env_vars - .get(EnvVarName::from_str_unsafe("OPENSEARCH_PATH_CONF")) + .get(&EnvVarName::from_str_unsafe("OPENSEARCH_PATH_CONF")) .and_then(|env_var| env_var.value.clone()) .unwrap_or(format!("{opensearch_home}/config")); - ContainerBuilder::new("opensearch") - .expect("should be a valid container name") + let volume_mounts = [ + VolumeMount { + mount_path: format!("{opensearch_path_conf}/{CONFIGURATION_FILE_OPENSEARCH_YML}"), + name: CONFIG_VOLUME_NAME.to_string(), + read_only: Some(true), + sub_path: Some(CONFIGURATION_FILE_OPENSEARCH_YML.to_owned()), + ..VolumeMount::default() + }, + VolumeMount { + mount_path: format!( + "{opensearch_path_conf}/{CONFIGURATION_FILE_LOG4J2_PROPERTIES}" + ), + name: LOG_CONFIG_VOLUME_NAME.to_string(), + read_only: Some(true), + sub_path: Some(CONFIGURATION_FILE_LOG4J2_PROPERTIES.to_owned()), + ..VolumeMount::default() + }, + VolumeMount { + mount_path: format!("{opensearch_home}/data"), + name: DATA_VOLUME_NAME.to_string(), + ..VolumeMount::default() + }, + VolumeMount { + mount_path: LISTENER_VOLUME_DIR.to_owned(), + name: LISTENER_VOLUME_NAME.to_string(), + ..VolumeMount::default() + }, + VolumeMount { + mount_path: LOG_VOLUME_DIR.to_owned(), + name: LOG_VOLUME_NAME.to_string(), + ..VolumeMount::default() + }, + ]; + + new_container_builder(&v1alpha1::Container::OpenSearch.to_container_name()) .image_from_product_image(&self.cluster.image) - .command(vec![format!( - "{opensearch_home}/opensearch-docker-entrypoint.sh" + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![format!( + "{COMMON_BASH_TRAP_FUNCTIONS}\n\ + {remove_vector_shutdown_file_command}\n\ + prepare_signal_handlers\n\ + if command --search containerdebug >/dev/null 2>&1; then\n\ + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &\n\ + else\n\ + echo >&2 \"containerdebug not installed; Proceed without it.\"\n\ + fi\n\ + ./opensearch-docker-entrypoint.sh {extra_args} &\n\ + wait_for_termination $!\n\ + {create_vector_shutdown_file_command}", + extra_args = self.role_group_config.cli_overrides_to_vec().join(" "), + remove_vector_shutdown_file_command = + remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), + create_vector_shutdown_file_command = + create_vector_shutdown_file_command(STACKABLE_LOG_DIR), )]) - .args(role_group_config.cli_overrides_to_vec()) .add_env_vars(env_vars.into()) - .add_volume_mounts([ - VolumeMount { - mount_path: format!( - "{opensearch_path_conf}/{CONFIGURATION_FILE_OPENSEARCH_YML}" - ), - name: config_volume_name().to_string(), - read_only: Some(true), - sub_path: Some(CONFIGURATION_FILE_OPENSEARCH_YML.to_owned()), - ..VolumeMount::default() - }, - VolumeMount { - mount_path: format!("{opensearch_home}/data"), - name: data_volume_name().to_string(), - ..VolumeMount::default() - }, - VolumeMount { - mount_path: LISTENER_VOLUME_DIR.to_owned(), - name: LISTENER_VOLUME_NAME.to_owned(), - ..VolumeMount::default() - }, - ]) + .add_volume_mounts(volume_mounts) .expect("The mount paths are statically defined and there should be no duplicates.") .add_container_ports(vec![ ContainerPort { @@ -453,7 +576,7 @@ impl<'a> RoleGroupBuilder<'a> { listener::v1alpha1::Listener { metadata, spec: listener::v1alpha1::ListenerSpec { - class_name: Some(listener_class), + class_name: Some(listener_class.to_string()), ports: Some(ports.to_vec()), ..listener::v1alpha1::ListenerSpec::default() }, @@ -511,6 +634,7 @@ mod tests { str::FromStr, }; + use pretty_assertions::assert_eq; use serde_json::json; use stackable_operator::{ commons::{ @@ -519,30 +643,39 @@ mod tests { }, k8s_openapi::api::core::v1::PodTemplateSpec, kvp::LabelValue, + product_logging::spec::AutomaticContainerLogConfig, role_utils::GenericRoleConfig, }; use strum::IntoEnumIterator; use uuid::uuid; - use super::{RoleGroupBuilder, config_volume_name, data_volume_name, listener_volume_name}; + use super::{ + CONFIG_VOLUME_NAME, DATA_VOLUME_NAME, LISTENER_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, + LOG_VOLUME_NAME, RoleGroupBuilder, + }; use crate::{ controller::{ - ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster, ValidatedOpenSearchConfig, + ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster, + ValidatedContainerLogConfigChoice, ValidatedLogging, ValidatedOpenSearchConfig, }, crd::{NodeRoles, v1alpha1}, framework::{ - ClusterName, ControllerName, NamespaceName, OperatorName, ProductName, ProductVersion, - RoleGroupName, ServiceAccountName, ServiceName, builder::pod::container::EnvVarSet, + ClusterName, ConfigMapName, ControllerName, ListenerClassName, NamespaceName, + OperatorName, ProductName, ProductVersion, RoleGroupName, ServiceAccountName, + ServiceName, builder::pod::container::EnvVarSet, + product_logging::framework::VectorContainerLogConfig, role_utils::GenericProductSpecificCommonConfig, }, }; #[test] - fn test_volume_names() { + fn test_constants() { // Test that the functions do not panic - config_volume_name(); - data_volume_name(); - listener_volume_name(); + let _ = CONFIG_VOLUME_NAME; + let _ = LOG_CONFIG_VOLUME_NAME; + let _ = DATA_VOLUME_NAME; + let _ = LISTENER_VOLUME_NAME; + let _ = LOG_VOLUME_NAME; } fn context_names() -> ContextNames { @@ -567,6 +700,20 @@ mod tests { replicas: 1, config: ValidatedOpenSearchConfig { affinity: StackableAffinity::default(), + listener_class: ListenerClassName::from_str_unsafe("cluster-internal"), + logging: ValidatedLogging { + opensearch_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + vector_container: Some(VectorContainerLogConfig { + log_config: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + vector_aggregator_config_map_name: ConfigMapName::from_str_unsafe( + "vector-aggregator", + ), + }), + }, node_roles: NodeRoles(vec![ v1alpha1::NodeRole::ClusterManager, v1alpha1::NodeRole::Data, @@ -575,7 +722,6 @@ mod tests { ]), resources: Resources::default(), termination_grace_period_seconds: 30, - listener_class: "cluster-internal".to_string(), }, config_overrides: HashMap::default(), env_overrides: EnvVarSet::default(), @@ -625,9 +771,18 @@ mod tests { let context_names = context_names(); let role_group_builder = role_group_builder(&context_names); - let config_map = serde_json::to_value(role_group_builder.build_config_map()) + let mut config_map = serde_json::to_value(role_group_builder.build_config_map()) .expect("should be serializable"); + // The content of log4j2.properties is already tested in the + // `controller::build::product_logging::config` module. + config_map["data"]["log4j2.properties"].take(); + // The content of opensearch.yml is already tested in the `controller::build::node_config` + // module. + config_map["data"]["opensearch.yml"].take(); + // vector.yaml is a static file and does not have to be repeated here. + config_map["data"]["vector.yaml"].take(); + assert_eq!( json!({ "apiVersion": "v1", @@ -655,12 +810,9 @@ mod tests { ] }, "data": { - "opensearch.yml": concat!( - "cluster.name: \"my-opensearch-cluster\"\n", - "discovery.type: \"single-node\"\n", - "network.host: \"0.0.0.0\"\n", - "plugins.security.nodes_dn: [\"CN=generated certificate for pod\"]" - ) + "log4j2.properties": null, + "opensearch.yml": null, + "vector.yaml": null } }), config_map @@ -715,6 +867,9 @@ mod tests { "serviceName": "my-opensearch-cluster-nodes-default-headless", "template": { "metadata": { + "annotations": { + "kubectl.kubernetes.io/default-container": "opensearch", + }, "labels": { "app.kubernetes.io/component": "nodes", "app.kubernetes.io/instance": "my-opensearch-cluster", @@ -733,9 +888,56 @@ mod tests { "affinity": {}, "containers": [ { - "args": [], + "args": [ + concat!( + "\n", + "prepare_signal_handlers()\n", + "{\n", + " unset term_child_pid\n", + " unset term_kill_needed\n", + " trap 'handle_term_signal' TERM\n", + "}\n", + "\n", + "handle_term_signal()\n", + "{\n", + " if [ \"${term_child_pid}\" ]; then\n", + " kill -TERM \"${term_child_pid}\" 2>/dev/null\n", + " else\n", + " term_kill_needed=\"yes\"\n", + " fi\n", + "}\n", + "\n", + "wait_for_termination()\n", + "{\n", + " set +e\n", + " term_child_pid=$1\n", + " if [[ -v term_kill_needed ]]; then\n", + " kill -TERM \"${term_child_pid}\" 2>/dev/null\n", + " fi\n", + " wait ${term_child_pid} 2>/dev/null\n", + " trap - TERM\n", + " wait ${term_child_pid} 2>/dev/null\n", + " set -e\n", + "}\n", + "\n", + "rm -f /stackable/log/_vector/shutdown\n", + "prepare_signal_handlers\n", + "if command --search containerdebug >/dev/null 2>&1; then\n", + "containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n", + "else\n", + "echo >&2 \"containerdebug not installed; Proceed without it.\"\n", + "fi\n", + "./opensearch-docker-entrypoint.sh &\n", + "wait_for_termination $!\n", + "mkdir -p /stackable/log/_vector && touch /stackable/log/_vector/shutdown" + ) + ], "command": [ - "/stackable/opensearch/opensearch-docker-entrypoint.sh" + "/bin/bash", + "-x", + "-euo", + "pipefail", + "-c" ], "env": [ { @@ -797,6 +999,12 @@ mod tests { "readOnly": true, "subPath": "opensearch.yml" }, + { + "mountPath": "/stackable/opensearch/config/log4j2.properties", + "name": "log-config", + "readOnly": true, + "subPath": "log4j2.properties" + }, { "mountPath": "/stackable/opensearch/data", "name": "data" @@ -804,9 +1012,110 @@ mod tests { { "mountPath": "/stackable/listener", "name": "listener" + }, + { + "mountPath": "/stackable/log", + "name": "log" } ] - } + }, + { + "args": [ + concat!( + "# Vector will ignore SIGTERM (as PID != 1) and must be shut down by writing a shutdown trigger file\n", + "vector & vector_pid=$!\n", + "if [ ! -f \"/stackable/log/_vector/shutdown\" ]; then\n", + "mkdir -p /stackable/log/_vector\n", + "inotifywait -qq --event create /stackable/log/_vector;\n", + "fi\n", + "sleep 1\n", + "kill $vector_pid" + ), + ], + "command": [ + "/bin/bash", + "-x", + "-euo", + "pipefail", + "-c" + ], + "env": [ + { + "name": "CLUSTER_NAME", + "value":"my-opensearch-cluster", + }, + { + "name": "LOG_DIR", + "value": "/stackable/log", + }, + { + "name": "NAMESPACE", + "valueFrom": { + "fieldRef": { + "fieldPath": "metadata.namespace", + }, + }, + }, + { + "name": "OPENSEARCH_SERVER_LOG_FILE", + "value": "opensearch_server.json", + }, + { + "name": "ROLE_GROUP_NAME", + "value": "default", + }, + { + "name": "ROLE_NAME", + "value": "nodes", + }, + { + "name": "VECTOR_AGGREGATOR_ADDRESS", + "valueFrom": { + "configMapKeyRef": { + "key": "ADDRESS", + "name": "vector-aggregator", + }, + }, + }, + { + "name": "VECTOR_CONFIG_YAML", + "value": "/stackable/config/vector.yaml", + }, + { + "name": "VECTOR_FILE_LOG_LEVEL", + "value": "info", + }, + { + "name": "VECTOR_LOG", + "value": "info", + }, + ], + "image": "oci.stackable.tech/sdp/opensearch:3.1.0-stackable0.0.0-dev", + "imagePullPolicy": "Always", + "name": "vector", + "resources": { + "limits": { + "cpu": "500m", + "memory": "128Mi", + }, + "requests": { + "cpu": "250m", + "memory": "128Mi", + }, + }, + "volumeMounts": [ + { + "mountPath": "/stackable/config/vector.yaml", + "name": "config", + "readOnly": true, + "subPath": "vector.yaml", + }, + { + "mountPath": "/stackable/log", + "name": "log", + }, + ], + }, ], "securityContext": { "fsGroup": 1000 @@ -819,7 +1128,19 @@ mod tests { "name": "my-opensearch-cluster-nodes-default" }, "name": "config" - } + }, + { + "configMap": { + "name": "my-opensearch-cluster-nodes-default" + }, + "name": "log-config" + }, + { + "emptyDir": { + "sizeLimit": "30Mi" + }, + "name": "log" + } ] } }, diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index bbf763a..e3dd839 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -5,6 +5,7 @@ use std::{collections::BTreeMap, num::TryFromIntError, str::FromStr}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ kube::{Resource, ResourceExt}, + product_logging::spec::Logging, role_utils::RoleGroup, shared::time::Duration, }; @@ -12,13 +13,16 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use super::{ ContextNames, OpenSearchRoleGroupConfig, ProductVersion, RoleGroupName, ValidatedCluster, - ValidatedOpenSearchConfig, + ValidatedLogging, ValidatedOpenSearchConfig, }; use crate::{ - crd::v1alpha1::{self, OpenSearchConfig, OpenSearchConfigFragment}, + crd::v1alpha1::{self}, framework::{ - ClusterName, NamespaceName, Uid, + ClusterName, ConfigMapName, NamespaceName, Uid, builder::pod::container::{EnvVarName, EnvVarSet}, + product_logging::framework::{ + VectorContainerLogConfig, validate_logging_configuration_for_container, + }, role_utils::{GenericProductSpecificCommonConfig, RoleGroupConfig, with_validated_config}, }, }; @@ -35,6 +39,11 @@ pub enum Error { #[snafu(display("failed to get the cluster UID"))] GetClusterUid {}, + #[snafu(display( + "failed to get vectorAggregatorConfigMapName; It must be set if enableVectorAgent is true." + ))] + GetVectorAggregatorConfigMapName {}, + #[snafu(display("failed to set cluster name"))] ParseClusterName { source: crate::framework::Error }, @@ -60,6 +69,11 @@ pub enum Error { source: stackable_operator::commons::product_image_selection::Error, }, + #[snafu(display("failed to validate the logging configuration"))] + ValidateLoggingConfig { + source: crate::framework::product_logging::framework::Error, + }, + #[snafu(display("fragment validation failure"))] ValidateOpenSearchConfig { source: stackable_operator::config::fragment::ValidationError, @@ -133,9 +147,12 @@ fn validate_role_group_config( context_names: &ContextNames, cluster_name: &ClusterName, cluster: &v1alpha1::OpenSearchCluster, - role_group_config: &RoleGroup, + role_group_config: &RoleGroup< + v1alpha1::OpenSearchConfigFragment, + GenericProductSpecificCommonConfig, + >, ) -> Result { - let merged_role_group: RoleGroup = with_validated_config( + let merged_role_group: RoleGroup = with_validated_config( role_group_config, &cluster.spec.nodes, &v1alpha1::OpenSearchConfig::default_config( @@ -146,6 +163,14 @@ fn validate_role_group_config( ) .context(ValidateOpenSearchConfigSnafu)?; + let logging = validate_logging_configuration( + &merged_role_group.config.config.logging, + &cluster + .spec + .cluster_config + .vector_aggregator_config_map_name, + )?; + let graceful_shutdown_timeout = merged_role_group.config.config.graceful_shutdown_timeout; let termination_grace_period_seconds = graceful_shutdown_timeout.as_secs().try_into().context( TerminationGracePeriodTooLongSnafu { @@ -155,17 +180,18 @@ fn validate_role_group_config( let validated_config = ValidatedOpenSearchConfig { affinity: merged_role_group.config.config.affinity, + listener_class: merged_role_group.config.config.listener_class, + logging, node_roles: merged_role_group.config.config.node_roles, resources: merged_role_group.config.config.resources, termination_grace_period_seconds, - listener_class: merged_role_group.config.config.listener_class, }; let mut env_overrides = EnvVarSet::new(); for (env_var_name, env_var_value) in merged_role_group.config.env_overrides { env_overrides = env_overrides.with_value( - EnvVarName::from_str(&env_var_name).context(ParseEnvironmentVariableSnafu)?, + &EnvVarName::from_str(&env_var_name).context(ParseEnvironmentVariableSnafu)?, env_var_value, ); } @@ -182,10 +208,41 @@ fn validate_role_group_config( }) } +fn validate_logging_configuration( + logging: &Logging, + vector_aggregator_config_map_name: &Option, +) -> Result { + let opensearch_container = + validate_logging_configuration_for_container(logging, v1alpha1::Container::OpenSearch) + .context(ValidateLoggingConfigSnafu)?; + + let vector_container = if logging.enable_vector_agent { + let vector_aggregator_config_map_name = vector_aggregator_config_map_name + .clone() + .context(GetVectorAggregatorConfigMapNameSnafu)?; + Some(VectorContainerLogConfig { + log_config: validate_logging_configuration_for_container( + logging, + v1alpha1::Container::Vector, + ) + .context(ValidateLoggingConfigSnafu)?, + vector_aggregator_config_map_name, + }) + } else { + None + }; + + Ok(ValidatedLogging { + opensearch_container, + vector_container, + }) +} + #[cfg(test)] mod tests { - use std::str::FromStr; + use std::{collections::BTreeMap, str::FromStr}; + use pretty_assertions::assert_eq; use stackable_operator::{ commons::{ affinity::StackableAffinity, @@ -201,6 +258,11 @@ mod tests { }, kube::api::ObjectMeta, kvp::LabelValue, + product_logging::spec::{ + AppenderConfig, AutomaticContainerLogConfig, ConfigMapLogConfigFragment, + ContainerLogConfigChoiceFragment, ContainerLogConfigFragment, + CustomContainerLogConfigFragment, LogLevel, LoggerConfig, LoggingFragment, + }, role_utils::{CommonConfiguration, GenericRoleConfig, Role, RoleGroup}, shared::time::Duration, }; @@ -208,15 +270,18 @@ mod tests { use super::{ErrorDiscriminants, validate}; use crate::{ - controller::{ContextNames, ValidatedCluster, ValidatedOpenSearchConfig}, + controller::{ContextNames, ValidatedCluster, ValidatedLogging, ValidatedOpenSearchConfig}, crd::{ NodeRoles, - v1alpha1::{self, OpenSearchClusterSpec, OpenSearchConfigFragment, StorageConfig}, + v1alpha1::{self}, }, framework::{ - ClusterName, ControllerName, NamespaceName, OperatorName, ProductName, ProductVersion, - RoleGroupName, + ClusterName, ConfigMapName, ControllerName, ListenerClassName, NamespaceName, + OperatorName, ProductName, ProductVersion, RoleGroupName, builder::pod::container::{EnvVarName, EnvVarSet}, + product_logging::framework::{ + ValidatedContainerLogConfigChoice, VectorContainerLogConfig, + }, role_utils::{GenericProductSpecificCommonConfig, RoleGroupConfig}, }, }; @@ -283,6 +348,49 @@ mod tests { }), ..StackableAffinity::default() }, + listener_class: ListenerClassName::from_str_unsafe( + "listener-class-from-role-group-level" + ), + logging: ValidatedLogging { + opensearch_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig { + loggers: [( + "ROOT".to_owned(), + LoggerConfig { + level: LogLevel::INFO + } + )] + .into(), + console: Some(AppenderConfig { + level: Some(LogLevel::INFO) + }), + file: Some(AppenderConfig { + level: Some(LogLevel::INFO) + }), + }, + ), + vector_container: Some(VectorContainerLogConfig { + log_config: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig { + loggers: [( + "ROOT".to_owned(), + LoggerConfig { + level: LogLevel::INFO + }, + )] + .into(), + console: Some(AppenderConfig { + level: Some(LogLevel::INFO) + }), + file: Some(AppenderConfig { + level: Some(LogLevel::INFO) + }), + }, + ), + vector_aggregator_config_map_name: + ConfigMapName::from_str_unsafe("vector-aggregator"), + }), + }, node_roles: NodeRoles( [ v1alpha1::NodeRole::ClusterManager, @@ -301,7 +409,7 @@ mod tests { min: Some(Quantity("1".to_owned())), max: Some(Quantity("4".to_owned())) }, - storage: StorageConfig { + storage: v1alpha1::StorageConfig { data: PvcConfig { capacity: Some(Quantity("8Gi".to_owned())), ..PvcConfig::default() @@ -309,7 +417,6 @@ mod tests { } }, termination_grace_period_seconds: 300, - listener_class: "listener-class-from-role-group-level".to_owned(), }, config_overrides: [( "opensearch.yml".to_owned(), @@ -463,6 +570,41 @@ mod tests { ); } + #[test] + fn test_validate_err_validate_logging_config() { + test_validate_err( + |cluster| { + cluster.spec.nodes.config.config.logging.containers = [( + v1alpha1::Container::OpenSearch, + ContainerLogConfigFragment { + choice: Some(ContainerLogConfigChoiceFragment::Custom( + CustomContainerLogConfigFragment { + custom: ConfigMapLogConfigFragment { + config_map: Some("invalid ConfigMap name".to_owned()), + }, + }, + )), + }, + )] + .into() + }, + ErrorDiscriminants::ValidateLoggingConfig, + ); + } + + #[test] + fn test_validate_err_get_vector_aggregator_config_map_name() { + test_validate_err( + |cluster| { + cluster + .spec + .cluster_config + .vector_aggregator_config_map_name = None + }, + ErrorDiscriminants::GetVectorAggregatorConfigMapName, + ); + } + #[test] fn test_validate_err_termination_grace_period_too_long() { test_validate_err( @@ -516,16 +658,27 @@ mod tests { uid: Some("e6ac237d-a6d4-43a1-8135-f36506110912".to_owned()), ..ObjectMeta::default() }, - spec: OpenSearchClusterSpec { + spec: v1alpha1::OpenSearchClusterSpec { image: serde_json::from_str(r#"{"productVersion": "3.1.0"}"#) .expect("should be a valid ProductImage structure"), + cluster_config: v1alpha1::OpenSearchClusterConfig { + vector_aggregator_config_map_name: Some(ConfigMapName::from_str_unsafe( + "vector-aggregator", + )), + }, cluster_operation: ClusterOperation::default(), nodes: Role { config: CommonConfiguration { - config: OpenSearchConfigFragment { + config: v1alpha1::OpenSearchConfigFragment { graceful_shutdown_timeout: Some(Duration::from_minutes_unchecked(5)), - listener_class: Some("listener-class-from-role-level".to_owned()), - ..OpenSearchConfigFragment::default() + listener_class: Some(ListenerClassName::from_str_unsafe( + "listener-class-from-role-level", + )), + logging: LoggingFragment { + enable_vector_agent: Some(true), + containers: BTreeMap::default(), + }, + ..v1alpha1::OpenSearchConfigFragment::default() }, config_overrides: [( "opensearch.yml".to_owned(), @@ -567,11 +720,11 @@ mod tests { "default".to_owned(), RoleGroup { config: CommonConfiguration { - config: OpenSearchConfigFragment { - listener_class: Some( - "listener-class-from-role-group-level".to_owned(), - ), - ..OpenSearchConfigFragment::default() + config: v1alpha1::OpenSearchConfigFragment { + listener_class: Some(ListenerClassName::from_str_unsafe( + "listener-class-from-role-group-level", + )), + ..v1alpha1::OpenSearchConfigFragment::default() }, config_overrides: [( "opensearch.yml".to_owned(), diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 199c51d..4803e70 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -17,6 +17,7 @@ use stackable_operator::{ }, k8s_openapi::{api::core::v1::PodAntiAffinity, apimachinery::pkg::api::resource::Quantity}, kube::CustomResource, + product_logging::{self, spec::Logging}, role_utils::{GenericRoleConfig, Role}, schemars::{self, JsonSchema}, shared::time::Duration, @@ -25,12 +26,15 @@ use stackable_operator::{ }; use strum::{Display, EnumIter}; -use crate::framework::{ - ClusterName, NameIsValidLabelValue, ProductName, RoleName, - role_utils::GenericProductSpecificCommonConfig, +use crate::{ + constant, + framework::{ + ClusterName, ConfigMapName, ContainerName, ListenerClassName, NameIsValidLabelValue, + ProductName, RoleName, role_utils::GenericProductSpecificCommonConfig, + }, }; -const DEFAULT_LISTENER_CLASS: &str = "cluster-internal"; +constant!(DEFAULT_LISTENER_CLASS: ListenerClassName = "cluster-internal"); #[versioned( version(name = "v1alpha1"), @@ -43,7 +47,6 @@ const DEFAULT_LISTENER_CLASS: &str = "cluster-internal"; ) )] pub mod versioned { - /// An OpenSearch cluster stacklet. This resource is managed by the Stackable operator for /// OpenSearch. Find more information on how to use it and the resources that the operator /// generates in the [operator documentation](DOCS_BASE_URL_PLACEHOLDER/opensearch/). @@ -58,18 +61,33 @@ pub mod versioned { ))] #[serde(rename_all = "camelCase")] pub struct OpenSearchClusterSpec { - // no doc - docs in ProductImage struct. + // no doc - docs in ProductImage struct pub image: ProductImage, - // no doc - docs in ClusterOperation struct. + /// Configuration that applies to all roles and role groups + #[serde(default)] + pub cluster_config: v1alpha1::OpenSearchClusterConfig, + + // no doc - docs in ClusterOperation struct #[serde(default)] pub cluster_operation: ClusterOperation, - // no doc - docs in Role struct. + // no doc - docs in Role struct pub nodes: Role, } + #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct OpenSearchClusterConfig { + /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery). + /// It must contain the key `ADDRESS` with the address of the Vector aggregator. + /// Follow the [logging tutorial](DOCS_BASE_URL_PLACEHOLDER/tutorials/logging-vector-aggregator) + /// to learn how to configure log aggregation with Vector. + #[serde(skip_serializing_if = "Option::is_none")] + pub vector_aggregator_config_map_name: Option, + } + // The possible node roles are by default the built-in roles and the search role, see // https://github.com/opensearch-project/OpenSearch/blob/3.0.0/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java#L609-L614. // @@ -134,6 +152,16 @@ pub mod versioned { #[fragment_attrs(serde(default))] pub graceful_shutdown_timeout: Duration, + /// This field controls which + /// [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) + /// is used to expose the HTTP communication. + #[fragment_attrs(serde(default))] + pub listener_class: ListenerClassName, + + // no doc - docs in Logging struct + #[fragment_attrs(serde(default))] + pub logging: Logging, + /// Roles of the OpenSearch node. /// /// Consult the [node roles @@ -142,10 +170,27 @@ pub mod versioned { #[fragment_attrs(serde(default))] pub resources: Resources, + } - /// This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. - #[fragment_attrs(serde(default))] - pub listener_class: String, + #[derive( + Clone, + Debug, + Deserialize, + Display, + Eq, + EnumIter, + JsonSchema, + Ord, + PartialEq, + PartialOrd, + Serialize, + )] + pub enum Container { + #[serde(rename = "opensearch")] + OpenSearch, + + #[serde(rename = "vector")] + Vector, } #[derive(Clone, Debug, Default, JsonSchema, PartialEq, Fragment)] @@ -215,6 +260,8 @@ impl v1alpha1::OpenSearchConfig { graceful_shutdown_timeout: Some( Duration::from_str("2m").expect("should be a valid duration"), ), + listener_class: Some(DEFAULT_LISTENER_CLASS.to_owned()), + logging: product_logging::spec::default_logging(), // Defaults taken from the Helm chart, see // https://github.com/opensearch-project/helm-charts/blob/opensearch-3.0.0/charts/opensearch/values.yaml#L16-L20 node_roles: Some(NodeRoles(vec![ @@ -248,7 +295,6 @@ impl v1alpha1::OpenSearchConfig { }, }, }, - listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), } } } @@ -268,8 +314,24 @@ impl NodeRoles { impl Atomic for NodeRoles {} +impl v1alpha1::Container { + /// Returns the validated container name + /// + /// This name should match the one defined by the user (see the serde annotation at + /// [`v1alpha1::Container`], but it could differ if it was renamed. + pub fn to_container_name(&self) -> ContainerName { + ContainerName::from_str(match self { + v1alpha1::Container::OpenSearch => "opensearch", + v1alpha1::Container::Vector => "vector", + }) + .expect("should be a valid container name") + } +} + #[cfg(test)] mod tests { + use strum::IntoEnumIterator; + use crate::crd::v1alpha1; #[test] @@ -292,4 +354,12 @@ mod tests { serde_json::from_str("\"cluster_manager\"").expect("should be deserializable") ); } + + #[test] + fn test_to_container_name() { + for container in v1alpha1::Container::iter() { + // Test that the function does not panic + container.to_container_name(); + } + } } diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs index 46c1d5a..3442b2e 100644 --- a/rust/operator-binary/src/framework.rs +++ b/rust/operator-binary/src/framework.rs @@ -20,6 +20,8 @@ //! become less frequent, then this module can be incorporated into stackable-operator. The module //! structure should already resemble the one of stackable-operator. +use std::str::FromStr; + use snafu::Snafu; use stackable_operator::validation::{ RFC_1035_LABEL_MAX_LENGTH, RFC_1123_LABEL_MAX_LENGTH, RFC_1123_SUBDOMAIN_MAX_LENGTH, @@ -29,10 +31,12 @@ use strum::{EnumDiscriminants, IntoStaticStr}; pub mod builder; pub mod cluster_resources; pub mod kvp; +pub mod product_logging; pub mod role_group_utils; pub mod role_utils; +pub mod validation; -#[derive(Snafu, Debug, EnumDiscriminants)] +#[derive(Debug, EnumDiscriminants, Snafu)] #[strum_discriminants(derive(IntoStaticStr))] pub enum Error { #[snafu(display("empty strings are not allowed"))] @@ -41,6 +45,11 @@ pub enum Error { #[snafu(display("maximum length exceeded"))] LengthExceeded { length: usize, max_length: usize }, + #[snafu(display("not a valid ConfigMap key"))] + InvalidConfigMapKey { + source: crate::framework::validation::Error, + }, + #[snafu(display("not a valid label value"))] InvalidLabelValue { source: stackable_operator::kvp::LabelValueError, @@ -92,6 +101,18 @@ pub trait NameIsValidLabelValue { /// Restricted string type with attributes like maximum length. /// /// Fully-qualified types are used to ease the import into other modules. +/// +/// # Examples +/// +/// ```rust +/// attributed_string_type! { +/// ConfigMapName, +/// "The name of a ConfigMap", +/// "opensearch-nodes-default", +/// (max_length = RFC_1123_SUBDOMAIN_MAX_LENGTH), +/// is_rfc_1123_dns_subdomain_name +/// } +/// ``` #[macro_export(local_inner_macros)] macro_rules! attributed_string_type { ($name:ident, $description:literal, $example:literal $(, $attribute:tt)*) => { @@ -142,6 +163,27 @@ macro_rules! attributed_string_type { } } + impl<'de> serde::Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let string: String = serde::Deserialize::deserialize(deserializer)?; + $name::from_str(&string).map_err(|err| serde::de::Error::custom(&err)) + } + } + + impl serde::Serialize for $name { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.0.serialize(serializer) + } + } + + impl stackable_operator::config::merge::Atomic for $name {} + #[cfg(test)] impl $name { #[allow(dead_code)] @@ -167,15 +209,18 @@ macro_rules! attributed_string_type { } ); }; + (@from_str $name:ident, $s:expr, is_config_map_key) => { + $crate::framework::validation::is_config_map_key($s).context($crate::framework::InvalidConfigMapKeySnafu)?; + }; + (@from_str $name:ident, $s:expr, is_rfc_1035_label_name) => { + stackable_operator::validation::is_lowercase_rfc_1035_label($s).context($crate::framework::InvalidRfc1035LabelNameSnafu)?; + }; (@from_str $name:ident, $s:expr, is_rfc_1123_dns_subdomain_name) => { stackable_operator::validation::is_lowercase_rfc_1123_subdomain($s).context($crate::framework::InvalidRfc1123DnsSubdomainNameSnafu)?; }; (@from_str $name:ident, $s:expr, is_rfc_1123_label_name) => { stackable_operator::validation::is_lowercase_rfc_1123_label($s).context($crate::framework::InvalidRfc1123LabelNameSnafu)?; }; - (@from_str $name:ident, $s:expr, is_rfc_1035_label_name) => { - stackable_operator::validation::is_lowercase_rfc_1035_label($s).context($crate::framework::InvalidRfc1035LabelNameSnafu)?; - }; (@from_str $name:ident, $s:expr, is_valid_label_value) => { stackable_operator::kvp::LabelValue::from_str($s).context($crate::framework::InvalidLabelValueSnafu)?; }; @@ -187,6 +232,23 @@ macro_rules! attributed_string_type { // type arithmetic would be better pub const MAX_LENGTH: usize = $max_length; } + + // The JsonSchema implementation requires `max_length`. + impl schemars::JsonSchema for $name { + fn schema_name() -> std::borrow::Cow<'static, str> { + std::stringify!($name).into() + } + + fn json_schema(_generator: &mut schemars::generate::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "string", + "minLength": 1, + "maxLength": $name::MAX_LENGTH + }) + } + } + }; + (@trait_impl $name:ident, is_config_map_key) => { }; (@trait_impl $name:ident, is_rfc_1035_label_name) => { impl $name { @@ -232,6 +294,24 @@ macro_rules! attributed_string_type { }; } +/// Use [`std::sync::LazyLock`] to define a static "constant" from a string. +/// +/// The string is converted into the given type with [`std::str::FromStr::from_str`]. +/// +/// # Examples +/// +/// ```rust +/// constant!(DATA_VOLUME_NAME: VolumeName = "data"); +/// constant!(pub CONFIG_VOLUME_NAME: VolumeName = "config"); +/// ``` +#[macro_export(local_inner_macros)] +macro_rules! constant { + ($qualifier:vis $name:ident: $type:ident = $value:literal) => { + $qualifier static $name: std::sync::LazyLock<$type> = + std::sync::LazyLock::new(|| $type::from_str($value).expect("should be a valid $name")); + }; +} + /// Returns the minimum of the given values. /// /// As opposed to [`std::cmp::min`], this function can be used at compile-time. @@ -256,6 +336,21 @@ attributed_string_type! { (max_length = RFC_1123_SUBDOMAIN_MAX_LENGTH), is_rfc_1123_dns_subdomain_name } +attributed_string_type! { + ConfigMapKey, + "The key for a ConfigMap or Secret", + "log4j2.properties", + // see https://github.com/kubernetes/kubernetes/blob/v1.34.1/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L435-L451 + (max_length = RFC_1123_SUBDOMAIN_MAX_LENGTH), + is_config_map_key +} +attributed_string_type! { + ContainerName, + "The name of a container in a Pod", + "opensearch", + (max_length = RFC_1123_LABEL_MAX_LENGTH), + is_rfc_1123_label_name +} attributed_string_type! { ClusterRoleName, "The name of a ClusterRole", @@ -422,19 +517,26 @@ attributed_string_type! { mod tests { use std::str::FromStr; + use schemars::{JsonSchema, SchemaGenerator}; + use serde_json::{Number, Value, json}; use uuid::uuid; use super::{ - ClusterName, ClusterRoleName, ConfigMapName, ControllerName, ErrorDiscriminants, - NamespaceName, OperatorName, PersistentVolumeClaimName, ProductVersion, RoleBindingName, - RoleGroupName, RoleName, ServiceAccountName, ServiceName, StatefulSetName, Uid, VolumeName, + ClusterName, ClusterRoleName, ConfigMapKey, ConfigMapName, ContainerName, ControllerName, + ErrorDiscriminants, ListenerClassName, ListenerName, NamespaceName, OperatorName, + PersistentVolumeClaimName, ProductVersion, RoleBindingName, RoleGroupName, RoleName, + ServiceAccountName, ServiceName, StatefulSetName, Uid, VolumeName, }; use crate::framework::{NameIsValidLabelValue, ProductName}; #[test] fn test_attributed_string_type_examples() { ConfigMapName::test_example(); + ConfigMapKey::test_example(); + ContainerName::test_example(); ClusterRoleName::test_example(); + ListenerName::test_example(); + ListenerClassName::test_example(); NamespaceName::test_example(); PersistentVolumeClaimName::test_example(); RoleBindingName::test_example(); @@ -504,6 +606,123 @@ mod tests { ); } + attributed_string_type! { + JsonSchemaTest, + "JsonSchemaTest test", + "test", + (max_length = 4) + } + + #[test] + fn test_attributed_string_type_json_schema() { + type T = JsonSchemaTest; + + T::test_example(); + assert_eq!("JsonSchemaTest", JsonSchemaTest::schema_name()); + assert_eq!( + json!({ + "type": "string", + "minLength": 1, + "maxLength": 4 + }), + JsonSchemaTest::json_schema(&mut SchemaGenerator::default()) + ); + } + + attributed_string_type! { + SerializeTest, + "serde::Serialize test", + "test" + } + + #[test] + fn test_attributed_string_type_serialize() { + type T = SerializeTest; + + T::test_example(); + assert_eq!( + "\"test\"".to_owned(), + serde_json::to_string(&T::from_str_unsafe("test")).expect("should be serializable") + ); + } + + attributed_string_type! { + DeserializeTest, + "serde::Deserialize test", + "test", + (max_length = 4), + is_rfc_1123_label_name + } + + #[test] + fn test_attributed_string_type_deserialize() { + type T = DeserializeTest; + + T::test_example(); + assert_eq!( + T::from_str_unsafe("test"), + serde_json::from_value(Value::String("test".to_owned())) + .expect("should be deserializable") + ); + assert_eq!( + Err("empty strings are not allowed".to_owned()), + serde_json::from_value::(Value::String("".to_owned())) + .map_err(|err| err.to_string()) + ); + assert_eq!( + Err("maximum length exceeded".to_owned()), + serde_json::from_value::(Value::String("testx".to_owned())) + .map_err(|err| err.to_string()) + ); + assert_eq!( + Err("not a valid label name as defined in RFC 1123".to_owned()), + serde_json::from_value::(Value::String("-".to_owned())) + .map_err(|err| err.to_string()) + ); + assert_eq!( + Err("invalid type: null, expected a string".to_owned()), + serde_json::from_value::(Value::Null).map_err(|err| err.to_string()) + ); + assert_eq!( + Err("invalid type: boolean `true`, expected a string".to_owned()), + serde_json::from_value::(Value::Bool(true)).map_err(|err| err.to_string()) + ); + assert_eq!( + Err("invalid type: integer `1`, expected a string".to_owned()), + serde_json::from_value::(Value::Number( + Number::from_i128(1).expect("should be a valid number") + )) + .map_err(|err| err.to_string()) + ); + assert_eq!( + Err("invalid type: sequence, expected a string".to_owned()), + serde_json::from_value::(Value::Array(vec![])).map_err(|err| err.to_string()) + ); + assert_eq!( + Err("invalid type: map, expected a string".to_owned()), + serde_json::from_value::(Value::Object(serde_json::Map::new())) + .map_err(|err| err.to_string()) + ); + } + + attributed_string_type! { + IsConfigMapKeyTest, + "is_config_map_key test", + "a_B-c.1", + is_config_map_key + } + + #[test] + fn test_attributed_string_type_is_config_map_key() { + type T = IsConfigMapKeyTest; + + T::test_example(); + assert_eq!( + Err(ErrorDiscriminants::InvalidConfigMapKey), + T::from_str(" ").map_err(ErrorDiscriminants::from) + ); + } + attributed_string_type! { IsRfc1035LabelNameTest, "is_rfc_1035_label_name test", diff --git a/rust/operator-binary/src/framework/builder/pod/container.rs b/rust/operator-binary/src/framework/builder/pod/container.rs index 589922f..a7a8ced 100644 --- a/rust/operator-binary/src/framework/builder/pod/container.rs +++ b/rust/operator-binary/src/framework/builder/pod/container.rs @@ -2,11 +2,13 @@ use std::{collections::BTreeMap, fmt::Display, str::FromStr}; use snafu::Snafu; use stackable_operator::{ - builder::pod::container::FieldPathEnvVar, - k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, ObjectFieldSelector}, + builder::pod::container::{ContainerBuilder, FieldPathEnvVar}, + k8s_openapi::api::core::v1::{ConfigMapKeySelector, EnvVar, EnvVarSource, ObjectFieldSelector}, }; use strum::{EnumDiscriminants, IntoStaticStr}; +use crate::framework::{ConfigMapKey, ConfigMapName, ContainerName}; + #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] pub enum Error { @@ -17,6 +19,12 @@ pub enum Error { ParseEnvVarName { env_var_name: String }, } +/// Infallible variant of [`stackable_operator::builder::pod::container::ContainerBuilder::new`] +pub fn new_container_builder(container_name: &ContainerName) -> ContainerBuilder { + ContainerBuilder::new(container_name.as_ref()).expect("should be a valid container name") +} + +// TODO Use attributed_string_type instead /// Validated environment variable name #[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct EnvVarName(String); @@ -65,8 +73,8 @@ impl EnvVarSet { } /// Returns a reference to the [`EnvVar`] with the given name - pub fn get(&self, env_var_name: impl Into) -> Option<&EnvVar> { - self.0.get(&env_var_name.into()) + pub fn get(&self, env_var_name: &EnvVarName) -> Option<&EnvVar> { + self.0.get(env_var_name) } /// Moves all [`EnvVar`]s from the given set into this one. @@ -81,25 +89,22 @@ impl EnvVarSet { /// Adds the given [`EnvVar`]s to this set /// /// [`EnvVar`]s with the same name are overridden. - pub fn with_values(self, env_vars: I) -> Self + pub fn with_values(self, env_vars: I) -> Self where - I: IntoIterator, - K: Into, + I: IntoIterator, V: Into, { env_vars .into_iter() .fold(self, |extended_env_vars, (name, value)| { - extended_env_vars.with_value(name, value) + extended_env_vars.with_value(&name, value) }) } /// Adds an environment variable with the given name and string value to this set /// /// An [`EnvVar`] with the same name is overridden. - pub fn with_value(mut self, name: impl Into, value: impl Into) -> Self { - let name: EnvVarName = name.into(); - + pub fn with_value(mut self, name: &EnvVarName, value: impl Into) -> Self { self.0.insert( name.clone(), EnvVar { @@ -115,13 +120,7 @@ impl EnvVarSet { /// Adds an environment variable with the given name and field path to this set /// /// An [`EnvVar`] with the same name is overridden. - pub fn with_field_path( - mut self, - name: impl Into, - field_path: FieldPathEnvVar, - ) -> Self { - let name: EnvVarName = name.into(); - + pub fn with_field_path(mut self, name: &EnvVarName, field_path: FieldPathEnvVar) -> Self { self.0.insert( name.clone(), EnvVar { @@ -139,6 +138,34 @@ impl EnvVarSet { self } + + /// Adds an environment variable with the given ConfigMap key reference to this set + /// + /// An [`EnvVar`] with the same name is overridden. + pub fn with_config_map_key_ref( + mut self, + name: &EnvVarName, + config_map_name: &ConfigMapName, + config_map_key: &ConfigMapKey, + ) -> Self { + self.0.insert( + name.clone(), + EnvVar { + name: name.to_string(), + value: None, + value_from: Some(EnvVarSource { + config_map_key_ref: Some(ConfigMapKeySelector { + key: config_map_key.to_string(), + name: config_map_name.to_string(), + ..ConfigMapKeySelector::default() + }), + ..EnvVarSource::default() + }), + }, + ); + + self + } } impl From for Vec { @@ -153,10 +180,15 @@ mod tests { use stackable_operator::{ builder::pod::container::FieldPathEnvVar, - k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, ObjectFieldSelector}, + k8s_openapi::api::core::v1::{ + ConfigMapKeySelector, EnvVar, EnvVarSource, ObjectFieldSelector, + }, }; use super::{EnvVarName, EnvVarSet}; + use crate::framework::{ + ConfigMapKey, ConfigMapName, ContainerName, builder::pod::container::new_container_builder, + }; #[test] fn test_envvarname_fromstr() { @@ -172,6 +204,12 @@ mod tests { assert!(EnvVarName::from_str("=").is_err()); } + #[test] + fn test_new_container_builder() { + // Test that the function does not panic + new_container_builder(&ContainerName::from_str_unsafe("valid-container-name")); + } + #[test] fn test_envvarname_format() { assert_eq!( @@ -198,12 +236,12 @@ mod tests { ]); let env_var_set2 = EnvVarSet::new() .with_value( - EnvVarName::from_str_unsafe("ENV2"), + &EnvVarName::from_str_unsafe("ENV2"), "value2 from env_var_set2", ) - .with_field_path(EnvVarName::from_str_unsafe("ENV3"), FieldPathEnvVar::Name) + .with_field_path(&EnvVarName::from_str_unsafe("ENV3"), FieldPathEnvVar::Name) .with_value( - EnvVarName::from_str_unsafe("ENV4"), + &EnvVarName::from_str_unsafe("ENV4"), "value4 from env_var_set2", ); @@ -268,7 +306,7 @@ mod tests { #[test] fn test_envvarset_with_value() { - let env_var_set = EnvVarSet::new().with_value(EnvVarName::from_str_unsafe("ENV"), "value"); + let env_var_set = EnvVarSet::new().with_value(&EnvVarName::from_str_unsafe("ENV"), "value"); assert_eq!( Some(&EnvVar { @@ -276,14 +314,14 @@ mod tests { value: Some("value".to_owned()), value_from: None }), - env_var_set.get(EnvVarName::from_str_unsafe("ENV")) + env_var_set.get(&EnvVarName::from_str_unsafe("ENV")) ); } #[test] fn test_envvarset_with_field_path() { let env_var_set = EnvVarSet::new() - .with_field_path(EnvVarName::from_str_unsafe("ENV"), FieldPathEnvVar::Name); + .with_field_path(&EnvVarName::from_str_unsafe("ENV"), FieldPathEnvVar::Name); assert_eq!( Some(&EnvVar { @@ -297,7 +335,32 @@ mod tests { ..EnvVarSource::default() }), }), - env_var_set.get(EnvVarName::from_str_unsafe("ENV")) + env_var_set.get(&EnvVarName::from_str_unsafe("ENV")) + ); + } + + #[test] + fn test_envvarset_with_config_map_key_ref() { + let env_var_set = EnvVarSet::new().with_config_map_key_ref( + &EnvVarName::from_str_unsafe("ENV"), + &ConfigMapName::from_str_unsafe("config-map"), + &ConfigMapKey::from_str_unsafe("key"), + ); + + assert_eq!( + Some(&EnvVar { + name: "ENV".to_owned(), + value: None, + value_from: Some(EnvVarSource { + config_map_key_ref: Some(ConfigMapKeySelector { + key: "key".to_owned(), + name: "config-map".to_owned(), + ..ConfigMapKeySelector::default() + }), + ..EnvVarSource::default() + }), + }), + env_var_set.get(&EnvVarName::from_str_unsafe("ENV")) ); } } diff --git a/rust/operator-binary/src/framework/product_logging.rs b/rust/operator-binary/src/framework/product_logging.rs new file mode 100644 index 0000000..0c71749 --- /dev/null +++ b/rust/operator-binary/src/framework/product_logging.rs @@ -0,0 +1 @@ +pub mod framework; diff --git a/rust/operator-binary/src/framework/product_logging/framework.rs b/rust/operator-binary/src/framework/product_logging/framework.rs new file mode 100644 index 0000000..1b49109 --- /dev/null +++ b/rust/operator-binary/src/framework/product_logging/framework.rs @@ -0,0 +1,484 @@ +use std::{fmt::Display, str::FromStr}; + +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::{ + builder::pod::{container::FieldPathEnvVar, resources::ResourceRequirementsBuilder}, + commons::product_image_selection::ResolvedProductImage, + k8s_openapi::api::core::v1::{Container, VolumeMount}, + product_logging::{ + framework::VECTOR_CONFIG_FILE, + spec::{ + AppenderConfig, AutomaticContainerLogConfig, ConfigMapLogConfig, + ContainerLogConfigChoice, CustomContainerLogConfig, LogLevel, Logging, + }, + }, +}; +use strum::{EnumDiscriminants, IntoStaticStr}; + +use crate::{ + constant, + framework::{ + ConfigMapKey, ConfigMapName, ContainerName, VolumeName, + builder::pod::container::{EnvVarName, EnvVarSet, new_container_builder}, + role_group_utils, + }, +}; + +// Copy of the private constant `stackable_operator::product_logging::framework::STACKABLE_CONFIG_DIR` +const STACKABLE_CONFIG_DIR: &str = "/stackable/config"; + +// Copy of the private constant `stackable_operator::product_logging::framework::VECTOR_LOG_DIR` +const VECTOR_CONTROL_DIR: &str = "_vector"; + +// Copy of the private constant `stackable_operator::product_logging::framework::SHUTDOWN_FILE` +const SHUTDOWN_FILE: &str = "shutdown"; + +// Public variant of `stackable_operator::product_logging::framework::STACKABLE_LOG_DIR` +/// Directory where the logs are stored +pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; + +// Copy of the private constant `stackable_operator::product_logging::framework::VECTOR_AGGREGATOR_CM_KEY` +constant!(VECTOR_AGGREGATOR_CM_KEY: ConfigMapKey = "ADDRESS"); + +// Copy of the private constant `stackable_operator::product_logging::framework::VECTOR_AGGREGATOR_ADDRESS` +constant!(VECTOR_AGGREGATOR_ENV_NAME: EnvVarName = "VECTOR_AGGREGATOR_ADDRESS"); + +#[derive(Debug, EnumDiscriminants, Snafu)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("failed to get the container log configuration"))] + GetContainerLogConfiguration { container: String }, + + #[snafu(display("failed to parse the container name"))] + ParseContainerName { source: crate::framework::Error }, +} + +type Result = std::result::Result; + +/// Validated [`ContainerLogConfigChoice`] +/// +/// The ConfigMap name in the Custom variant is valid. +#[derive(Clone, Debug, PartialEq)] +pub enum ValidatedContainerLogConfigChoice { + Automatic(AutomaticContainerLogConfig), + Custom(ConfigMapName), +} + +/// Validated [`ContainerLogConfigChoice`] for the Vector container +/// +/// It includes the discovery ConfigMap name of the Vector aggregator. +#[derive(Clone, Debug, PartialEq)] +pub struct VectorContainerLogConfig { + pub log_config: ValidatedContainerLogConfigChoice, + pub vector_aggregator_config_map_name: ConfigMapName, +} + +/// Validates the log configuration of the container +pub fn validate_logging_configuration_for_container( + logging: &Logging, + container: T, +) -> Result +where + T: Clone + Display + Ord, +{ + let container_log_config_choice = logging + .containers + .get(&container) + .and_then(|container_log_config| container_log_config.choice.as_ref()) + // This should never happen because default configurations should have been set for all + // containers. + .context(GetContainerLogConfigurationSnafu { + container: container.to_string(), + })?; + + let validated_container_log_config_choice = match container_log_config_choice { + ContainerLogConfigChoice::Automatic(automatic_log_config) => { + ValidatedContainerLogConfigChoice::Automatic(automatic_log_config.clone()) + } + ContainerLogConfigChoice::Custom(CustomContainerLogConfig { + custom: ConfigMapLogConfig { config_map }, + }) => ValidatedContainerLogConfigChoice::Custom( + ConfigMapName::from_str(config_map).context(ParseContainerNameSnafu)?, + ), + }; + + Ok(validated_container_log_config_choice) +} + +/// Builds the Vector container +pub fn vector_container( + container_name: &ContainerName, + image: &ResolvedProductImage, + vector_container_log_config: &VectorContainerLogConfig, + resource_names: &role_group_utils::ResourceNames, + log_config_volume_name: &VolumeName, + log_volume_name: &VolumeName, + extra_env_vars: EnvVarSet, +) -> Container { + let log_level = if let ValidatedContainerLogConfigChoice::Automatic(log_config) = + &vector_container_log_config.log_config + { + log_config.root_log_level() + } else { + LogLevel::default() + }; + let vector_file_log_level = + if let ValidatedContainerLogConfigChoice::Automatic(AutomaticContainerLogConfig { + file: Some(AppenderConfig { + level: Some(log_level), + }), + .. + }) = vector_container_log_config.log_config + { + log_level + } else { + LogLevel::default() + }; + + let env_vars = EnvVarSet::new() + .with_value( + &EnvVarName::from_str_unsafe("CLUSTER_NAME"), + &resource_names.cluster_name, + ) + .with_value(&EnvVarName::from_str_unsafe("LOG_DIR"), "/stackable/log") + .with_field_path( + &EnvVarName::from_str_unsafe("NAMESPACE"), + FieldPathEnvVar::Namespace, + ) + .with_value( + &EnvVarName::from_str_unsafe("ROLE_GROUP_NAME"), + &resource_names.role_group_name, + ) + .with_value( + &EnvVarName::from_str_unsafe("ROLE_NAME"), + &resource_names.role_name, + ) + .with_config_map_key_ref( + &VECTOR_AGGREGATOR_ENV_NAME, + &vector_container_log_config.vector_aggregator_config_map_name, + &VECTOR_AGGREGATOR_CM_KEY, + ) + .with_value( + &EnvVarName::from_str_unsafe("VECTOR_CONFIG_YAML"), + format!("{STACKABLE_CONFIG_DIR}/{VECTOR_CONFIG_FILE}"), + ) + .with_value( + &EnvVarName::from_str_unsafe("VECTOR_FILE_LOG_LEVEL"), + vector_file_log_level.to_vector_literal(), + ) + .with_value( + &EnvVarName::from_str_unsafe("VECTOR_LOG"), + log_level.to_vector_literal(), + ) + .merge(extra_env_vars); + + let resources = ResourceRequirementsBuilder::new() + .with_cpu_request("250m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(); + + new_container_builder(container_name) + .image_from_product_image(image) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![format!( + "# Vector will ignore SIGTERM (as PID != 1) and must be shut down by writing a shutdown trigger file\n\ + vector & vector_pid=$!\n\ + if [ ! -f \"{vector_control_directory}/{SHUTDOWN_FILE}\" ]; then\n\ + mkdir -p {vector_control_directory}\n\ + inotifywait -qq --event create {vector_control_directory};\n\ + fi\n\ + sleep 1\n\ + kill $vector_pid", + vector_control_directory = format!("{STACKABLE_LOG_DIR}/{VECTOR_CONTROL_DIR}"), + )]) + .add_env_vars(env_vars.into()) + .add_volume_mounts([ + VolumeMount { + mount_path: format!( + "{STACKABLE_CONFIG_DIR}/{VECTOR_CONFIG_FILE}" + ), + name: log_config_volume_name.to_string(), + read_only: Some(true), + sub_path: Some(VECTOR_CONFIG_FILE.to_owned()), + ..VolumeMount::default() + }, + VolumeMount { + mount_path: STACKABLE_LOG_DIR.to_owned(), + name: log_volume_name.to_string(), + ..VolumeMount::default() + }, + ]) + .expect("The mount paths are statically defined and there should be no duplicates.") + .resources(resources) + .build() +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use pretty_assertions::assert_eq; + use serde_json::json; + use stackable_operator::{ + commons::product_image_selection::ResolvedProductImage, + kvp::LabelValue, + product_logging::spec::{ + AutomaticContainerLogConfig, ConfigMapLogConfig, ContainerLogConfig, + ContainerLogConfigChoice, CustomContainerLogConfig, Logging, + }, + }; + + use super::{ + ErrorDiscriminants, ValidatedContainerLogConfigChoice, VectorContainerLogConfig, + validate_logging_configuration_for_container, vector_container, + }; + use crate::framework::{ + ClusterName, ConfigMapName, ContainerName, RoleGroupName, RoleName, VolumeName, + builder::pod::container::{EnvVarName, EnvVarSet}, + role_group_utils, + }; + + #[test] + fn test_validate_logging_configuration_for_container_ok_automatic_log_config() { + let logging = Logging { + enable_vector_agent: false, + containers: [( + "container", + ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + )), + }, + )] + .into(), + }; + + assert_eq!( + ValidatedContainerLogConfigChoice::Automatic(AutomaticContainerLogConfig::default()), + validate_logging_configuration_for_container(&logging, "container") + .expect("should be a valid log config") + ); + } + + #[test] + fn test_validate_logging_configuration_for_container_ok_custom_log_config() { + let logging = Logging { + enable_vector_agent: false, + containers: [( + "container", + ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { + custom: ConfigMapLogConfig { + config_map: "valid-config-map-name".to_owned(), + }, + })), + }, + )] + .into(), + }; + + assert_eq!( + ValidatedContainerLogConfigChoice::Custom(ConfigMapName::from_str_unsafe( + "valid-config-map-name" + )), + validate_logging_configuration_for_container(&logging, "container") + .expect("should be a valid log config") + ); + } + + #[test] + fn test_validate_logging_configuration_for_container_err_get_container_log_configuration() { + let logging_without_container = Logging { + enable_vector_agent: false, + containers: [].into(), + }; + let logging_without_container_log_config_choice = Logging { + enable_vector_agent: false, + containers: [("container", ContainerLogConfig { choice: None })].into(), + }; + + assert_eq!( + Err(ErrorDiscriminants::GetContainerLogConfiguration), + validate_logging_configuration_for_container(&logging_without_container, "container") + .map_err(ErrorDiscriminants::from) + ); + + assert_eq!( + Err(ErrorDiscriminants::GetContainerLogConfiguration), + validate_logging_configuration_for_container( + &logging_without_container_log_config_choice, + "container" + ) + .map_err(ErrorDiscriminants::from) + ); + } + + #[test] + fn test_validate_logging_configuration_for_container_err_parse_container_name() { + let logging = Logging { + enable_vector_agent: false, + containers: [( + "container", + ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { + custom: ConfigMapLogConfig { + config_map: "invalid ConfigMap name".to_owned(), + }, + })), + }, + )] + .into(), + }; + + assert_eq!( + Err(ErrorDiscriminants::ParseContainerName), + validate_logging_configuration_for_container(&logging, "container") + .map_err(ErrorDiscriminants::from) + ); + } + + #[test] + fn test_vector_container() { + let image = ResolvedProductImage { + product_version: "1.0.0".to_owned(), + app_version_label_value: LabelValue::from_str("1.0.0-stackable0.0.0-dev") + .expect("should be a valid label value"), + image: "oci.stackable.tech/sdp/product:1.0.0-stackable0.0.0-dev".to_string(), + image_pull_policy: "Always".to_owned(), + pull_secrets: None, + }; + + let vector_container_log_config = VectorContainerLogConfig { + log_config: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + vector_aggregator_config_map_name: ConfigMapName::from_str_unsafe("vector-aggregator"), + }; + + let resource_names = role_group_utils::ResourceNames { + cluster_name: ClusterName::from_str_unsafe("test-cluster"), + role_name: RoleName::from_str_unsafe("role"), + role_group_name: RoleGroupName::from_str_unsafe("role-group"), + }; + + let vector_container = vector_container( + &ContainerName::from_str_unsafe("vector"), + &image, + &vector_container_log_config, + &resource_names, + &VolumeName::from_str_unsafe("config"), + &VolumeName::from_str_unsafe("log"), + EnvVarSet::new().with_value(&EnvVarName::from_str_unsafe("CUSTOM_ENV"), "test"), + ); + + assert_eq!( + json!( + { + "args": [ + concat!( + "# Vector will ignore SIGTERM (as PID != 1) and must be shut down by writing a shutdown trigger file\n", + "vector & vector_pid=$!\n", + "if [ ! -f \"/stackable/log/_vector/shutdown\" ]; then\n", + "mkdir -p /stackable/log/_vector\n", + "inotifywait -qq --event create /stackable/log/_vector;\n", + "fi\n", + "sleep 1\n", + "kill $vector_pid" + ), + ], + "command": [ + "/bin/bash", + "-x", + "-euo", + "pipefail", + "-c", + ], + "env": [ + { + "name": "CLUSTER_NAME", + "value": "test-cluster", + }, + { + "name": "CUSTOM_ENV", + "value": "test", + }, + { + "name": "LOG_DIR", + "value": "/stackable/log", + }, + { + "name": "NAMESPACE", + "valueFrom": { + "fieldRef": { + "fieldPath": "metadata.namespace", + }, + }, + }, + { + "name": "ROLE_GROUP_NAME", + "value": "role-group", + }, + { + "name": "ROLE_NAME", + "value": "role", + }, + { + "name": "VECTOR_AGGREGATOR_ADDRESS", + "valueFrom": { + "configMapKeyRef": { + "key": "ADDRESS", + "name": "vector-aggregator", + }, + }, + }, + { + "name": "VECTOR_CONFIG_YAML", + "value": "/stackable/config/vector.yaml", + }, + { + "name": "VECTOR_FILE_LOG_LEVEL", + "value": "info", + }, + { + "name": "VECTOR_LOG", + "value": "info", + }, + ], + "image": "oci.stackable.tech/sdp/product:1.0.0-stackable0.0.0-dev", + "imagePullPolicy": "Always", + "name": "vector", + "resources": { + "limits": { + "cpu": "500m", + "memory": "128Mi", + }, + "requests": { + "cpu": "250m", + "memory": "128Mi", + }, + }, + "volumeMounts": [ + { + "mountPath": "/stackable/config/vector.yaml", + "name": "config", + "readOnly": true, + "subPath": "vector.yaml", + }, + { + "mountPath": "/stackable/log", + "name": "log", + }, + ], + }), + serde_json::to_value(vector_container).expect("should be serializable") + ); + } +} diff --git a/rust/operator-binary/src/framework/validation.rs b/rust/operator-binary/src/framework/validation.rs new file mode 100644 index 0000000..b4e3a20 --- /dev/null +++ b/rust/operator-binary/src/framework/validation.rs @@ -0,0 +1,96 @@ +use std::sync::LazyLock; + +use regex::Regex; +use snafu::{Snafu, ensure}; +use stackable_operator::validation::RFC_1123_SUBDOMAIN_MAX_LENGTH; + +/// Format of a key for a ConfigMap or Secret +pub const CONFIG_MAP_KEY_FMT: &str = "[-._a-zA-Z0-9]+"; +const CONFIG_MAP_KEY_ERROR_MSG: &str = + "a valid config key must consist of alphanumeric characters, '-', '_' or '.'"; +static CONFIG_MAP_KEY_REGEX: LazyLock = LazyLock::new(|| { + Regex::new(&format!("^{CONFIG_MAP_KEY_FMT}$")).expect("failed to compile ConfigMap key regex") +}); + +#[derive(Debug, Eq, PartialEq, Snafu)] +pub enum Error { + #[snafu(display("value is empty"))] + Empty { value: String }, + + #[snafu(display("value does not match the regular expression"))] + Regex { + value: String, + regex: &'static str, + message: &'static str, + }, + + #[snafu(display("value exceeds the maximum length"))] + TooLong { value: String, max_length: usize }, +} + +type Result = std::result::Result<(), Error>; + +/// Tests if the given value is a valid key for a ConfigMap or Secret +/// +/// see +pub fn is_config_map_key(value: &str) -> Result { + // When adding this function to stackable_operator, use the private functions like + // validate_all. + + ensure!(!value.is_empty(), EmptySnafu { value }); + + let max_length = RFC_1123_SUBDOMAIN_MAX_LENGTH; + ensure!( + value.len() <= max_length, + TooLongSnafu { + value: value.to_owned(), + max_length + } + ); + + ensure!( + CONFIG_MAP_KEY_REGEX.is_match(value), + RegexSnafu { + value: value.to_owned(), + regex: CONFIG_MAP_KEY_FMT, + message: CONFIG_MAP_KEY_ERROR_MSG + } + ); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{CONFIG_MAP_KEY_ERROR_MSG, CONFIG_MAP_KEY_FMT, Error, is_config_map_key}; + + #[test] + fn test_is_config_map_key() { + assert_eq!(Ok(()), is_config_map_key("_a-A.1")); + + assert_eq!( + Err(Error::Empty { + value: "".to_owned() + }), + is_config_map_key("") + ); + + assert_eq!(Ok(()), is_config_map_key(&"a".repeat(253))); + assert_eq!( + Err(Error::TooLong { + value: "a".repeat(254), + max_length: 253 + }), + is_config_map_key(&"a".repeat(254)) + ); + + assert_eq!( + Err(Error::Regex { + value: " ".to_string(), + regex: CONFIG_MAP_KEY_FMT, + message: CONFIG_MAP_KEY_ERROR_MSG, + }), + is_config_map_key(" ") + ); + } +} diff --git a/tests/templates/kuttl/external-access/02-assert.yaml.j2 b/tests/templates/kuttl/external-access/02-assert.yaml.j2 new file mode 100644 index 0000000..50b1d4c --- /dev/null +++ b/tests/templates/kuttl/external-access/02-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/external-access/02-install-vector-aggregator-discovery-config-map.yaml.j2 b/tests/templates/kuttl/external-access/02-install-vector-aggregator-discovery-config-map.yaml.j2 new file mode 100644 index 0000000..2d6a0df --- /dev/null +++ b/tests/templates/kuttl/external-access/02-install-vector-aggregator-discovery-config-map.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/external-access/opensearch.yaml.j2 b/tests/templates/kuttl/external-access/opensearch.yaml.j2 index 3dc66a3..484627d 100644 --- a/tests/templates/kuttl/external-access/opensearch.yaml.j2 +++ b/tests/templates/kuttl/external-access/opensearch.yaml.j2 @@ -12,7 +12,14 @@ spec: productVersion: "{{ test_scenario['values']['opensearch'] }}" {% endif %} pullPolicy: IfNotPresent +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + clusterConfig: + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: cluster-manager: config: diff --git a/tests/templates/kuttl/ldap/02-assert.yaml.j2 b/tests/templates/kuttl/ldap/02-assert.yaml.j2 new file mode 100644 index 0000000..50b1d4c --- /dev/null +++ b/tests/templates/kuttl/ldap/02-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/ldap/02-install-vector-aggregator-discovery-config-map.yaml.j2 b/tests/templates/kuttl/ldap/02-install-vector-aggregator-discovery-config-map.yaml.j2 new file mode 100644 index 0000000..2d6a0df --- /dev/null +++ b/tests/templates/kuttl/ldap/02-install-vector-aggregator-discovery-config-map.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/ldap/11-create-ldap-user.yaml b/tests/templates/kuttl/ldap/11-create-ldap-user.yaml index 455e62d..0d0a016 100644 --- a/tests/templates/kuttl/ldap/11-create-ldap-user.yaml +++ b/tests/templates/kuttl/ldap/11-create-ldap-user.yaml @@ -78,6 +78,8 @@ spec: requests: storage: "1" serviceAccountName: test-service-account + securityContext: + fsGroup: 1000 restartPolicy: OnFailure --- apiVersion: v1 diff --git a/tests/templates/kuttl/ldap/21-install-opensearch.yaml.j2 b/tests/templates/kuttl/ldap/21-install-opensearch.yaml.j2 index 79c13fb..5f0b615 100644 --- a/tests/templates/kuttl/ldap/21-install-opensearch.yaml.j2 +++ b/tests/templates/kuttl/ldap/21-install-opensearch.yaml.j2 @@ -12,7 +12,14 @@ spec: productVersion: "{{ test_scenario['values']['opensearch'] }}" {% endif %} pullPolicy: IfNotPresent +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + clusterConfig: + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: default: replicas: 3 diff --git a/tests/templates/kuttl/logging/00-patch-ns.yaml b/tests/templates/kuttl/logging/00-patch-ns.yaml new file mode 100644 index 0000000..d4f91fa --- /dev/null +++ b/tests/templates/kuttl/logging/00-patch-ns.yaml @@ -0,0 +1,15 @@ +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl patch namespace $NAMESPACE --patch=' + { + "metadata": { + "labels": { + "pod-security.kubernetes.io/enforce": "privileged" + } + } + }' + timeout: 120 diff --git a/tests/templates/kuttl/logging/01-rbac.yaml b/tests/templates/kuttl/logging/01-rbac.yaml new file mode 100644 index 0000000..64eced8 --- /dev/null +++ b/tests/templates/kuttl/logging/01-rbac.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-service-account +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role +rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - privileged + verbs: + - use +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role-binding +subjects: + - kind: ServiceAccount + name: test-service-account +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: test-role diff --git a/tests/templates/kuttl/logging/10-assert.yaml b/tests/templates/kuttl/logging/10-assert.yaml new file mode 100644 index 0000000..88b268d --- /dev/null +++ b/tests/templates/kuttl/logging/10-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: opensearch-vector-aggregator +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/logging/10-install-opensearch-vector-aggregator.yaml b/tests/templates/kuttl/logging/10-install-opensearch-vector-aggregator.yaml new file mode 100644 index 0000000..dfef758 --- /dev/null +++ b/tests/templates/kuttl/logging/10-install-opensearch-vector-aggregator.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install opensearch-vector-aggregator vector + --namespace $NAMESPACE + --version 0.45.0 + --repo https://helm.vector.dev + --values 10_opensearch-vector-aggregator-values.yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: opensearch-vector-aggregator-discovery +data: + ADDRESS: opensearch-vector-aggregator:6123 diff --git a/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 new file mode 100644 index 0000000..c6cb2ed --- /dev/null +++ b/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 @@ -0,0 +1,86 @@ +--- +role: Aggregator +service: + ports: + - name: api + port: 8686 + protocol: TCP + targetPort: 8686 + - name: vector + port: 6123 + protocol: TCP + targetPort: 6000 +customConfig: + api: + address: 0.0.0.0:8686 + enabled: true + sources: + vector: + address: 0.0.0.0:6000 + type: vector + version: "2" + transforms: + validEvents: + type: filter + inputs: + - vector + condition: is_null(.errors) + filteredAutomaticLogConfigOpenSearch: + type: filter + inputs: + - validEvents + condition: >- + .pod == "opensearch-nodes-automatic-0" && + .container == "opensearch" + filteredAutomaticLogConfigVector: + type: filter + inputs: + - validEvents + condition: >- + .pod == "opensearch-nodes-automatic-0" && + .container == "vector" + filteredCustomLogConfigOpenSearch: + type: filter + inputs: + - validEvents + condition: >- + .pod == "opensearch-nodes-custom-0" && + .container == "opensearch" + filteredCustomLogConfigVector: + type: filter + inputs: + - validEvents + condition: >- + .pod == "opensearch-nodes-custom-0" && + .container == "vector" + filteredInvalidEvents: + type: filter + inputs: + - vector + condition: |- + .timestamp == from_unix_timestamp!(0) || + is_null(.level) || + is_null(.logger) || + is_null(.message) + sinks: + test: + inputs: + - filtered* + type: blackhole + stdout: + inputs: + - vector + type: console + encoding: + codec: json +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + aggregator: + inputs: + - vector + type: vector + address: {{ lookup('env', 'VECTOR_AGGREGATOR') }} + buffer: + # Avoid back pressure from VECTOR_AGGREGATOR. The test should + # not fail if the aggregator is not available. + when_full: drop_newest +{% endif %} diff --git a/tests/templates/kuttl/logging/20-assert.yaml.j2 b/tests/templates/kuttl/logging/20-assert.yaml.j2 new file mode 100644 index 0000000..e705ea3 --- /dev/null +++ b/tests/templates/kuttl/logging/20-assert.yaml.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: opensearch-nodes-automatic +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/logging/20-install-opensearch.yaml.j2 b/tests/templates/kuttl/logging/20-install-opensearch.yaml.j2 new file mode 100644 index 0000000..f17b427 --- /dev/null +++ b/tests/templates/kuttl/logging/20-install-opensearch.yaml.j2 @@ -0,0 +1,224 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: custom-log-config +data: + log4j2.properties: | + rootLogger.level = INFO + rootLogger.appenderRef.FILE.ref = FILE + appender.FILE.type = File + appender.FILE.name = FILE + appender.FILE.fileName = /stackable/log/opensearch/opensearch_server.json + appender.FILE.layout.type = OpenSearchJsonLayout + appender.FILE.layout.type_name = server +--- +apiVersion: opensearch.stackable.tech/v1alpha1 +kind: OpenSearchCluster +metadata: + name: opensearch +spec: + image: +{% if test_scenario['values']['opensearch'].find(",") > 0 %} + custom: "{{ test_scenario['values']['opensearch'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['opensearch'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['opensearch'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + vectorAggregatorConfigMapName: opensearch-vector-aggregator-discovery + nodes: + roleGroups: + automatic: + config: + logging: + enableVectorAgent: true + containers: + opensearch: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO + vector: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO + replicas: 1 + podOverrides: + spec: + volumes: + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-automatic-headless + custom: + config: + logging: + enableVectorAgent: true + containers: + opensearch: + custom: + configMap: custom-log-config + replicas: 1 + podOverrides: + spec: + volumes: + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-custom-headless + configOverrides: + opensearch.yml: + # Disable memory mapping in this test; If memory mapping were activated, the kernel setting + # vm.max_map_count would have to be increased to 262144 on the node. + node.store.allow_mmap: "false" + # Disable the disk allocation decider in this test; Otherwise the test depends on the disk + # usage of the node and if the relative watermark set in + # `cluster.routing.allocation.disk.watermark.high` is reached then the security index could + # not be created even if enough disk space would be available. + cluster.routing.allocation.disk.threshold_enabled: "false" + plugins.security.allow_default_init_securityindex: "true" + plugins.security.ssl.transport.enabled: "true" + plugins.security.ssl.transport.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt + plugins.security.ssl.transport.pemkey_filepath: /stackable/opensearch/config/tls/tls.key + plugins.security.ssl.transport.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt + plugins.security.ssl.http.enabled: "true" + plugins.security.ssl.http.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt + plugins.security.ssl.http.pemkey_filepath: /stackable/opensearch/config/tls/tls.key + plugins.security.ssl.http.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt + podOverrides: + spec: + containers: + - name: opensearch + volumeMounts: + - name: security-config + mountPath: /stackable/opensearch/config/opensearch-security + readOnly: true + - name: tls + mountPath: /stackable/opensearch/config/tls + readOnly: true + volumes: + - name: security-config + secret: + secretName: opensearch-security-config + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" +--- +apiVersion: v1 +kind: Secret +metadata: + name: opensearch-security-config +stringData: + action_groups.yml: | + --- + _meta: + type: actiongroups + config_version: 2 + allowlist.yml: | + --- + _meta: + type: allowlist + config_version: 2 + + config: + enabled: false + audit.yml: | + --- + _meta: + type: audit + config_version: 2 + + config: + enabled: false + config.yml: | + --- + _meta: + type: config + config_version: 2 + + config: + dynamic: + authc: + basic_internal_auth_domain: + description: Authenticate via HTTP Basic against internal users database + http_enabled: true + transport_enabled: true + order: 1 + http_authenticator: + type: basic + challenge: true + authentication_backend: + type: intern + authz: {} + internal_users.yml: | + --- + # The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh + + _meta: + type: internalusers + config_version: 2 + + admin: + hash: $2y$10$xRtHZFJ9QhG9GcYhRpAGpufCZYsk//nxsuel5URh0GWEBgmiI4Q/e + reserved: true + backend_roles: + - admin + description: OpenSearch admin user + + kibanaserver: + hash: $2y$10$vPgQ/6ilKDM5utawBqxoR.7euhVQ0qeGl8mPTeKhmFT475WUDrfQS + reserved: true + description: OpenSearch Dashboards user + nodes_dn.yml: | + --- + _meta: + type: nodesdn + config_version: 2 + roles.yml: | + --- + _meta: + type: roles + config_version: 2 + roles_mapping.yml: | + --- + _meta: + type: rolesmapping + config_version: 2 + + all_access: + reserved: false + backend_roles: + - admin + + kibana_server: + reserved: true + users: + - kibanaserver + tenants.yml: | + --- + _meta: + type: tenants + config_version: 2 diff --git a/tests/templates/kuttl/logging/30-assert.yaml b/tests/templates/kuttl/logging/30-assert.yaml new file mode 100644 index 0000000..cae2443 --- /dev/null +++ b/tests/templates/kuttl/logging/30-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-log-aggregation +status: + succeeded: 1 diff --git a/tests/templates/kuttl/logging/30-test-opensearch.yaml b/tests/templates/kuttl/logging/30-test-opensearch.yaml new file mode 100644 index 0000000..ee9f1f4 --- /dev/null +++ b/tests/templates/kuttl/logging/30-test-opensearch.yaml @@ -0,0 +1,93 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-log-aggregation +spec: + template: + spec: + containers: + - name: test-log-aggregation + image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev + command: + - python + args: + - scripts/test.py + volumeMounts: + - name: script + mountPath: /stackable/scripts + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + resources: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 128Mi + cpu: 400m + volumes: + - name: script + configMap: + name: test-log-aggregation + serviceAccountName: test-service-account + securityContext: + fsGroup: 1000 + restartPolicy: OnFailure +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-log-aggregation +data: + test.py: | + import requests + + + def check_sent_events(): + response = requests.post( + 'http://opensearch-vector-aggregator:8686/graphql', + json={ + 'query': """ + { + transforms(first:100) { + nodes { + componentId + metrics { + sentEventsTotal { + sentEventsTotal + } + } + } + } + } + """ + } + ) + + assert response.status_code == 200, \ + 'Cannot access the API of the vector aggregator.' + + result = response.json() + + transforms = result['data']['transforms']['nodes'] + for transform in transforms: + sentEvents = transform['metrics']['sentEventsTotal'] + componentId = transform['componentId'] + + if componentId == 'filteredInvalidEvents': + assert sentEvents is None or \ + sentEvents['sentEventsTotal'] == 0, \ + 'Invalid log events were sent.' + else: + assert sentEvents is not None and \ + sentEvents['sentEventsTotal'] > 0, \ + f'No events were sent in "{componentId}".' + + + if __name__ == '__main__': + check_sent_events() + print('Test successful!') diff --git a/tests/templates/kuttl/metrics/02-assert.yaml.j2 b/tests/templates/kuttl/metrics/02-assert.yaml.j2 new file mode 100644 index 0000000..50b1d4c --- /dev/null +++ b/tests/templates/kuttl/metrics/02-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/metrics/02-install-vector-aggregator-discovery-config-map.yaml.j2 b/tests/templates/kuttl/metrics/02-install-vector-aggregator-discovery-config-map.yaml.j2 new file mode 100644 index 0000000..2d6a0df --- /dev/null +++ b/tests/templates/kuttl/metrics/02-install-vector-aggregator-discovery-config-map.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 b/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 index c5cf3ad..d6e2d91 100644 --- a/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 +++ b/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 @@ -12,7 +12,14 @@ spec: productVersion: "{{ test_scenario['values']['opensearch'] }}" {% endif %} pullPolicy: IfNotPresent +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + clusterConfig: + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: default: config: @@ -22,8 +29,6 @@ spec: capacity: 100Mi listenerClass: external-stable replicas: 3 - envOverrides: - OPENSEARCH_HOME: {{ test_scenario['values']['opensearch_home'] }} configOverrides: opensearch.yml: # Disable memory mapping in this test; If memory mapping were activated, the kernel setting @@ -36,23 +41,23 @@ spec: cluster.routing.allocation.disk.threshold_enabled: "false" plugins.security.allow_default_init_securityindex: "true" plugins.security.ssl.transport.enabled: "true" - plugins.security.ssl.transport.pemcert_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt - plugins.security.ssl.transport.pemkey_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key - plugins.security.ssl.transport.pemtrustedcas_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt + plugins.security.ssl.transport.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt + plugins.security.ssl.transport.pemkey_filepath: /stackable/opensearch/config/tls/tls.key + plugins.security.ssl.transport.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt plugins.security.ssl.http.enabled: "true" - plugins.security.ssl.http.pemcert_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt - plugins.security.ssl.http.pemkey_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key - plugins.security.ssl.http.pemtrustedcas_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt + plugins.security.ssl.http.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt + plugins.security.ssl.http.pemkey_filepath: /stackable/opensearch/config/tls/tls.key + plugins.security.ssl.http.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt podOverrides: spec: containers: - name: opensearch volumeMounts: - name: security-config - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch-security + mountPath: /stackable/opensearch/config/opensearch-security readOnly: true - name: tls - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/tls + mountPath: /stackable/opensearch/config/tls readOnly: true volumes: - name: security-config diff --git a/tests/templates/kuttl/metrics/30-check-metrics.yaml b/tests/templates/kuttl/metrics/30-check-metrics.yaml index 0aa4f4d..489c77c 100644 --- a/tests/templates/kuttl/metrics/30-check-metrics.yaml +++ b/tests/templates/kuttl/metrics/30-check-metrics.yaml @@ -14,8 +14,24 @@ spec: - -euo - pipefail - -c + args: - > curl http://prometheus-operated:9090/api/v1/query?query=opensearch_cluster_nodes_number%7Bpod%3D%22opensearch-nodes-default-0%22%7D | jq --exit-status '.data.result[0].value[1] == "3"' + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + resources: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 128Mi + cpu: 400m serviceAccountName: test-service-account + securityContext: + fsGroup: 1000 restartPolicy: OnFailure diff --git a/tests/templates/kuttl/smoke/03-assert.yaml.j2 b/tests/templates/kuttl/smoke/03-assert.yaml.j2 new file mode 100644 index 0000000..50b1d4c --- /dev/null +++ b/tests/templates/kuttl/smoke/03-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/smoke/03-install-vector-aggregator-discovery-config-map.yaml.j2 b/tests/templates/kuttl/smoke/03-install-vector-aggregator-discovery-config-map.yaml.j2 new file mode 100644 index 0000000..2d6a0df --- /dev/null +++ b/tests/templates/kuttl/smoke/03-install-vector-aggregator-discovery-config-map.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/smoke/10-assert.yaml.j2 b/tests/templates/kuttl/smoke/10-assert.yaml.j2 index 729b588..c128217 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-assert.yaml.j2 @@ -35,6 +35,8 @@ spec: serviceName: opensearch-nodes-cluster-manager-headless template: metadata: + annotations: + kubectl.kubernetes.io/default-container: opensearch labels: app.kubernetes.io/component: nodes app.kubernetes.io/instance: opensearch @@ -57,8 +59,54 @@ spec: topologyKey: kubernetes.io/hostname weight: 1 containers: - - command: - - {{ test_scenario['values']['opensearch_home'] }}/opensearch-docker-entrypoint.sh + - args: + - |- + + prepare_signal_handlers() + { + unset term_child_pid + unset term_kill_needed + trap 'handle_term_signal' TERM + } + + handle_term_signal() + { + if [ "${term_child_pid}" ]; then + kill -TERM "${term_child_pid}" 2>/dev/null + else + term_kill_needed="yes" + fi + } + + wait_for_termination() + { + set +e + term_child_pid=$1 + if [[ -v term_kill_needed ]]; then + kill -TERM "${term_child_pid}" 2>/dev/null + fi + wait ${term_child_pid} 2>/dev/null + trap - TERM + wait ${term_child_pid} 2>/dev/null + set -e + } + + rm -f /stackable/log/_vector/shutdown + prepare_signal_handlers + if command --search containerdebug >/dev/null 2>&1; then + containerdebug --output=/stackable/log/containerdebug-state.json --loop & + else + echo >&2 "containerdebug not installed; Proceed without it." + fi + ./opensearch-docker-entrypoint.sh & + wait_for_termination $! + mkdir -p /stackable/log/_vector && touch /stackable/log/_vector/shutdown + command: + - /bin/bash + - -x + - -euo + - pipefail + - -c env: - name: DISABLE_INSTALL_DEMO_CONFIG value: "true" @@ -111,16 +159,86 @@ spec: name: config readOnly: true subPath: opensearch.yml + - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/log4j2.properties + name: log-config + readOnly: true + subPath: log4j2.properties - mountPath: {{ test_scenario['values']['opensearch_home'] }}/data name: data - mountPath: /stackable/listener name: listener + - mountPath: /stackable/log + name: log - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch-security name: security-config readOnly: true - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/tls name: tls readOnly: true +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + - args: + - |- + # Vector will ignore SIGTERM (as PID != 1) and must be shut down by writing a shutdown trigger file + vector & vector_pid=$! + if [ ! -f "/stackable/log/_vector/shutdown" ]; then + mkdir -p /stackable/log/_vector + inotifywait -qq --event create /stackable/log/_vector; + fi + sleep 1 + kill $vector_pid + command: + - /bin/bash + - -x + - -euo + - pipefail + - -c + env: + - name: CLUSTER_NAME + value: opensearch + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: OPENSEARCH_SERVER_LOG_FILE + value: opensearch_server.json + - name: ROLE_GROUP_NAME + value: cluster-manager + - name: ROLE_NAME + value: nodes + - name: VECTOR_AGGREGATOR_ADDRESS + valueFrom: + configMapKeyRef: + key: ADDRESS + name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info + image: oci.stackable.tech/sdp/opensearch:3.1.0-stackable0.0.0-dev + imagePullPolicy: IfNotPresent + name: vector + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 250m + memory: 128Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /stackable/config/vector.yaml + name: config + readOnly: true + subPath: vector.yaml + - mountPath: /stackable/log + name: log +{% endif %} securityContext: fsGroup: 1000 serviceAccount: opensearch-serviceaccount @@ -131,6 +249,13 @@ spec: defaultMode: 420 name: opensearch-nodes-cluster-manager name: config + - configMap: + defaultMode: 420 + name: opensearch-nodes-cluster-manager + name: log-config + - emptyDir: + sizeLimit: 30Mi + name: log - name: security-config secret: defaultMode: 420 @@ -221,6 +346,8 @@ spec: serviceName: opensearch-nodes-data-headless template: metadata: + annotations: + kubectl.kubernetes.io/default-container: opensearch labels: app.kubernetes.io/component: nodes app.kubernetes.io/instance: opensearch @@ -245,8 +372,54 @@ spec: topologyKey: kubernetes.io/hostname weight: 1 containers: - - command: - - {{ test_scenario['values']['opensearch_home'] }}/opensearch-docker-entrypoint.sh + - args: + - |- + + prepare_signal_handlers() + { + unset term_child_pid + unset term_kill_needed + trap 'handle_term_signal' TERM + } + + handle_term_signal() + { + if [ "${term_child_pid}" ]; then + kill -TERM "${term_child_pid}" 2>/dev/null + else + term_kill_needed="yes" + fi + } + + wait_for_termination() + { + set +e + term_child_pid=$1 + if [[ -v term_kill_needed ]]; then + kill -TERM "${term_child_pid}" 2>/dev/null + fi + wait ${term_child_pid} 2>/dev/null + trap - TERM + wait ${term_child_pid} 2>/dev/null + set -e + } + + rm -f /stackable/log/_vector/shutdown + prepare_signal_handlers + if command --search containerdebug >/dev/null 2>&1; then + containerdebug --output=/stackable/log/containerdebug-state.json --loop & + else + echo >&2 "containerdebug not installed; Proceed without it." + fi + ./opensearch-docker-entrypoint.sh & + wait_for_termination $! + mkdir -p /stackable/log/_vector && touch /stackable/log/_vector/shutdown + command: + - /bin/bash + - -x + - -euo + - pipefail + - -c env: - name: DISABLE_INSTALL_DEMO_CONFIG value: "true" @@ -298,16 +471,86 @@ spec: name: config readOnly: true subPath: opensearch.yml + - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/log4j2.properties + name: log-config + readOnly: true + subPath: log4j2.properties - mountPath: {{ test_scenario['values']['opensearch_home'] }}/data name: data - mountPath: /stackable/listener name: listener + - mountPath: /stackable/log + name: log - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch-security name: security-config readOnly: true - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/tls name: tls readOnly: true +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + - args: + - |- + # Vector will ignore SIGTERM (as PID != 1) and must be shut down by writing a shutdown trigger file + vector & vector_pid=$! + if [ ! -f "/stackable/log/_vector/shutdown" ]; then + mkdir -p /stackable/log/_vector + inotifywait -qq --event create /stackable/log/_vector; + fi + sleep 1 + kill $vector_pid + command: + - /bin/bash + - -x + - -euo + - pipefail + - -c + env: + - name: CLUSTER_NAME + value: opensearch + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: OPENSEARCH_SERVER_LOG_FILE + value: opensearch_server.json + - name: ROLE_GROUP_NAME + value: data + - name: ROLE_NAME + value: nodes + - name: VECTOR_AGGREGATOR_ADDRESS + valueFrom: + configMapKeyRef: + key: ADDRESS + name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info + image: oci.stackable.tech/sdp/opensearch:3.1.0-stackable0.0.0-dev + imagePullPolicy: IfNotPresent + name: vector + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 250m + memory: 128Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /stackable/config/vector.yaml + name: config + readOnly: true + subPath: vector.yaml + - mountPath: /stackable/log + name: log +{% endif %} securityContext: fsGroup: 1000 serviceAccount: opensearch-serviceaccount @@ -318,6 +561,13 @@ spec: defaultMode: 420 name: opensearch-nodes-data name: config + - configMap: + defaultMode: 420 + name: opensearch-nodes-data + name: log-config + - emptyDir: + sizeLimit: 30Mi + name: log - name: security-config secret: defaultMode: 420 @@ -610,7 +860,7 @@ metadata: app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster app.kubernetes.io/name: opensearch app.kubernetes.io/role-group: cluster-manager - app.kubernetes.io/version: 3.1.0 + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} stackable.tech/vendor: Stackable name: opensearch-nodes-cluster-manager ownerReferences: @@ -636,7 +886,7 @@ metadata: app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster app.kubernetes.io/name: opensearch app.kubernetes.io/role-group: data - app.kubernetes.io/version: 3.1.0 + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} stackable.tech/vendor: Stackable name: opensearch-nodes-data ownerReferences: diff --git a/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 b/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 index 1ee05fc..56553df 100644 --- a/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 @@ -12,7 +12,14 @@ spec: productVersion: "{{ test_scenario['values']['opensearch'] }}" {% endif %} pullPolicy: IfNotPresent +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + clusterConfig: + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: cluster-manager: config: diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index efad126..6e18a83 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -6,35 +6,30 @@ dimensions: # To use a custom image, add a comma and the full name after the product version, e.g.: # - 3.1.0,oci.stackable.tech/sandbox/opensearch:3.1.0-stackable0.0.0-dev # - 3.1.0,localhost:5000/sdp/opensearch:3.1.0-stackable0.0.0-dev - # - 3.1.0,opensearchproject/opensearch:3.1.0 - - name: openshift - values: - - "false" - name: opensearch_home values: - - /stackable/opensearch # for the Stackable image - # - /usr/share/opensearch # for the official image + - /stackable/opensearch tests: - name: smoke dimensions: - opensearch - - openshift - opensearch_home - name: external-access dimensions: - opensearch - - openshift - opensearch_home + # requires an image with the OpenSearch Prometheus exporter - name: metrics dimensions: - opensearch - - openshift - - opensearch_home - name: ldap dimensions: - opensearch - - openshift - opensearch_home + # requires an image with Vector + - name: logging + dimensions: + - opensearch suites: - name: nightly patch: @@ -52,7 +47,16 @@ suites: - dimensions: - expr: last - dimensions: - - name: openshift - expr: "true" - name: opensearch expr: last + - name: original-image + select: + - smoke + - external-access + - ldap + patch: + - dimensions: + - name: opensearch + expr: 3.1.0,opensearchproject/opensearch:3.1.0 + - name: opensearch_home + expr: /usr/share/opensearch