diff --git a/internal/constants/constants.go b/internal/constants/constants.go index fc5172f08..fd990e8ed 100644 --- a/internal/constants/constants.go +++ b/internal/constants/constants.go @@ -81,6 +81,8 @@ const ( STDOUT = "stdout" STDERR = "stderr" + + InputTypeOption = "inputTypeOption" ) var ExtraNoProxyList = []string{ElasticsearchFQDN} diff --git a/internal/generator/vector/output/factory_test_loki_with_throttle.toml b/internal/generator/vector/output/factory_test_loki_with_throttle.toml index 016f1e5e0..7723a3b03 100644 --- a/internal/generator/vector/output/factory_test_loki_with_throttle.toml +++ b/internal/generator/vector/output/factory_test_loki_with_throttle.toml @@ -38,16 +38,11 @@ codec = "json" except_fields = ["_internal"] [sinks.output_default_loki_apps.labels] -k8s_container_name = "{{kubernetes.container_name}}" -k8s_namespace_name = "{{kubernetes.namespace_name}}" -k8s_node_name = "${VECTOR_SELF_NODE_NAME}" -k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" -openshift_log_type = "{{log_type}}" [sinks.output_default_loki_apps.tls] min_tls_version = "VersionTLS12" diff --git a/internal/generator/vector/output/loki/loki.go b/internal/generator/vector/output/loki/loki.go index 04098cf8b..f7efbe5e7 100644 --- a/internal/generator/vector/output/loki/loki.go +++ b/internal/generator/vector/output/loki/loki.go @@ -2,9 +2,11 @@ package loki import ( "fmt" + "slices" "strings" "github.com/openshift/cluster-logging-operator/internal/api/observability" + "github.com/openshift/cluster-logging-operator/internal/constants" "github.com/openshift/cluster-logging-operator/internal/generator/vector/output/common/tls" @@ -37,38 +39,43 @@ const ( ) var ( - // DefaultLabelKeys contains the log entry keys that are used as Loki stream labels by default. - DefaultLabelKeys = []string{ + // DefaultViaqLabels contains the log entry keys that are used as Loki stream labels by default. + DefaultViaqLabels = []string{ logType, + } + + DefaultOtelLabels = []string{ + otellogType, + } - //container labels + viaqContainerLabels = []string{ lokiLabelKubernetesNamespaceName, lokiLabelKubernetesPodName, lokiLabelKubernetesContainerName, + } - // OTel labels - otellogType, + otelContainerLabels = []string{ otellokiLabelKubernetesNamespaceName, otellokiLabelKubernetesPodName, otellokiLabelKubernetesContainerName, } - containerLabels = []string{ - lokiLabelKubernetesNamespaceName, - lokiLabelKubernetesPodName, - lokiLabelKubernetesContainerName, + RequiredViaqLabels = []string{ + lokiLabelKubernetesHost, } - requiredLabelKeys = []string{ + RequiredOtelLabels = []string{ otellokiLabelKubernetesNodeName, - lokiLabelKubernetesHost, } + LokistackContainerLabels = slices.Concat(viaqContainerLabels, otelContainerLabels) + viaqOtelLabelMap = map[string]string{ logType: otellogType, lokiLabelKubernetesNamespaceName: otellokiLabelKubernetesNamespaceName, lokiLabelKubernetesPodName: otellokiLabelKubernetesPodName, lokiLabelKubernetesContainerName: otellokiLabelKubernetesContainerName, + lokiLabelKubernetesHost: otellokiLabelKubernetesNodeName, } ) @@ -149,34 +156,64 @@ func New(id string, o obs.OutputSpec, inputs []string, secrets observability.Sec } } componentID := vectorhelpers.MakeID(id, "remap") - remapLabelID := vectorhelpers.MakeID(id, "remap_label") + elements := []Element{ + CleanupFields(componentID, inputs), + } + + // Add remap labels based on input type + shouldRemapLabels := false + // Used to determine if OTel labels are needed for lokistack outputs + isLokistackOutput := false - var tenantTemplate Element - sink := Output(id, o, []string{remapLabelID}, "") + // InputTypeOption is set for Lokistack outputs + if input, ok := op[constants.InputTypeOption]; ok { + // Only add remapLabels for application or infrastructure for Lokistack outputs + if input == string(obs.InputTypeApplication) || input == string(obs.InputTypeInfrastructure) { + shouldRemapLabels = true + } + // Always add OTel labels for Lokistack outputs + isLokistackOutput = true + } else { + // For regular Loki outputs, always add remapLabels + shouldRemapLabels = true + } + + if shouldRemapLabels { + remapLabelID := vectorhelpers.MakeID(id, "remap_label") + remapLabels := RemapLabels(remapLabelID, o, []string{componentID}) + elements = append(elements, remapLabels) + componentID = remapLabelID + } + + // Add tenant template if tenant key is configured + var lokiTenantID string if hasTenantKey(o.Loki) { - lokiTenantID := vectorhelpers.MakeID(id, "loki_tenant") - tenantTemplate = commontemplate.TemplateRemap(lokiTenantID, []string{remapLabelID}, o.Loki.TenantKey, lokiTenantID, "Loki Tenant") - sink = Output(id, o, []string{lokiTenantID}, lokiTenantID) + lokiTenantID = vectorhelpers.MakeID(id, "loki_tenant") + tenantTemplate := commontemplate.TemplateRemap(lokiTenantID, []string{componentID}, o.Loki.TenantKey, lokiTenantID, "Loki Tenant") + elements = append(elements, tenantTemplate) + componentID = lokiTenantID } + sink := Output(id, o, []string{componentID}, lokiTenantID) + if strategy != nil { strategy.VisitSink(sink) } - return []Element{ - CleanupFields(componentID, inputs), - RemapLabels(remapLabelID, o, []string{componentID}), - tenantTemplate, + // Add remaining elements + elements = append(elements, sink, common.NewEncoding(id, common.CodecJSON), common.NewAcknowledgments(id, strategy), common.NewBatch(id, strategy), common.NewBuffer(id, strategy), common.NewRequest(id, strategy), - NewLabels(id, o), + NewLabels(id, o, isLokistackOutput), tls.New(id, o.TLS, secrets, op), auth.HTTPAuth(id, o.Loki.Authentication, secrets, op), - } + ) + + return elements } func Output(id string, o obs.OutputSpec, inputs []string, tenant string) *Loki { @@ -189,23 +226,26 @@ func Output(id string, o obs.OutputSpec, inputs []string, tenant string) *Loki { } } -func lokiLabelKeys(l *obs.Loki) []string { - var keys sets.String +func lokiLabelKeys(l *obs.Loki, isLokistackOutput bool) []string { + keys := sets.NewString(RequiredViaqLabels...) if l != nil && len(l.LabelKeys) != 0 { - keys = *sets.NewString(l.LabelKeys...) + keys.Insert(l.LabelKeys...) // Determine which of the OTel labels need to also be added based on spec'd custom labels - keys.Insert(addOtelEquivalentLabels(l.LabelKeys)...) + // Only required for Lokistack outputs + if isLokistackOutput { + keys.Insert(addOtelEquivalentLabels(l.LabelKeys)...) + } } else { - keys = *sets.NewString(DefaultLabelKeys...) + // Add default labels and container labels if none specified for regular Loki outputs + keys.Insert(DefaultViaqLabels...). + Insert(viaqContainerLabels...) } - // Ensure required tags for serialization - keys.Insert(requiredLabelKeys...) return keys.List() } -func lokiLabels(lo *obs.Loki) []LokiLabel { +func lokiLabels(lo *obs.Loki, isLokistackOutput bool) []LokiLabel { ls := []LokiLabel{} - for _, k := range lokiLabelKeys(lo) { + for _, k := range lokiLabelKeys(lo, isLokistackOutput) { r := strings.NewReplacer(".", "_", "/", "_", "\\", "_", "-", "_") name := r.Replace(k) l := LokiLabel{ @@ -257,7 +297,7 @@ func generateCustomLabelValues(value string) string { return fmt.Sprintf("{{%s}}", labelVal) } -func remapLabelsVrl(labels []string) string { +func remapContainerLabelsVrl(labels []string) string { k8sEventLabel := ` if !exists(.%s) { .%s = "" @@ -284,14 +324,14 @@ func RemapLabels(id string, o obs.OutputSpec, inputs []string) Element { return Remap{ ComponentID: id, Inputs: vectorhelpers.MakeInputs(inputs...), - VRL: remapLabelsVrl(containerLabels), + VRL: remapContainerLabelsVrl(viaqContainerLabels), } } -func NewLabels(id string, o obs.OutputSpec) Element { +func NewLabels(id string, o obs.OutputSpec, isLokistackOutput bool) Element { return LokiLabels{ ComponentID: id, - Labels: lokiLabels(o.Loki), + Labels: lokiLabels(o.Loki, isLokistackOutput), } } diff --git a/internal/generator/vector/output/loki/loki_conf_test.go b/internal/generator/vector/output/loki/loki_conf_test.go index 75856d821..5d268461a 100644 --- a/internal/generator/vector/output/loki/loki_conf_test.go +++ b/internal/generator/vector/output/loki/loki_conf_test.go @@ -3,6 +3,7 @@ package loki import ( _ "embed" "fmt" + "slices" "sort" "time" @@ -33,16 +34,16 @@ var _ = Describe("outputLabelConf", func() { Context("#lokiLabelKeys when LabelKeys", func() { Context("are not spec'd", func() { It("should provide a default set of labels including the required ones", func() { - exp := append(DefaultLabelKeys, requiredLabelKeys...) + exp := slices.Concat(DefaultViaqLabels, RequiredViaqLabels, viaqContainerLabels) sort.Strings(exp) - Expect(lokiLabelKeys(loki)).To(BeEquivalentTo(exp)) + Expect(lokiLabelKeys(loki, false)).To(BeEquivalentTo(exp)) }) }) Context("are spec'd", func() { It("should use the ones provided and add the required ones", func() { loki.LabelKeys = []string{"foo"} - exp := append(loki.LabelKeys, requiredLabelKeys...) - Expect(lokiLabelKeys(loki)).To(BeEquivalentTo(exp)) + exp := append(loki.LabelKeys, RequiredViaqLabels...) + Expect(lokiLabelKeys(loki, false)).To(BeEquivalentTo(exp)) }) }) diff --git a/internal/generator/vector/output/loki/loki_test.go b/internal/generator/vector/output/loki/loki_test.go index c0bfd8a61..d27b5ce0e 100644 --- a/internal/generator/vector/output/loki/loki_test.go +++ b/internal/generator/vector/output/loki/loki_test.go @@ -25,7 +25,7 @@ var _ = Describe("Loki generator helpers", func() { lo := &obs.Loki{ LabelKeys: []string{label}, } - labels := lokiLabels(lo) + labels := lokiLabels(lo, false) Expect(labels).To(ContainElement(LokiLabel{ Name: expKey, Value: expValue, diff --git a/internal/generator/vector/output/loki/with_custom_bearer_token.toml b/internal/generator/vector/output/loki/with_custom_bearer_token.toml index 68a0ebe1f..fb33652fe 100644 --- a/internal/generator/vector/output/loki/with_custom_bearer_token.toml +++ b/internal/generator/vector/output/loki/with_custom_bearer_token.toml @@ -32,16 +32,11 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] -k8s_container_name = "{{kubernetes.container_name}}" -k8s_namespace_name = "{{kubernetes.namespace_name}}" -k8s_node_name = "${VECTOR_SELF_NODE_NAME}" -k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" -openshift_log_type = "{{log_type}}" [sinks.loki_receiver.auth] strategy = "bearer" diff --git a/internal/generator/vector/output/loki/with_custom_labels.toml b/internal/generator/vector/output/loki/with_custom_labels.toml index e3ed00089..8985c5851 100644 --- a/internal/generator/vector/output/loki/with_custom_labels.toml +++ b/internal/generator/vector/output/loki/with_custom_labels.toml @@ -32,8 +32,6 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] -k8s_container_name = "{{kubernetes.container_name}}" -k8s_node_name = "${VECTOR_SELF_NODE_NAME}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_labels_app = "{{kubernetes.labels.\"app\"}}" diff --git a/internal/generator/vector/output/loki/with_default_labels.toml b/internal/generator/vector/output/loki/with_default_labels.toml index 309c5130e..a22cd8f6f 100644 --- a/internal/generator/vector/output/loki/with_default_labels.toml +++ b/internal/generator/vector/output/loki/with_default_labels.toml @@ -32,13 +32,8 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] -k8s_container_name = "{{kubernetes.container_name}}" -k8s_namespace_name = "{{kubernetes.namespace_name}}" -k8s_node_name = "${VECTOR_SELF_NODE_NAME}" -k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" -openshift_log_type = "{{log_type}}" diff --git a/internal/generator/vector/output/loki/with_default_tls.toml b/internal/generator/vector/output/loki/with_default_tls.toml index 44d019495..0745ba695 100644 --- a/internal/generator/vector/output/loki/with_default_tls.toml +++ b/internal/generator/vector/output/loki/with_default_tls.toml @@ -32,16 +32,11 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] -k8s_container_name = "{{kubernetes.container_name}}" -k8s_namespace_name = "{{kubernetes.namespace_name}}" -k8s_node_name = "${VECTOR_SELF_NODE_NAME}" -k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" -openshift_log_type = "{{log_type}}" [sinks.loki_receiver.tls] min_tls_version = "VersionTLS12" diff --git a/internal/generator/vector/output/loki/with_insecure.toml b/internal/generator/vector/output/loki/with_insecure.toml index 01e7628a4..595c891a7 100644 --- a/internal/generator/vector/output/loki/with_insecure.toml +++ b/internal/generator/vector/output/loki/with_insecure.toml @@ -33,17 +33,11 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] -k8s_container_name = "{{kubernetes.container_name}}" -k8s_namespace_name = "{{kubernetes.namespace_name}}" -k8s_node_name = "${VECTOR_SELF_NODE_NAME}" -k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" -openshift_log_type = "{{log_type}}" - [sinks.loki_receiver.tls] verify_certificate = false diff --git a/internal/generator/vector/output/loki/with_insecure_nocert.toml b/internal/generator/vector/output/loki/with_insecure_nocert.toml index 46bac4cd3..5b2638613 100644 --- a/internal/generator/vector/output/loki/with_insecure_nocert.toml +++ b/internal/generator/vector/output/loki/with_insecure_nocert.toml @@ -32,17 +32,11 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] -k8s_container_name = "{{kubernetes.container_name}}" -k8s_namespace_name = "{{kubernetes.namespace_name}}" -k8s_node_name = "${VECTOR_SELF_NODE_NAME}" -k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" -openshift_log_type = "{{log_type}}" - [sinks.loki_receiver.tls] verify_certificate = false diff --git a/internal/generator/vector/output/loki/with_sa_token.toml b/internal/generator/vector/output/loki/with_sa_token.toml index 46b70c841..5a9229347 100644 --- a/internal/generator/vector/output/loki/with_sa_token.toml +++ b/internal/generator/vector/output/loki/with_sa_token.toml @@ -32,16 +32,11 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] -k8s_container_name = "{{kubernetes.container_name}}" -k8s_namespace_name = "{{kubernetes.namespace_name}}" -k8s_node_name = "${VECTOR_SELF_NODE_NAME}" -k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" -openshift_log_type = "{{log_type}}" [sinks.loki_receiver.auth] strategy = "bearer" diff --git a/internal/generator/vector/output/loki/with_tenant_id.toml b/internal/generator/vector/output/loki/with_tenant_id.toml index 498f843c0..436b70bc6 100644 --- a/internal/generator/vector/output/loki/with_tenant_id.toml +++ b/internal/generator/vector/output/loki/with_tenant_id.toml @@ -41,13 +41,8 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] -k8s_container_name = "{{kubernetes.container_name}}" -k8s_namespace_name = "{{kubernetes.namespace_name}}" -k8s_node_name = "${VECTOR_SELF_NODE_NAME}" -k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" -openshift_log_type = "{{log_type}}" diff --git a/internal/generator/vector/output/loki/with_tls.toml b/internal/generator/vector/output/loki/with_tls.toml index f612d3002..ac4294cad 100644 --- a/internal/generator/vector/output/loki/with_tls.toml +++ b/internal/generator/vector/output/loki/with_tls.toml @@ -33,16 +33,11 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] -k8s_container_name = "{{kubernetes.container_name}}" -k8s_namespace_name = "{{kubernetes.namespace_name}}" -k8s_node_name = "${VECTOR_SELF_NODE_NAME}" -k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" -openshift_log_type = "{{log_type}}" [sinks.loki_receiver.tls] diff --git a/internal/generator/vector/output/loki/with_tuning.toml b/internal/generator/vector/output/loki/with_tuning.toml index ea4c08b12..4de60a2a9 100644 --- a/internal/generator/vector/output/loki/with_tuning.toml +++ b/internal/generator/vector/output/loki/with_tuning.toml @@ -44,14 +44,9 @@ retry_initial_backoff_secs = 20 retry_max_duration_secs = 35 [sinks.loki_receiver.labels] -k8s_container_name = "{{kubernetes.container_name}}" -k8s_namespace_name = "{{kubernetes.namespace_name}}" -k8s_node_name = "${VECTOR_SELF_NODE_NAME}" -k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" -openshift_log_type = "{{log_type}}" diff --git a/internal/generator/vector/output/loki/with_username_password.toml b/internal/generator/vector/output/loki/with_username_password.toml index f23367609..f77a3049d 100644 --- a/internal/generator/vector/output/loki/with_username_password.toml +++ b/internal/generator/vector/output/loki/with_username_password.toml @@ -32,16 +32,11 @@ codec = "json" except_fields = ["_internal"] [sinks.loki_receiver.labels] -k8s_container_name = "{{kubernetes.container_name}}" -k8s_namespace_name = "{{kubernetes.namespace_name}}" -k8s_node_name = "${VECTOR_SELF_NODE_NAME}" -k8s_pod_name = "{{kubernetes.pod_name}}" kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" kubernetes_namespace_name = "{{kubernetes.namespace_name}}" kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" -openshift_log_type = "{{log_type}}" [sinks.loki_receiver.auth] strategy = "basic" diff --git a/internal/generator/vector/output/lokistack/init_lokiStack.go b/internal/generator/vector/output/lokistack/init_lokiStack.go index 2a15badc5..29cb4761a 100644 --- a/internal/generator/vector/output/lokistack/init_lokiStack.go +++ b/internal/generator/vector/output/lokistack/init_lokiStack.go @@ -33,6 +33,14 @@ func GenerateOutput(outSpec obs.OutputSpec, tenant string) obs.OutputSpec { // GenerateLokiSpec generates and returns a Loki spec for the defined lokistack output func GenerateLokiSpec(ls *obs.LokiStack, tenant string) *obs.Loki { + // Determine the default label keys for the tenant + // Audit logs do not have container labels + defaultLabelKeys := slices.Concat(lokioutput.DefaultViaqLabels, lokioutput.RequiredViaqLabels) + switch tenant { + case string(obs.InputTypeApplication), string(obs.InputTypeInfrastructure): + defaultLabelKeys = slices.Concat(defaultLabelKeys, lokioutput.LokistackContainerLabels) + } + return &obs.Loki{ URLSpec: obs.URLSpec{ URL: lokiStackURL(ls, tenant, false), @@ -41,7 +49,7 @@ func GenerateLokiSpec(ls *obs.LokiStack, tenant string) *obs.Loki { Token: ls.Authentication.Token, }, Tuning: ls.Tuning, - LabelKeys: lokiStackLabelKeysForTenant(ls.LabelKeys, tenant, lokioutput.DefaultLabelKeys), + LabelKeys: lokiStackLabelKeysForTenant(ls.LabelKeys, tenant, defaultLabelKeys), } } @@ -80,7 +88,7 @@ func lokiStackGatewayService(lokiStackServiceName string) string { // A return value of "nil" indicates that the defaults of the Loki output should be used. func lokiStackLabelKeysForTenant(labelKeys *obs.LokiStackLabelKeys, tenant string, defaultKeys []string) []string { if labelKeys == nil { - return nil + return defaultKeys } var tenantConfig *obs.LokiStackTenantLabelKeys diff --git a/internal/generator/vector/output/lokistack/init_lokiStack_test.go b/internal/generator/vector/output/lokistack/init_lokiStack_test.go index c599506a9..6022c9ae7 100644 --- a/internal/generator/vector/output/lokistack/init_lokiStack_test.go +++ b/internal/generator/vector/output/lokistack/init_lokiStack_test.go @@ -1,12 +1,13 @@ package lokistack import ( + "slices" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" obs "github.com/openshift/cluster-logging-operator/api/observability/v1" "github.com/openshift/cluster-logging-operator/internal/constants" lokioutput "github.com/openshift/cluster-logging-operator/internal/generator/vector/output/loki" - "github.com/openshift/cluster-logging-operator/internal/utils/sets" ) var _ = Describe("#GenerateOutput", func() { @@ -64,6 +65,8 @@ var _ = Describe("#GenerateOutput", func() { From: obs.BearerTokenFromServiceAccount, }, }, + LabelKeys: slices.Concat(lokioutput.DefaultViaqLabels, lokioutput.RequiredViaqLabels, + lokioutput.LokistackContainerLabels), }, }, string(obs.InputTypeApplication), @@ -115,7 +118,7 @@ var _ = Describe("#GenerateOutput", func() { From: obs.BearerTokenFromServiceAccount, }, }, - LabelKeys: sets.NewString(lokioutput.DefaultLabelKeys...).Insert("objectRef.apiGroup").List(), + LabelKeys: slices.Concat(lokioutput.RequiredViaqLabels, lokioutput.DefaultViaqLabels, []string{"objectRef.apiGroup"}), }, }, string(obs.InputTypeAudit), @@ -162,7 +165,10 @@ var _ = Describe("#GenerateOutput", func() { "no config", nil, string(obs.InputTypeApplication), - nil, + []string{ + "default_one", + "default_two", + }, ), Entry( "empty slices -> still nil", diff --git a/internal/generator/vector/output/lokistack/lokistack.go b/internal/generator/vector/output/lokistack/lokistack.go index 469de0342..1cd2b87f2 100644 --- a/internal/generator/vector/output/lokistack/lokistack.go +++ b/internal/generator/vector/output/lokistack/lokistack.go @@ -7,6 +7,7 @@ import ( log "github.com/ViaQ/logerr/v2/log/static" obs "github.com/openshift/cluster-logging-operator/api/observability/v1" "github.com/openshift/cluster-logging-operator/internal/api/observability" + "github.com/openshift/cluster-logging-operator/internal/constants" "github.com/openshift/cluster-logging-operator/internal/generator/framework" "github.com/openshift/cluster-logging-operator/internal/generator/vector/elements" vectorhelpers "github.com/openshift/cluster-logging-operator/internal/generator/vector/helpers" @@ -59,6 +60,7 @@ func New(id string, o obs.OutputSpec, inputs []string, secrets observability.Sec inputSources = append(inputSources, observability.ReservedInfrastructureSources.List()...) } op[otlp.OtlpLogSourcesOption] = inputSources + op[constants.InputTypeOption] = inputType confs = append(confs, factory(outputID, migratedOutput, []string{vectorhelpers.MakeRouteInputID(routeID, inputType)}, secrets, strategy, op)...) } return confs diff --git a/internal/generator/vector/output/lokistack/lokistack_viaq.toml b/internal/generator/vector/output/lokistack/lokistack_viaq.toml index 19d906471..06a5afdae 100644 --- a/internal/generator/vector/output/lokistack/lokistack_viaq.toml +++ b/internal/generator/vector/output/lokistack/lokistack_viaq.toml @@ -76,24 +76,9 @@ source = ''' del(.tag) ''' -[transforms.output_default_lokistack_audit_remap_label] -type = "remap" -inputs = ["output_default_lokistack_audit_remap"] -source = ''' - if !exists(.kubernetes.namespace_name) { - .kubernetes.namespace_name = "" - } - if !exists(.kubernetes.pod_name) { - .kubernetes.pod_name = "" - } - if !exists(.kubernetes.container_name) { - .kubernetes.container_name = "" - } -''' - [sinks.output_default_lokistack_audit] type = "loki" -inputs = ["output_default_lokistack_audit_remap_label"] +inputs = ["output_default_lokistack_audit_remap"] endpoint = "https://logging-loki-gateway-http.openshift-logging.svc:8080/api/logs/v1/audit" out_of_order_action = "accept" healthcheck.enabled = false @@ -103,14 +88,8 @@ codec = "json" except_fields = ["_internal"] [sinks.output_default_lokistack_audit.labels] -k8s_container_name = "{{kubernetes.container_name}}" -k8s_namespace_name = "{{kubernetes.namespace_name}}" k8s_node_name = "${VECTOR_SELF_NODE_NAME}" -k8s_pod_name = "{{kubernetes.pod_name}}" -kubernetes_container_name = "{{kubernetes.container_name}}" kubernetes_host = "${VECTOR_SELF_NODE_NAME}" -kubernetes_namespace_name = "{{kubernetes.namespace_name}}" -kubernetes_pod_name = "{{kubernetes.pod_name}}" log_type = "{{log_type}}" openshift_log_type = "{{log_type}}" diff --git a/test/functional/outputs/loki/application_logs_vector_test.go b/test/functional/outputs/loki/application_logs_vector_test.go index 87ea06cf8..82c60e044 100644 --- a/test/functional/outputs/loki/application_logs_vector_test.go +++ b/test/functional/outputs/loki/application_logs_vector_test.go @@ -93,9 +93,6 @@ var _ = Describe("[Functional][Outputs][Loki] Forwarding to Loki", func() { Expect(len(lines)).To(Equal(1)) want := map[string]string{ - "k8s_namespace_name": f.Namespace, - "k8s_pod_name": f.Pod.Name, - "k8s_node_name": f.Pod.Spec.NodeName, "kubernetes_namespace_name": f.Namespace, "kubernetes_pod_name": f.Pod.Name, "kubernetes_labels_app_kubernetes_io_name": myValue, @@ -106,43 +103,37 @@ var _ = Describe("[Functional][Outputs][Loki] Forwarding to Loki", func() { want["service_name"] = "unknown_service" labels := result[0].Stream - Expect(len(labels)).To(Equal(9)) + Expect(len(labels)).To(Equal(6)) Expect(labels).To(BeEquivalentTo(want)) }) - It("should add all otel equivalent default labels when loki.LabelKeys are not defined", func() { + It("should not add all otel equivalent default labels when loki.LabelKeys are not defined", func() { Expect(f.Deploy()).To(BeNil()) now := time.Now() tsNow := functional.CRIOTime(now) msg := functional.NewFullCRIOLogMessage(tsNow, "Present days") Expect(f.WriteMessagesToApplicationLog(msg, 1)).To(Succeed()) - query := fmt.Sprintf(`{openshift_log_type=%q}`, obs.InputTypeApplication) + query := fmt.Sprintf(`{log_type=%q}`, obs.InputTypeApplication) result, err := l.QueryUntil(query, "", 1) Expect(err).To(BeNil()) Expect(result).NotTo(BeNil()) Expect(len(result)).To(Equal(1)) lines := result[0].Lines() Expect(len(lines)).To(Equal(1)) - want := map[string]string{ - "k8s_container_name": f.Pod.Spec.Containers[0].Name, - "k8s_namespace_name": f.Namespace, - "k8s_pod_name": f.Pod.Name, - "k8s_node_name": f.Pod.Spec.NodeName, "kubernetes_container_name": f.Pod.Spec.Containers[0].Name, "kubernetes_namespace_name": f.Namespace, "kubernetes_pod_name": f.Pod.Name, "kubernetes_host": f.Pod.Spec.NodeName, "log_type": string(obs.InputTypeApplication), - "openshift_log_type": string(obs.InputTypeApplication), } // quick fix since unable to disable service_name discovery via functional test arguments - want["service_name"] = f.Pod.Spec.Containers[0].Name + want["service_name"] = "unknown_service" labels := result[0].Stream - Expect(len(labels)).To(Equal(11)) + Expect(len(labels)).To(Equal(6)) Expect(labels).To(BeEquivalentTo(want)) }) }) diff --git a/test/functional/outputs/loki/audit_logs_vector_test.go b/test/functional/outputs/loki/audit_logs_vector_test.go index e2cb24266..933e81b80 100644 --- a/test/functional/outputs/loki/audit_logs_vector_test.go +++ b/test/functional/outputs/loki/audit_logs_vector_test.go @@ -49,10 +49,8 @@ var _ = Describe("[Functional][Outputs][Loki] Forwarding to Loki", func() { Expect(records).To(HaveCap(1), "Exp. the record to be ingested") expLabels := map[string]string{ - "kubernetes_host": f.Pod.Spec.NodeName, - "log_type": string(obs.InputTypeAudit), - "openshift_log_type": string(obs.InputTypeAudit), - "k8s_node_name": f.Pod.Spec.NodeName, + "kubernetes_host": f.Pod.Spec.NodeName, + "log_type": string(obs.InputTypeAudit), } // quick fix since unable to disable service_name discovery via functional test arguments expLabels["service_name"] = "unknown_service"