LOG-7889: Fields not getting pruned for loki output

internal/constants/constants.go

@@ -81,6 +81,8 @@ const (
 
 	STDOUT = "stdout"
 	STDERR = "stderr"
+
+	InputTypeOption = "inputTypeOption"
 )
 
 var ExtraNoProxyList = []string{ElasticsearchFQDN}

internal/generator/vector/output/factory_test_loki_with_throttle.toml

@@ -38,16 +38,11 @@ codec = "json"
 except_fields = ["_internal"]
 
 [sinks.output_default_loki_apps.labels]
-k8s_container_name = "{{kubernetes.container_name}}"
-k8s_namespace_name = "{{kubernetes.namespace_name}}"
-k8s_node_name = "${VECTOR_SELF_NODE_NAME}"
-k8s_pod_name = "{{kubernetes.pod_name}}"
 kubernetes_container_name = "{{kubernetes.container_name}}"
 kubernetes_host = "${VECTOR_SELF_NODE_NAME}"
 kubernetes_namespace_name = "{{kubernetes.namespace_name}}"
 kubernetes_pod_name = "{{kubernetes.pod_name}}"
 log_type = "{{log_type}}"
-openshift_log_type = "{{log_type}}"
 
 [sinks.output_default_loki_apps.tls]
 min_tls_version = "VersionTLS12"

internal/generator/vector/output/loki/loki.go

@@ -2,9 +2,11 @@ package loki
 
 import (
 	"fmt"
+	"slices"
 	"strings"
 
 	"github.com/openshift/cluster-logging-operator/internal/api/observability"
+	"github.com/openshift/cluster-logging-operator/internal/constants"
 
 	"github.com/openshift/cluster-logging-operator/internal/generator/vector/output/common/tls"
 
@@ -37,38 +39,43 @@ const (
 )
 
 var (
-	// DefaultLabelKeys contains the log entry keys that are used as Loki stream labels by default.
-	DefaultLabelKeys = []string{
+	// DefaultViaqLabels contains the log entry keys that are used as Loki stream labels by default.
+	DefaultViaqLabels = []string{
 		logType,
+	}
+
+	DefaultOtelLabels = []string{
+		otellogType,
+	}
 
-		//container labels
+	viaqContainerLabels = []string{
 		lokiLabelKubernetesNamespaceName,
 		lokiLabelKubernetesPodName,
 		lokiLabelKubernetesContainerName,
+	}
 
-		// OTel labels
-		otellogType,
+	otelContainerLabels = []string{
 		otellokiLabelKubernetesNamespaceName,
 		otellokiLabelKubernetesPodName,
 		otellokiLabelKubernetesContainerName,
 	}
 
-	containerLabels = []string{
-		lokiLabelKubernetesNamespaceName,
-		lokiLabelKubernetesPodName,
-		lokiLabelKubernetesContainerName,
+	RequiredViaqLabels = []string{
+		lokiLabelKubernetesHost,
 	}
 
-	requiredLabelKeys = []string{
+	RequiredOtelLabels = []string{
 		otellokiLabelKubernetesNodeName,
-		lokiLabelKubernetesHost,
 	}
 
+	LokistackContainerLabels = slices.Concat(viaqContainerLabels, otelContainerLabels)
+
 	viaqOtelLabelMap = map[string]string{
 		logType:                          otellogType,
 		lokiLabelKubernetesNamespaceName: otellokiLabelKubernetesNamespaceName,
 		lokiLabelKubernetesPodName:       otellokiLabelKubernetesPodName,
 		lokiLabelKubernetesContainerName: otellokiLabelKubernetesContainerName,
+		lokiLabelKubernetesHost:          otellokiLabelKubernetesNodeName,
 	}
 )
 
@@ -149,34 +156,64 @@ func New(id string, o obs.OutputSpec, inputs []string, secrets observability.Sec
 		}
 	}
 	componentID := vectorhelpers.MakeID(id, "remap")
-	remapLabelID := vectorhelpers.MakeID(id, "remap_label")
+	elements := []Element{
+		CleanupFields(componentID, inputs),
+	}
+
+	// Add remap labels based on input type
+	shouldRemapLabels := false
+	// Used to determine if OTel labels are needed for lokistack outputs
+	isLokistackOutput := false
 
-	var tenantTemplate Element
-	sink := Output(id, o, []string{remapLabelID}, "")
+	// InputTypeOption is set for Lokistack outputs
+	if input, ok := op[constants.InputTypeOption]; ok {
+		// Only add remapLabels for application or infrastructure for Lokistack outputs
+		if input == string(obs.InputTypeApplication) || input == string(obs.InputTypeInfrastructure) {
+			shouldRemapLabels = true
+		}
+		// Always add OTel labels for Lokistack outputs
+		isLokistackOutput = true
+	} else {
+		// For regular Loki outputs, always add remapLabels
+		shouldRemapLabels = true
+	}
+
+	if shouldRemapLabels {
+		remapLabelID := vectorhelpers.MakeID(id, "remap_label")
+		remapLabels := RemapLabels(remapLabelID, o, []string{componentID})
+		elements = append(elements, remapLabels)
+		componentID = remapLabelID
+	}
+
+	// Add tenant template if tenant key is configured
+	var lokiTenantID string
 	if hasTenantKey(o.Loki) {
-		lokiTenantID := vectorhelpers.MakeID(id, "loki_tenant")
-		tenantTemplate = commontemplate.TemplateRemap(lokiTenantID, []string{remapLabelID}, o.Loki.TenantKey, lokiTenantID, "Loki Tenant")
-		sink = Output(id, o, []string{lokiTenantID}, lokiTenantID)
+		lokiTenantID = vectorhelpers.MakeID(id, "loki_tenant")
+		tenantTemplate := commontemplate.TemplateRemap(lokiTenantID, []string{componentID}, o.Loki.TenantKey, lokiTenantID, "Loki Tenant")
+		elements = append(elements, tenantTemplate)
+		componentID = lokiTenantID
 	}
 
+	sink := Output(id, o, []string{componentID}, lokiTenantID)
+
 	if strategy != nil {
 		strategy.VisitSink(sink)
 	}
 
-	return []Element{
-		CleanupFields(componentID, inputs),
-		RemapLabels(remapLabelID, o, []string{componentID}),
-		tenantTemplate,
+	// Add remaining elements
+	elements = append(elements,
 		sink,
 		common.NewEncoding(id, common.CodecJSON),
 		common.NewAcknowledgments(id, strategy),
 		common.NewBatch(id, strategy),
 		common.NewBuffer(id, strategy),
 		common.NewRequest(id, strategy),
-		NewLabels(id, o),
+		NewLabels(id, o, isLokistackOutput),
 		tls.New(id, o.TLS, secrets, op),
 		auth.HTTPAuth(id, o.Loki.Authentication, secrets, op),
-	}
+	)
+
+	return elements
 }
 
 func Output(id string, o obs.OutputSpec, inputs []string, tenant string) *Loki {
@@ -189,23 +226,26 @@ func Output(id string, o obs.OutputSpec, inputs []string, tenant string) *Loki {
 	}
 }
 
-func lokiLabelKeys(l *obs.Loki) []string {
-	var keys sets.String
+func lokiLabelKeys(l *obs.Loki, isLokistackOutput bool) []string {
+	keys := sets.NewString(RequiredViaqLabels...)
 	if l != nil && len(l.LabelKeys) != 0 {
-		keys = *sets.NewString(l.LabelKeys...)
+		keys.Insert(l.LabelKeys...)
 		// Determine which of the OTel labels need to also be added based on spec'd custom labels
-		keys.Insert(addOtelEquivalentLabels(l.LabelKeys)...)
+		// Only required for Lokistack outputs
+		if isLokistackOutput {
+			keys.Insert(addOtelEquivalentLabels(l.LabelKeys)...)
+		}
 	} else {
-		keys = *sets.NewString(DefaultLabelKeys...)
+		// Add default labels and container labels if none specified for regular Loki outputs
+		keys.Insert(DefaultViaqLabels...).
+			Insert(viaqContainerLabels...)
 	}
-	// Ensure required tags for serialization
-	keys.Insert(requiredLabelKeys...)
 	return keys.List()
 }
 
-func lokiLabels(lo *obs.Loki) []LokiLabel {
+func lokiLabels(lo *obs.Loki, isLokistackOutput bool) []LokiLabel {
 	ls := []LokiLabel{}
-	for _, k := range lokiLabelKeys(lo) {
+	for _, k := range lokiLabelKeys(lo, isLokistackOutput) {
 		r := strings.NewReplacer(".", "_", "/", "_", "\\", "_", "-", "_")
 		name := r.Replace(k)
 		l := LokiLabel{
@@ -257,7 +297,7 @@ func generateCustomLabelValues(value string) string {
 	return fmt.Sprintf("{{%s}}", labelVal)
 }
 
-func remapLabelsVrl(labels []string) string {
+func remapContainerLabelsVrl(labels []string) string {
 	k8sEventLabel := `
 if !exists(.%s) {
   .%s = ""
@@ -284,14 +324,14 @@ func RemapLabels(id string, o obs.OutputSpec, inputs []string) Element {
 	return Remap{
 		ComponentID: id,
 		Inputs:      vectorhelpers.MakeInputs(inputs...),
-		VRL:         remapLabelsVrl(containerLabels),
+		VRL:         remapContainerLabelsVrl(viaqContainerLabels),
 	}
 }
 
-func NewLabels(id string, o obs.OutputSpec) Element {
+func NewLabels(id string, o obs.OutputSpec, isLokistackOutput bool) Element {
 	return LokiLabels{
 		ComponentID: id,
-		Labels:      lokiLabels(o.Loki),
+		Labels:      lokiLabels(o.Loki, isLokistackOutput),
 	}
 }
 

internal/generator/vector/output/loki/loki_conf_test.go

@@ -3,6 +3,7 @@ package loki
 import (
 	_ "embed"
 	"fmt"
+	"slices"
 	"sort"
 	"time"
 
@@ -33,16 +34,16 @@ var _ = Describe("outputLabelConf", func() {
 	Context("#lokiLabelKeys when LabelKeys", func() {
 		Context("are not spec'd", func() {
 			It("should provide a default set of labels including the required ones", func() {
-				exp := append(DefaultLabelKeys, requiredLabelKeys...)
+				exp := slices.Concat(DefaultViaqLabels, RequiredViaqLabels, viaqContainerLabels)
 				sort.Strings(exp)
-				Expect(lokiLabelKeys(loki)).To(BeEquivalentTo(exp))
+				Expect(lokiLabelKeys(loki, false)).To(BeEquivalentTo(exp))
 			})
 		})
 		Context("are spec'd", func() {
 			It("should use the ones provided and add the required ones", func() {
 				loki.LabelKeys = []string{"foo"}
-				exp := append(loki.LabelKeys, requiredLabelKeys...)
-				Expect(lokiLabelKeys(loki)).To(BeEquivalentTo(exp))
+				exp := append(loki.LabelKeys, RequiredViaqLabels...)
+				Expect(lokiLabelKeys(loki, false)).To(BeEquivalentTo(exp))
 			})
 		})
 

internal/generator/vector/output/loki/loki_test.go

@@ -25,7 +25,7 @@ var _ = Describe("Loki generator helpers", func() {
 		lo := &obs.Loki{
 			LabelKeys: []string{label},
 		}
-		labels := lokiLabels(lo)
+		labels := lokiLabels(lo, false)
 		Expect(labels).To(ContainElement(LokiLabel{
 			Name:  expKey,
 			Value: expValue,

internal/generator/vector/output/loki/with_custom_bearer_token.toml

@@ -32,16 +32,11 @@ codec = "json"
 except_fields = ["_internal"]
 
 [sinks.loki_receiver.labels]
-k8s_container_name = "{{kubernetes.container_name}}"
-k8s_namespace_name = "{{kubernetes.namespace_name}}"
-k8s_node_name = "${VECTOR_SELF_NODE_NAME}"
-k8s_pod_name = "{{kubernetes.pod_name}}"
 kubernetes_container_name = "{{kubernetes.container_name}}"
 kubernetes_host = "${VECTOR_SELF_NODE_NAME}"
 kubernetes_namespace_name = "{{kubernetes.namespace_name}}"
 kubernetes_pod_name = "{{kubernetes.pod_name}}"
 log_type = "{{log_type}}"
-openshift_log_type = "{{log_type}}"
 
 [sinks.loki_receiver.auth]
 strategy = "bearer"

internal/generator/vector/output/loki/with_custom_labels.toml

@@ -32,8 +32,6 @@ codec = "json"
 except_fields = ["_internal"]
 
 [sinks.loki_receiver.labels]
-k8s_container_name = "{{kubernetes.container_name}}"
-k8s_node_name = "${VECTOR_SELF_NODE_NAME}"
 kubernetes_container_name = "{{kubernetes.container_name}}"
 kubernetes_host = "${VECTOR_SELF_NODE_NAME}"
 kubernetes_labels_app = "{{kubernetes.labels.\"app\"}}"

internal/generator/vector/output/loki/with_default_labels.toml

@@ -32,13 +32,8 @@ codec = "json"
 except_fields = ["_internal"]
 
 [sinks.loki_receiver.labels]
-k8s_container_name = "{{kubernetes.container_name}}"
-k8s_namespace_name = "{{kubernetes.namespace_name}}"
-k8s_node_name = "${VECTOR_SELF_NODE_NAME}"
-k8s_pod_name = "{{kubernetes.pod_name}}"
 kubernetes_container_name = "{{kubernetes.container_name}}"
 kubernetes_host = "${VECTOR_SELF_NODE_NAME}"
 kubernetes_namespace_name = "{{kubernetes.namespace_name}}"
 kubernetes_pod_name = "{{kubernetes.pod_name}}"
 log_type = "{{log_type}}"
-openshift_log_type = "{{log_type}}"

internal/generator/vector/output/loki/with_default_tls.toml

@@ -32,16 +32,11 @@ codec = "json"
 except_fields = ["_internal"]
 
 [sinks.loki_receiver.labels]
-k8s_container_name = "{{kubernetes.container_name}}"
-k8s_namespace_name = "{{kubernetes.namespace_name}}"
-k8s_node_name = "${VECTOR_SELF_NODE_NAME}"
-k8s_pod_name = "{{kubernetes.pod_name}}"
 kubernetes_container_name = "{{kubernetes.container_name}}"
 kubernetes_host = "${VECTOR_SELF_NODE_NAME}"
 kubernetes_namespace_name = "{{kubernetes.namespace_name}}"
 kubernetes_pod_name = "{{kubernetes.pod_name}}"
 log_type = "{{log_type}}"
-openshift_log_type = "{{log_type}}"
 
 [sinks.loki_receiver.tls]
 min_tls_version = "VersionTLS12"

internal/generator/vector/output/loki/with_insecure.toml

@@ -33,17 +33,11 @@ codec = "json"
 except_fields = ["_internal"]
 
 [sinks.loki_receiver.labels]
-k8s_container_name = "{{kubernetes.container_name}}"
-k8s_namespace_name = "{{kubernetes.namespace_name}}"
-k8s_node_name = "${VECTOR_SELF_NODE_NAME}"
-k8s_pod_name = "{{kubernetes.pod_name}}"
 kubernetes_container_name = "{{kubernetes.container_name}}"
 kubernetes_host = "${VECTOR_SELF_NODE_NAME}"
 kubernetes_namespace_name = "{{kubernetes.namespace_name}}"
 kubernetes_pod_name = "{{kubernetes.pod_name}}"
 log_type = "{{log_type}}"
-openshift_log_type = "{{log_type}}"
-
 
 [sinks.loki_receiver.tls]
 verify_certificate = false

internal/generator/vector/output/loki/with_insecure_nocert.toml

@@ -32,17 +32,11 @@ codec = "json"
 except_fields = ["_internal"]
 
 [sinks.loki_receiver.labels]
-k8s_container_name = "{{kubernetes.container_name}}"
-k8s_namespace_name = "{{kubernetes.namespace_name}}"
-k8s_node_name = "${VECTOR_SELF_NODE_NAME}"
-k8s_pod_name = "{{kubernetes.pod_name}}"
 kubernetes_container_name = "{{kubernetes.container_name}}"
 kubernetes_host = "${VECTOR_SELF_NODE_NAME}"
 kubernetes_namespace_name = "{{kubernetes.namespace_name}}"
 kubernetes_pod_name = "{{kubernetes.pod_name}}"
 log_type = "{{log_type}}"
-openshift_log_type = "{{log_type}}"
-
 
 [sinks.loki_receiver.tls]
 verify_certificate = false

internal/generator/vector/output/loki/with_sa_token.toml

@@ -32,16 +32,11 @@ codec = "json"
 except_fields = ["_internal"]
 
 [sinks.loki_receiver.labels]
-k8s_container_name = "{{kubernetes.container_name}}"
-k8s_namespace_name = "{{kubernetes.namespace_name}}"
-k8s_node_name = "${VECTOR_SELF_NODE_NAME}"
-k8s_pod_name = "{{kubernetes.pod_name}}"
 kubernetes_container_name = "{{kubernetes.container_name}}"
 kubernetes_host = "${VECTOR_SELF_NODE_NAME}"
 kubernetes_namespace_name = "{{kubernetes.namespace_name}}"
 kubernetes_pod_name = "{{kubernetes.pod_name}}"
 log_type = "{{log_type}}"
-openshift_log_type = "{{log_type}}"
 
 [sinks.loki_receiver.auth]
 strategy = "bearer"

internal/generator/vector/output/loki/with_tenant_id.toml

@@ -41,13 +41,8 @@ codec = "json"
 except_fields = ["_internal"]
 
 [sinks.loki_receiver.labels]
-k8s_container_name = "{{kubernetes.container_name}}"
-k8s_namespace_name = "{{kubernetes.namespace_name}}"
-k8s_node_name = "${VECTOR_SELF_NODE_NAME}"
-k8s_pod_name = "{{kubernetes.pod_name}}"
 kubernetes_container_name = "{{kubernetes.container_name}}"
 kubernetes_host = "${VECTOR_SELF_NODE_NAME}"
 kubernetes_namespace_name = "{{kubernetes.namespace_name}}"
 kubernetes_pod_name = "{{kubernetes.pod_name}}"
 log_type = "{{log_type}}"
-openshift_log_type = "{{log_type}}"