LOG-7896: Add alert when forwarder sink is generating errors

Author: jcantrill

bundle/manifests/collector_monitoring.coreos.com_v1_prometheusrule.yaml

@@ -20,6 +20,21 @@ spec:
       labels:
         service: collector
         severity: error
+    - alert: ClusterLogForwarderSinkErrorRate
+      annotations:
+        message: The rate of errors detected for pod '{{ $labels.pod }}' owned by
+          ClusterLogForwarder '{{ $labels.namespace }}/{{ $labels.app_kubernetes_io_instance
+          }}' for output '{{ $labels.component_id }}' exceeds 0.
+        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-logging-operator/ClusterLogForwarderSinkErrorRate.md
+        summary: |-
+          The pod '{{ $labels.pod }}' owned by ClusterLogForwarder "{{ $labels.namespace }}/{{ $labels.app_kubernetes_io_instance }}" for output "{{ $labels.component_id }}" is generating the error: "{{ $labels.error_kind }}".
+          This could indicate the output URL is misconfigured, the receiver is unavailable, or there are networking issues for that pod.
+      expr: |
+        sum by (namespace,app_kubernetes_io_instance, pod, component_id, error_kind)(irate(vector_http_client_errors_total[2m])) > 0
+      for: 1m
+      labels:
+        service: clusterlogforwarder
+        severity: critical
     - alert: ClusterLogForwarderDeprecations
       annotations:
         message: The Cluster Logging Operator version {{$labels.version}} includes

config/prometheus/alert_test.yaml

@@ -0,0 +1,23 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: collector-test
+  namespace: openshift-logging
+spec:
+  groups:
+    - name: logging_collector_new.alerts
+      rules:
+        - alert: ClusterLogForwarderSinkErrorRate
+          annotations:
+            description: |-
+              The rate of errors detected for ClusterLogForwarder "{{ $labels.app_kubernetes_io_instance }}" in namespace "{{ $labels.namespace }}" for output "{{ $labels.component_id }}" exceeds 0.
+            summary: |-
+              The ClusterLogForwarder "{{ $labels.namespace }}/{{labels.app_kubernetes_io_instance}}" output "{{ $labels.component_id }}" is generating the error: "{{error_kind}}".
+              This could indicate the output URL is misconfigured or the receiver is unavailable.
+            runbook_url: fixme
+          expr: |
+            sum by (namespace,app_kubernetes_io_instance, component_id, error_kind)(irate(vector_http_client_errors_total[2m])) > 0
+          for: 1m
+          labels:
+            service: collector
+            severity: critical

config/prometheus/collector_alerts.yaml

@@ -20,6 +20,19 @@ spec:
       labels:
         service: collector
         severity: error
+    - alert: ClusterLogForwarderSinkErrorRate
+      annotations:
+        message: "The rate of errors detected for pod '{{ $labels.pod }}' owned by ClusterLogForwarder '{{ $labels.namespace }}/{{ $labels.app_kubernetes_io_instance }}' for output '{{ $labels.component_id }}' exceeds 0."
+        summary: |-
+          The pod '{{ $labels.pod }}' owned by ClusterLogForwarder "{{ $labels.namespace }}/{{ $labels.app_kubernetes_io_instance }}" for output "{{ $labels.component_id }}" is generating the error: "{{ $labels.error_kind }}".
+          This could indicate: the output URL is misconfigured, the receiver is unavailable, or there are networking issues for that pod.
+        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-logging-operator/ClusterLogForwarderSinkErrorRate.md
+      expr: |
+        sum by (namespace,app_kubernetes_io_instance, pod, component_id, error_kind)(irate(vector_http_client_errors_total[2m])) > 0
+      for: 1m
+      labels:
+        service: clusterlogforwarder
+        severity: critical
     - alert: ClusterLogForwarderDeprecations
       annotations:
         message: "The Cluster Logging Operator version {{$labels.version}} includes deprecations to some feature of ClusterLogForwarder."