diff --git a/CHANGELOG.md b/CHANGELOG.md index 530d1163..dc776236 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ * [BUGFIX] Honor configured `per_instance_label` in all panels. #239 * [BUGFIX] `CortexRequestLatency` alert now ignores long-running requests on query-scheduler. #242 * [BUGFIX] Honor configured `job_names` in the "Memory (go heap inuse)" panel. #247 -* [BUGFIX] Fixed ingester "Disk Space Utilization" to include any `ingester.*` PV. #248 +* [BUGFIX] Fixed ingester "Disk Space Utilization" to include any `ingester.*` PV. #249 +* [BUGFIX] Fixed ingester alerts to include any `ingester.*` job. #252 ## 1.6.0 / 2021-01-05 diff --git a/cortex-mixin/alerts/alerts.libsonnet b/cortex-mixin/alerts/alerts.libsonnet index 6cf9f2d5..b2cff6b8 100644 --- a/cortex-mixin/alerts/alerts.libsonnet +++ b/cortex-mixin/alerts/alerts.libsonnet @@ -198,7 +198,7 @@ { alert: 'CortexIngesterRestarts', expr: ||| - changes(process_start_time_seconds{job=~".+(cortex|ingester)"}[30m]) > 1 + changes(process_start_time_seconds{job=~".+(cortex|ingester.*)"}[30m]) > 1 |||, labels: { severity: 'critical', @@ -243,7 +243,7 @@ { alert: 'CortexMemoryMapAreasTooHigh', expr: ||| - process_memory_map_areas{job=~".+(cortex|ingester|store-gateway)"} / process_memory_map_areas_limit{job=~".+(cortex|ingester|store-gateway)"} > 0.8 + process_memory_map_areas{job=~".+(cortex|ingester.*|store-gateway)"} / process_memory_map_areas_limit{job=~".+(cortex|ingester.*|store-gateway)"} > 0.8 |||, 'for': '5m', labels: { @@ -502,7 +502,7 @@ expr: ||| memberlist_client_cluster_members_count != on (%s) group_left - sum by (%s) (up{job=~".+/(distributor|ingester|querier|cortex|ruler)"}) + sum by (%s) (up{job=~".+/(distributor|ingester.*|querier|cortex|ruler)"}) ||| % [$._config.alert_aggregation_labels, $._config.alert_aggregation_labels], 'for': '5m', labels: { diff --git a/cortex-mixin/alerts/blocks.libsonnet b/cortex-mixin/alerts/blocks.libsonnet index 8207746e..12e9160d 100644 --- a/cortex-mixin/alerts/blocks.libsonnet +++ b/cortex-mixin/alerts/blocks.libsonnet @@ -9,9 +9,9 @@ alert: 'CortexIngesterHasNotShippedBlocks', 'for': '15m', expr: ||| - (min by(namespace, instance) (time() - thanos_objstore_bucket_last_successful_upload_time{job=~".+/ingester"}) > 60 * 60 * 4) + (min by(namespace, instance) (time() - thanos_objstore_bucket_last_successful_upload_time{job=~".+/ingester.*"}) > 60 * 60 * 4) and - (max by(namespace, instance) (thanos_objstore_bucket_last_successful_upload_time{job=~".+/ingester"}) > 0) + (max by(namespace, instance) (thanos_objstore_bucket_last_successful_upload_time{job=~".+/ingester.*"}) > 0) and (max by(namespace, instance) (rate(cortex_ingester_ingested_samples_total[4h])) > 0) |||, @@ -28,7 +28,7 @@ alert: 'CortexIngesterHasNotShippedBlocksSinceStart', 'for': '4h', expr: ||| - (max by(namespace, instance) (thanos_objstore_bucket_last_successful_upload_time{job=~".+/ingester"}) == 0) + (max by(namespace, instance) (thanos_objstore_bucket_last_successful_upload_time{job=~".+/ingester.*"}) == 0) and (max by(namespace, instance) (rate(cortex_ingester_ingested_samples_total[4h])) > 0) |||,