diff --git a/CHANGELOG.md b/CHANGELOG.md index c20969f9..26735c9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Cortex / Queries: added "Lazy loaded index-headers" and "Index-header lazy load duration" - Cortex / Compactor: added "Tenants compaction progress" - Alerts: added "CortexMemoryMapAreasTooHigh" +* [BUGFIX] Fixed workingset memory panel while rolling out a StatefulSet. #229 ## 1.5.0 / 2020-11-12 diff --git a/cortex-mixin/dashboards/dashboard-utils.libsonnet b/cortex-mixin/dashboards/dashboard-utils.libsonnet index e9824dea..5644af34 100644 --- a/cortex-mixin/dashboards/dashboard-utils.libsonnet +++ b/cortex-mixin/dashboards/dashboard-utils.libsonnet @@ -136,7 +136,9 @@ local utils = import 'mixin-utils/utils.libsonnet'; containerMemoryWorkingSetPanel(title, containerName):: $.panel(title) + $.queryPanel([ - 'sum by(pod) (container_memory_working_set_bytes{%s,container="%s"})' % [$.namespaceMatcher(), containerName], + // We use "max" instead of "sum" otherwise during a rolling update of a statefulset we will end up + // summing the memory of the old pod (whose metric will be stale for 5m) to the new pod. + 'max by(pod) (container_memory_working_set_bytes{%s,container="%s"})' % [$.namespaceMatcher(), containerName], 'min(container_spec_memory_limit_bytes{%s,container="%s"} > 0)' % [$.namespaceMatcher(), containerName], ], ['{{pod}}', 'limit']) + {