Skip to content
This repository was archived by the owner on Apr 28, 2025. It is now read-only.

Commit 86d8a94

Browse files
pracuccipracucci
authored
Merge pull request #239 from grafana/fix-instance-label
Honor per_instance_label in all panels
2 parents fda458b + a798c5a commit 86d8a94

File tree

9 files changed

+57
-26
lines changed

9 files changed

+57
-26
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
* [ENHANCEMENT] Improved blocks storage observability: #237
77
- Cortex / Queries: added bucket index load operations and latency (available only when bucket index is enabled)
88
- Alerts: added "CortexBucketIndexNotUpdated" (bucket index only) and "CortexTenantHasPartialBlocks"
9+
* [BUGFIX] Honor configured `per_instance_label` in all panels. #239
910

1011
## 1.6.0 / 2021-01-05
1112

‎cortex-mixin/dashboards/alertmanager-resources.libsonnet‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,6 @@ local utils = import 'mixin-utils/utils.libsonnet';
22

33
(import 'dashboard-utils.libsonnet') {
44
'alertmanager-resources.json':
5-
local filterNodeDiskByAlertmanager = |||
6-
ignoring(pod) group_right() (label_replace(count by(pod, instance, device) (container_fs_writes_bytes_total{%s,container="alertmanager",device!~".*sda.*"}), "device", "$1", "device", "/dev/(.*)") * 0)
7-
||| % $.namespaceMatcher();
85
($.dashboard('Cortex / Alertmanager Resources') + { uid: '68b66aed90ccab448009089544a8d6c6' })
96
.addClusterSelectorTemplates()
107
.addRow(
@@ -62,13 +59,19 @@ local utils = import 'mixin-utils/utils.libsonnet';
6259
$.row('Disk')
6360
.addPanel(
6461
$.panel('Writes') +
65-
$.queryPanel('sum by(instance, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % filterNodeDiskByAlertmanager, '{{pod}} - {{device}}') +
62+
$.queryPanel(
63+
'sum by(%s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('alertmanager')],
64+
'{{%s}} - {{device}}' % $._config.per_instance_label
65+
) +
6666
$.stack +
6767
{ yaxes: $.yaxes('Bps') },
6868
)
6969
.addPanel(
7070
$.panel('Reads') +
71-
$.queryPanel('sum by(instance, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % filterNodeDiskByAlertmanager, '{{pod}} - {{device}}') +
71+
$.queryPanel(
72+
'sum by(%s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('alertmanager')],
73+
'{{%s}} - {{device}}' % $._config.per_instance_label
74+
) +
7275
$.stack +
7376
{ yaxes: $.yaxes('Bps') },
7477
)

‎cortex-mixin/dashboards/chunks.libsonnet‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
4343
$.row('Flush Stats')
4444
.addPanel(
4545
$.panel('Queue Length') +
46-
$.queryPanel('cortex_ingester_flush_queue_length{%s}' % $.jobMatcher($._config.job_names.ingester), '{{instance}}'),
46+
$.queryPanel('cortex_ingester_flush_queue_length{%s}' % $.jobMatcher($._config.job_names.ingester), '{{%s}}' % $._config.per_instance_label),
4747
)
4848
.addPanel(
4949
$.panel('Flush Rate') +
@@ -88,13 +88,13 @@ local utils = import 'mixin-utils/utils.libsonnet';
8888
$.row('Checkpoint')
8989
.addPanel(
9090
$.panel('Checkpoint creation/deletion / sec') +
91-
$.queryPanel('rate(cortex_ingester_checkpoint_creations_total{%s}[$__rate_interval])' % $.jobMatcher($._config.job_names.ingester), '{{instance}}-creation') +
92-
$.queryPanel('rate(cortex_ingester_checkpoint_deletions_total{%s}[$__rate_interval])' % $.jobMatcher($._config.job_names.ingester), '{{instance}}-deletion'),
91+
$.queryPanel('rate(cortex_ingester_checkpoint_creations_total{%s}[$__rate_interval])' % $.jobMatcher($._config.job_names.ingester), '{{%s}}-creation' % $._config.per_instance_label) +
92+
$.queryPanel('rate(cortex_ingester_checkpoint_deletions_total{%s}[$__rate_interval])' % $.jobMatcher($._config.job_names.ingester), '{{%s}}-deletion' % $._config.per_instance_label),
9393
)
9494
.addPanel(
9595
$.panel('Checkpoint creation/deletion failed / sec') +
96-
$.queryPanel('rate(cortex_ingester_checkpoint_creations_failed_total{%s}[$__rate_interval])' % $.jobMatcher($._config.job_names.ingester), '{{instance}}-creation') +
97-
$.queryPanel('rate(cortex_ingester_checkpoint_deletions_failed_total{%s}[$__rate_interval])' % $.jobMatcher($._config.job_names.ingester), '{{instance}}-deletion'),
96+
$.queryPanel('rate(cortex_ingester_checkpoint_creations_failed_total{%s}[$__rate_interval])' % $.jobMatcher($._config.job_names.ingester), '{{%s}}-creation' % $._config.per_instance_label) +
97+
$.queryPanel('rate(cortex_ingester_checkpoint_deletions_failed_total{%s}[$__rate_interval])' % $.jobMatcher($._config.job_names.ingester), '{{%s}}-deletion' % $._config.per_instance_label),
9898
)
9999
),
100100
}

‎cortex-mixin/dashboards/compactor-resources.libsonnet‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,19 @@ local utils = import 'mixin-utils/utils.libsonnet';
3535
$.row('Disk')
3636
.addPanel(
3737
$.panel('Disk Writes') +
38-
$.queryPanel('sum by(instance, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % $.filterNodeDiskContainer('compactor'), '{{pod}} - {{device}}') +
38+
$.queryPanel(
39+
'sum by(%s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('compactor')],
40+
'{{%s}} - {{device}}' % $._config.per_instance_label
41+
) +
3942
$.stack +
4043
{ yaxes: $.yaxes('Bps') },
4144
)
4245
.addPanel(
4346
$.panel('Disk Reads') +
44-
$.queryPanel('sum by(instance, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % $.filterNodeDiskContainer('compactor'), '{{pod}} - {{device}}') +
47+
$.queryPanel(
48+
'sum by(%s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('compactor')],
49+
'{{%s}} - {{device}}' % $._config.per_instance_label
50+
) +
4551
$.stack +
4652
{ yaxes: $.yaxes('Bps') },
4753
)

‎cortex-mixin/dashboards/compactor.libsonnet‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
3030
cortex_compactor_tenants_processing_failed{%s} +
3131
cortex_compactor_tenants_skipped{%s}
3232
) / cortex_compactor_tenants_discovered{%s}
33-
||| % [$.jobMatcher('compactor'), $.jobMatcher('compactor'), $.jobMatcher('compactor'), $.jobMatcher('compactor')], '{{instance}}') +
33+
||| % [$.jobMatcher('compactor'), $.jobMatcher('compactor'), $.jobMatcher('compactor'), $.jobMatcher('compactor')], '{{%s}}' % $._config.per_instance_label) +
3434
{ yaxes: $.yaxes({ format: 'percentunit', max: 1 }) },
3535
)
3636
)

‎cortex-mixin/dashboards/dashboard-utils.libsonnet‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,10 @@ local utils = import 'mixin-utils/utils.libsonnet';
154154

155155
goHeapInUsePanel(title, jobName)::
156156
$.panel(title) +
157-
$.queryPanel('sum by(instance) (go_memstats_heap_inuse_bytes{%s})' % $.jobMatcher(jobName), '{{instance}}') +
157+
$.queryPanel(
158+
'sum by(%s) (go_memstats_heap_inuse_bytes{%s})' % [$._config.per_instance_label, $.jobMatcher(jobName)],
159+
'{{%s}}' % $._config.per_instance_label
160+
) +
158161
{ yaxes: $.yaxes('bytes') },
159162

160163
// Switches a panel from lines (default) to bars.
@@ -244,6 +247,6 @@ local utils = import 'mixin-utils/utils.libsonnet';
244247

245248
filterNodeDiskContainer(containerName)::
246249
|||
247-
ignoring(pod) group_right() (label_replace(count by(pod, instance, device) (container_fs_writes_bytes_total{%s,container="%s",device!~".*sda.*"}), "device", "$1", "device", "/dev/(.*)") * 0)
248-
||| % [$.namespaceMatcher(), containerName],
250+
ignoring(%s) group_right() (label_replace(count by(%s, device) (container_fs_writes_bytes_total{%s,container="%s",device!~".*sda.*"}), "device", "$1", "device", "/dev/(.*)") * 0)
251+
||| % [$._config.per_instance_label, $._config.per_instance_label, $.namespaceMatcher(), containerName],
249252
}

‎cortex-mixin/dashboards/queries.libsonnet‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
1818
)
1919
.addPanel(
2020
$.panel('Queue Length') +
21-
$.queryPanel('cortex_query_frontend_queue_length{%s}' % $.jobMatcher($._config.job_names.query_frontend), '{{cluster}} / {{namespace}} / {{instance}}'),
21+
$.queryPanel('cortex_query_frontend_queue_length{%s}' % $.jobMatcher($._config.job_names.query_frontend), '{{cluster}} / {{namespace}} / {{%s}}' % $._config.per_instance_label),
2222
)
2323
)
2424
.addRow(
@@ -29,7 +29,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
2929
)
3030
.addPanel(
3131
$.panel('Queue Length') +
32-
$.queryPanel('cortex_query_scheduler_queue_length{%s}' % $.jobMatcher($._config.job_names.query_scheduler), '{{cluster}} / {{namespace}} / {{instance}}'),
32+
$.queryPanel('cortex_query_scheduler_queue_length{%s}' % $.jobMatcher($._config.job_names.query_scheduler), '{{cluster}} / {{namespace}} / {{%s}}' % $._config.per_instance_label),
3333
)
3434
)
3535
.addRow(
@@ -220,7 +220,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
220220
$.row('')
221221
.addPanel(
222222
$.panel('Blocks currently loaded') +
223-
$.queryPanel('cortex_bucket_store_blocks_loaded{component="store-gateway",%s}' % $.jobMatcher($._config.job_names.store_gateway), '{{instance}}')
223+
$.queryPanel('cortex_bucket_store_blocks_loaded{component="store-gateway",%s}' % $.jobMatcher($._config.job_names.store_gateway), '{{%s}}' % $._config.per_instance_label)
224224
)
225225
.addPanel(
226226
$.successFailurePanel(
@@ -242,7 +242,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
242242
$.row('')
243243
.addPanel(
244244
$.panel('Lazy loaded index-headers') +
245-
$.queryPanel('cortex_bucket_store_indexheader_lazy_load_total{%s} - cortex_bucket_store_indexheader_lazy_unload_total{%s}' % [$.jobMatcher($._config.job_names.store_gateway), $.jobMatcher($._config.job_names.store_gateway)], '{{instance}}')
245+
$.queryPanel('cortex_bucket_store_indexheader_lazy_load_total{%s} - cortex_bucket_store_indexheader_lazy_unload_total{%s}' % [$.jobMatcher($._config.job_names.store_gateway), $.jobMatcher($._config.job_names.store_gateway)], '{{%s}}' % $._config.per_instance_label)
246246
)
247247
.addPanel(
248248
$.panel('Index-header lazy load duration') +

‎cortex-mixin/dashboards/reads-resources.libsonnet‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,10 @@ local utils = import 'mixin-utils/utils.libsonnet';
6868
$.row('Ruler')
6969
.addPanel(
7070
$.panel('Rules') +
71-
$.queryPanel('sum by(instance) (cortex_prometheus_rule_group_rules{%s})' % $.jobMatcher($._config.job_names.ruler), '{{instance}}'),
71+
$.queryPanel(
72+
'sum by(%s) (cortex_prometheus_rule_group_rules{%s})' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.ruler)],
73+
'{{%s}}' % $._config.per_instance_label
74+
),
7275
)
7376
.addPanel(
7477
$.containerCPUUsagePanel('CPU', 'ruler'),
@@ -101,13 +104,19 @@ local utils = import 'mixin-utils/utils.libsonnet';
101104
$.row('')
102105
.addPanel(
103106
$.panel('Disk Writes') +
104-
$.queryPanel('sum by(instance, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % $.filterNodeDiskContainer('store-gateway'), '{{pod}} - {{device}}') +
107+
$.queryPanel(
108+
'sum by(%s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('store-gateway')],
109+
'{{%s}} - {{device}}' % $._config.per_instance_label
110+
) +
105111
$.stack +
106112
{ yaxes: $.yaxes('Bps') },
107113
)
108114
.addPanel(
109115
$.panel('Disk Reads') +
110-
$.queryPanel('sum by(instance, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % $.filterNodeDiskContainer('store-gateway'), '{{pod}} - {{device}}') +
116+
$.queryPanel(
117+
'sum by(%s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('store-gateway')],
118+
'{{%s}} - {{device}}' % $._config.per_instance_label
119+
) +
111120
$.stack +
112121
{ yaxes: $.yaxes('Bps') },
113122
)

‎cortex-mixin/dashboards/writes-resources.libsonnet‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,10 @@ local utils = import 'mixin-utils/utils.libsonnet';
3232
$.row('Ingester')
3333
.addPanel(
3434
$.panel('In-memory series') +
35-
$.queryPanel('sum by(instance) (cortex_ingester_memory_series{%s})' % $.jobMatcher($._config.job_names.ingester), '{{instance}}'),
35+
$.queryPanel(
36+
'sum by(%s) (cortex_ingester_memory_series{%s})' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.ingester)],
37+
'{{%s}}' % $._config.per_instance_label
38+
),
3639
)
3740
.addPanel(
3841
$.containerCPUUsagePanel('CPU', 'ingester'),
@@ -51,13 +54,19 @@ local utils = import 'mixin-utils/utils.libsonnet';
5154
$.row('')
5255
.addPanel(
5356
$.panel('Disk Writes') +
54-
$.queryPanel('sum by(instance, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % $.filterNodeDiskContainer('ingester'), '{{pod}} - {{device}}') +
57+
$.queryPanel(
58+
'sum by(%s, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('ingester')],
59+
'{{%s}} - {{device}}' % $._config.per_instance_label
60+
) +
5561
$.stack +
5662
{ yaxes: $.yaxes('Bps') },
5763
)
5864
.addPanel(
5965
$.panel('Disk Reads') +
60-
$.queryPanel('sum by(instance, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % $.filterNodeDiskContainer('ingester'), '{{pod}} - {{device}}') +
66+
$.queryPanel(
67+
'sum by(%s, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + %s' % [$._config.per_instance_label, $.filterNodeDiskContainer('ingester')],
68+
'{{%s}} - {{device}}' % $._config.per_instance_label
69+
) +
6170
$.stack +
6271
{ yaxes: $.yaxes('Bps') },
6372
)

0 commit comments

Comments
 (0)