diff --git a/docker/dev-host/grafana/dashboards/api.json b/docker/dev-host/grafana/dashboards/api.json index 34004bf134..4ad455621b 100644 --- a/docker/dev-host/grafana/dashboards/api.json +++ b/docker/dev-host/grafana/dashboards/api.json @@ -1,1172 +1,1260 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 7, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 24, - "panels": [], - "title": "API", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 26, - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_api_request_pending'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Requests Pending", - "transformations": [ - { - "id": "groupingToMatrix", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 32, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING sum(Count) > 0\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (avg)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 33, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label, bounds, idx\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (p95)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "bounds" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 34, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n Attributes['method'] as method,\n Attributes['path'] as path,\n histogramQuantile(arrayZip(ExplicitBounds, BucketCounts), 0.99) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['watch'] = '0'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, method, path\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (p99)", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\n -- (\n -- SELECT\n -- sum(Value) / $__interval_ms * 1000 as value\n -- FROM otel.otel_metrics_sum\n -- WHERE MetricName = 'rivet_api_request_total'\n -- AND $__conditionalAll(Attributes['path'], $path)\n -- AND $__conditionalAll(Attributes['method'], $method)\n -- AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n -- AND $__timeFilter(TimeUnix)\n -- ) as prev_value\n -- lagInFrame(value, 1, value) OVER (ORDER BY time) as previous_value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_total'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Request Rate", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 28, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['status'], ' (',\n Attributes['error_code'], ')'\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (4xx)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['error_code'], ' (',\n Attributes['status'], ')'\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND Attributes['error_code']-- NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (5xx)", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 35, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200' OR Attributes['status'] LIKE '5%')\n AND Attributes['error_code']-- NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, method, path, status, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "200 vs 5xx (4xx excluded)", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", - "includeAll": true, - "label": "Datacenter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", - "includeAll": true, - "label": "Path", - "multi": true, - "name": "path", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", - "includeAll": true, - "label": "Method", - "multi": true, - "name": "method", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "API", - "uid": "feg8y72gpm2o0c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "API", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 26, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n concat(bounds[idx-1], 's - ', bounds[idx], 's') as label,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, label\nORDER BY Time", + "refId": "A" + } + ], + "title": "Request Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\label" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_pending'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Requests Pending", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": {} + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 32, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING sum(Count) > 0\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (avg)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 33, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING value >= (\n SELECT quantile(0.95)(avg_value)\n FROM (\n SELECT sum(Sum) / sum(Count) as avg_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\n GROUP BY \n $__timeInterval(TimeUnix),\n ResourceAttributes['datacenter_id'],\n Attributes['method'],\n Attributes['path']\n )\n)\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (p95)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 34, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING value >= (\n SELECT quantile(0.99)(avg_value)\n FROM (\n SELECT sum(Sum) / sum(Count) as avg_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\n GROUP BY \n $__timeInterval(TimeUnix),\n ResourceAttributes['datacenter_id'],\n Attributes['method'],\n Attributes['path']\n )\n)\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (p99)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 25, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_total'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Request Rate", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 28, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Name", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['status'], ' (',\n Attributes['error_code'], ')'\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Error Rate (4xx)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 31, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['error_code'], ' (',\n Attributes['status'], ')'\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Error Rate (5xx)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": {} + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 0, + "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "timeseries", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n -- Attributes['method'] as method,\n -- Attributes['path'] as path,\n Attributes['status'] as status,\n -- Attributes['error_code'] as error_code,\n sum(Count) / 30 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200 OK' OR Attributes['status'] LIKE '5%')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, status\nORDER BY time", + "refId": "A" + } + ], + "title": "200 vs 5xx (4xx excluded)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "time": 0, + "value 200 OK": 2, + "value 500 Internal Server Error": 1 + }, + "renameByName": { + "200 OK": "200", + "500 Internal Server Error": "500", + "time": "time", + "value 200 OK": "200", + "value 500 Internal Server Error": "500" + } + } + } + ], + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", + "includeAll": true, + "label": "Datacenter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", + "includeAll": true, + "label": "Path", + "multi": true, + "name": "path", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", + "includeAll": true, + "label": "Method", + "multi": true, + "name": "method", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "API", + "uid": "feg8y72gpm2o0c", + "version": 1, + "weekStart": "" } diff --git a/docker/dev-host/grafana/dashboards/cache.json b/docker/dev-host/grafana/dashboards/cache.json index f5806c749e..222196172e 100644 --- a/docker/dev-host/grafana/dashboards/cache.json +++ b/docker/dev-host/grafana/dashboards/cache.json @@ -1,1176 +1,1174 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 10, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n CASE\n WHEN ResourceAttributes['datacenter_id'] != '' AND ResourceAttributes['server_id'] != '' THEN concat(ResourceAttributes['datacenter_id'], ' - ', ResourceAttributes['server_id'])\n ELSE 'Route Cache Size'\n END as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_route_cache_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Route Cache Size", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 11, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Rate Limiters", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "In-Flight Counters", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 8 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active TCP Connections", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "req/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 8 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_tcp_connection'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "TCP Connection Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 8 - }, - "id": 1, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "TCP Connection Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Proxy Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "req/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_proxy_request'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Proxy Request Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 16 - }, - "id": 9, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Proxy Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 24 - }, - "id": 6, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Resolve Route Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", - "description": "", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", - "description": "", - "includeAll": true, - "label": "Dataceter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", - "description": "", - "includeAll": true, - "label": "Server ID", - "multi": true, - "name": "server_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Rivet Guard", - "uid": "cen785ige8fswd2", - "version": 1, - "weekStart": "" -} \ No newline at end of file + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n CASE\n WHEN ResourceAttributes['datacenter_id'] != '' AND ResourceAttributes['server_id'] != '' THEN concat(ResourceAttributes['datacenter_id'], ' - ', ResourceAttributes['server_id'])\n ELSE 'Route Cache Size'\n END as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_route_cache_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Route Cache Size", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "In-Flight Counters", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_tcp_connection'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "TCP Connection Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 1, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "TCP Connection Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active Proxy Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_proxy_request'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Proxy Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 9, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Proxy Request Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 6, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Resolve Route Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", + "description": "", + "includeAll": true, + "label": "Cluster ID", + "multi": true, + "name": "cluster_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", + "description": "", + "includeAll": true, + "label": "Dataceter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", + "description": "", + "includeAll": true, + "label": "Server ID", + "multi": true, + "name": "server_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Rivet Guard", + "uid": "cen785ige8fswd2", + "version": 1, + "weekStart": "" +} diff --git a/docker/dev-host/grafana/dashboards/futures.json b/docker/dev-host/grafana/dashboards/futures.json index 0c992c4bcb..34d0c27571 100644 --- a/docker/dev-host/grafana/dashboards/futures.json +++ b/docker/dev-host/grafana/dashboards/futures.json @@ -140,12 +140,8 @@ "list": [ { "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": ["All"], + "value": ["$__all"] }, "datasource": { "type": "grafana-clickhouse-datasource", @@ -168,12 +164,8 @@ }, { "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": ["All"], + "value": ["$__all"] }, "datasource": { "type": "grafana-clickhouse-datasource", @@ -193,6 +185,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, @@ -205,4 +209,4 @@ "title": "Futures", "version": 0, "weekStart": "" -} \ No newline at end of file +} diff --git a/docker/dev-host/grafana/dashboards/gasoline.json b/docker/dev-host/grafana/dashboards/gasoline.json index bac8c7b6da..6a2fc3a3d6 100644 --- a/docker/dev-host/grafana/dashboards/gasoline.json +++ b/docker/dev-host/grafana/dashboards/gasoline.json @@ -1,2583 +1,2563 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 3, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Running Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Sleeping Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Dead Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "({{workflow_name}}) {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n Attributes['error_code'] as error_code,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "Dead Workflow Errors", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "__auto", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Workers", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "datacenter_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\datacenter_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{signal_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as signal_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, signal_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Pending Signals", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "signal_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\signal_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size", - "value": "" - }, - "yBuckets": { - "mode": "size", - "scale": { - "log": 2, - "type": "log" - }, - "value": "" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "hide": false, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Signal Receive Lag", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY\n Time, le\nORDER BY\n Time, le", - "refId": "A" - } - ], - "title": "Pull Signals Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "instant": false, - "legendFormat": "{{worker_instance_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Last Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "worker_instance_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\worker_instance_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "instant": false, - "legendFormat": "{{worker_instance_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Last Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "worker_instance_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\worker_instance_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 16, - "panels": [], - "title": "Steps", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 12, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Activity Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 9, - "interval": "15s", - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{activity_name}}: {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['activity_name'] as activity_name,\n Attributes['error_code'] as error_code,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, activity_name, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "Activity Error Rate", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "activity_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\activity_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 21, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Loop Upsert Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ev/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 25, - "interval": "15s", - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "legendFormat": "{{workflow_name}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND Attributes['workflow_name'] LIKE '${workflow_name:regex}'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Loop Events Per Second", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 17, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Message Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 20, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Signal Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 22, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Find Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 76 - }, - "id": 19, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Sub Workflow Dispatch Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", - "includeAll": true, - "label": "Datacenter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", - "includeAll": true, - "label": "Workflow Name", - "multi": true, - "name": "workflow_name", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Gasoline", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Running Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Sleeping Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Dead Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "({{workflow_name}}) {{error_code}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n Attributes['error_code'] as error_code,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name, error_code\nORDER BY time", + "refId": "A" + } + ], + "title": "Dead Workflow Errors", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "__auto", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active Workers", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "datacenter_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\datacenter_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{signal_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as signal_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, signal_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Pending Signals", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "signal_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\signal_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size", + "value": "" + }, + "yBuckets": { + "mode": "size", + "scale": { + "log": 2, + "type": "log" + }, + "value": "" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "hide": false, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Signal Receive Lag", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY\n Time, le\nORDER BY\n Time, le", + "refId": "A" + } + ], + "title": "Pull Signals Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 13, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "instant": false, + "legendFormat": "{{worker_instance_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Last Pull Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "worker_instance_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\worker_instance_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 14, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "instant": false, + "legendFormat": "{{worker_instance_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Last Pull Workflows History Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "worker_instance_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\worker_instance_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Pull Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 24, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Pull Workflows History Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 16, + "panels": [], + "title": "Steps", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 12, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Activity Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{activity_name}}: {{error_code}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['activity_name'] as activity_name,\n Attributes['error_code'] as error_code,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, activity_name, error_code\nORDER BY time", + "refId": "A" + } + ], + "title": "Activity Error Rate", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "activity_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\activity_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 21, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Loop Upsert Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ev/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 25, + "interval": "15s", + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "legendFormat": "{{workflow_name}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND Attributes['workflow_name'] LIKE '${workflow_name:regex}'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Loop Events Per Second", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 17, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Message Send Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 20, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Signal Send Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 22, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Find Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 19, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Sub Workflow Dispatch Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", + "includeAll": true, + "label": "Cluster ID", + "multi": true, + "name": "cluster_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", + "includeAll": true, + "label": "Datacenter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", + "includeAll": true, + "label": "Workflow Name", + "multi": true, + "name": "workflow_name", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Gasoline", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", + "version": 1, + "weekStart": "" } diff --git a/docker/dev-host/grafana/dashboards/guard.json b/docker/dev-host/grafana/dashboards/guard.json index fb657d441b..b4d37655bc 100644 --- a/docker/dev-host/grafana/dashboards/guard.json +++ b/docker/dev-host/grafana/dashboards/guard.json @@ -1134,6 +1134,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, diff --git a/docker/dev-host/grafana/dashboards/tokio.json b/docker/dev-host/grafana/dashboards/tokio.json index efa5e4b49f..45bd84de2a 100644 --- a/docker/dev-host/grafana/dashboards/tokio.json +++ b/docker/dev-host/grafana/dashboards/tokio.json @@ -993,6 +993,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, @@ -1006,4 +1018,4 @@ "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/docker/dev-host/grafana/dashboards/traces.json b/docker/dev-host/grafana/dashboards/traces.json index c4c2c5a62f..54c2d0aefb 100644 --- a/docker/dev-host/grafana/dashboards/traces.json +++ b/docker/dev-host/grafana/dashboards/traces.json @@ -1,979 +1,969 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 21, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 9, - "panels": [], - "title": "Traces", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 0, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "sql", - "format": 0, - "meta": { - "builderOptions": { - "aggregates": [ - { - "aggregateType": "count", - "column": "" - } - ], - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "groupBy": [ - "ServiceName" - ], - "limit": 10000, - "mode": "trend", - "orderBy": [], - "queryType": "timeseries", - "table": "otel_traces" - } - }, - "pluginVersion": "4.0.6", - "queryType": "timeseries", - "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n ServiceName,\r\n count() as ` `\r\nFROM otel.otel_traces\r\nWHERE\r\n ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\nGROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000\r\n", - "refId": "A" - } - ], - "title": "Traces per Service", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Trace ID" - }, - "properties": [ - { - "id": "custom.width", - "value": 77 - }, - { - "id": "links", - "value": [ - { - "title": "__value.raw", - "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${service_name:queryparam}&var-trace_id=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Service Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 130 - }, - { - "id": "links", - "value": [ - { - "title": "__value.raw", - "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${trace_id:queryparam}&var-serviceName=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Duration" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "lcd", - "type": "gauge", - "valueDisplayMode": "text" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 216 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Service Tags" - }, - "properties": [ - { - "id": "custom.inspect", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 248 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 234 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": " " - }, - "properties": [ - { - "id": "custom.width", - "value": 49 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Span Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 140 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Ts" - }, - "properties": [ - { - "id": "custom.width", - "value": 168 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "URI/workflow_id/actor_id" - }, - "properties": [ - { - "id": "custom.width", - "value": 920 - } - ] - } - ] - }, - "gridPos": { - "h": 13, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 4, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Duration" - } - ] - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorType": "sql", - "format": 1, - "meta": { - "builderOptions": { - "columns": [], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "limit": 100, - "mode": "list", - "orderBy": [], - "queryType": "table", - "table": "otel_traces" - } - }, - "pluginVersion": "4.9.0", - "queryType": "table", - "rawSql": "SELECT\r\n (argMin(StatusCode, Timestamp) = 'Error' ? '⚠️' : '') as ` `,\r\n min(Timestamp) as Ts,\r\n TraceId as `Trace ID`,\r\n argMin(ServiceName, Timestamp) as `Service Name`,\r\n argMin(SpanName, Timestamp) as `Span Name`,\r\n argMin(coalesce(NULLIF(SpanAttributes['uri'], ''), NULLIF(SpanAttributes['workflow_id'], ''), SpanAttributes['actor_id']), Timestamp) as `URI/workflow_id/actor_id`,\r\n divide(max(Duration), 1000000) as Duration\r\nFROM otel.otel_traces\r\nWHERE\r\n ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\n AND ServiceName != 'loadgenerator'\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\nGROUP BY TraceId\r\nORDER BY Duration DESC\r\nLIMIT 100\r\n", - "refId": "A" - } - ], - "title": "Traces", - "type": "table" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 17, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 6, - "options": { - "spanFilters": { - "criticalPathOnly": false, - "matchesOnly": false, - "serviceNameOperator": "=", - "spanNameOperator": "=" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "builderOptions": { - "columns": [ - { - "hint": "trace_id", - "name": "TraceId" - }, - { - "hint": "trace_span_id", - "name": "SpanId" - }, - { - "hint": "trace_parent_span_id", - "name": "ParentSpanId" - }, - { - "hint": "trace_service_name", - "name": "ServiceName" - }, - { - "hint": "trace_operation_name", - "name": "SpanName" - }, - { - "hint": "time", - "name": "Timestamp" - }, - { - "hint": "trace_duration_time", - "name": "Duration" - }, - { - "hint": "trace_tags", - "name": "SpanAttributes" - }, - { - "hint": "trace_service_tags", - "name": "ResourceAttributes" - }, - { - "hint": "trace_status_code", - "name": "StatusCode" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "", - "operator": "WITH IN DASHBOARD TIME RANGE", - "type": "datetime" - }, - { - "condition": "AND", - "filterType": "custom", - "hint": "trace_duration_time", - "key": "", - "operator": ">", - "type": "UInt64", - "value": 0 - }, - { - "condition": "AND", - "filterType": "custom", - "hint": "trace_service_name", - "key": "", - "operator": "IS ANYTHING", - "type": "string", - "value": "" - } - ], - "limit": 1000, - "meta": { - "isTraceIdMode": true, - "otelEnabled": true, - "otelVersion": "latest", - "traceDurationUnit": "nanoseconds", - "traceId": "${trace_id}" - }, - "mode": "list", - "orderBy": [ - { - "default": true, - "dir": "DESC", - "hint": "time", - "name": "" - }, - { - "default": true, - "dir": "DESC", - "hint": "trace_duration_time", - "name": "" - } - ], - "queryType": "traces", - "table": "otel_traces" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "builder", - "format": 3, - "meta": { - "builderOptions": { - "columns": [], - "database": "", - "limit": 100, - "mode": "list", - "queryType": "table", - "table": "" - } - }, - "pluginVersion": "4.0.6", - "queryType": "traces", - "rawSql": "WITH\n\t'${trace_id}' as trace_id,\n\t(SELECT min(Start) FROM otel.otel_traces_trace_id_ts WHERE TraceId = trace_id) as trace_start,\n\t(SELECT max(End) + 1 FROM otel.otel_traces_trace_id_ts WHERE TraceId = trace_id) as trace_end\nSELECT\n\tTraceId as traceID,\n\tSpanId as spanID,\n\tParentSpanId as parentSpanID,\n\tServiceName as serviceName,\n\tSpanName as operationName, Timestamp as startTime,\n\tmultiply(Duration, 0.000001) as duration,\n\tarrayMap(key -> map('key', key, 'value', SpanAttributes[key]), mapKeys(SpanAttributes)) as tags,\n\tarrayMap(key -> map('key', key, 'value', ResourceAttributes[key]), mapKeys(ResourceAttributes)) as serviceTags,\n\tarrayMap((name, timestamp, attributes) -> tuple(name, toString(multiply(toUnixTimestamp64Nano(timestamp), 0.000001)), arrayMap( key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(name String, timestamp String, fields Array(Map(String, String))), `Events.Name`, `Events.Timestamp`, `Events.Attributes`) AS logs,\n\tarrayMap((traceID, spanID, attributes) -> tuple(traceID, spanID, arrayMap(key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(traceID String, spanID String, tags Array(Map(String, String))), `Links.TraceId`, `Links.SpanId`, `Links.Attributes`) AS references\nFROM otel.otel_traces\nWHERE\n\ttraceID = trace_id AND startTime >= trace_start AND startTime <= trace_end AND ( Duration > 0 )\nORDER BY Timestamp DESC, Duration DESC\nLIMIT 1000", - "refId": "A" - } - ], - "title": "Trace Details", - "type": "traces" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 24, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "sql", - "format": 0, - "meta": { - "builderOptions": { - "aggregates": [ - { - "aggregateType": "count", - "column": "" - } - ], - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "groupBy": [ - "ServiceName" - ], - "limit": 10000, - "mode": "trend", - "orderBy": [], - "queryType": "timeseries", - "table": "otel_traces" - } - }, - "pluginVersion": "4.0.6", - "queryType": "timeseries", - "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n count(*) as ` `,\r\n ServiceName\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(TraceId IN (${trace_id:singlequote}), $trace_id)\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\n AND StatusCode IN ('Error', 'STATUS_CODE_ERROR')\r\n AND ServiceName != 'loadgenerator' GROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000", - "refId": "A" - } - ], - "title": "Error rates", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 49 - }, - "id": 10, - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 61 - }, - "id": 11, - "options": { - "dedupStrategy": "none", - "enableInfiniteScrolling": false, - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "builderOptions": { - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - }, - { - "hint": "log_level", - "name": "SeverityText", - "type": "LowCardinality(String)" - }, - { - "hint": "log_message", - "name": "Body", - "type": "String" - } - ], - "database": "otel", - "filters": [], - "limit": 1000, - "meta": { - "logMessageLike": "", - "otelVersion": "latest" - }, - "mode": "list", - "orderBy": [], - "queryType": "logs", - "table": "otel_logs" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorType": "builder", - "format": 2, - "pluginVersion": "4.0.6", - "rawSql": "SELECT Timestamp as timestamp, Body as body, SeverityText as level FROM \"otel\".\"otel_logs\" LIMIT 1000", - "refId": "A" - } - ], - "title": "Trace Logs", - "type": "logs" - } - ], - "title": "Logs", - "type": "row" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "ClickHouse", - "value": "ClickHouse" - }, - "includeAll": false, - "label": "ClickHouse instance", - "name": "datasource", - "options": [], - "query": "grafana-clickhouse-datasource", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ServiceName FROM otel.otel_traces", - "includeAll": true, - "label": "Service Name", - "multi": true, - "name": "service_name", - "options": [], - "query": "SELECT DISTINCT ServiceName FROM otel.otel_traces", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", - "includeAll": true, - "label": "Trace ID", - "name": "trace_id", - "options": [], - "query": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Span", - "multi": true, - "name": "span_name", - "options": [], - "query": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "allValue": "'All'", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Ray ID", - "multi": true, - "name": "ray_id", - "options": [], - "query": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "allValue": "'All'", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Workflow ID", - "multi": true, - "name": "workflow_id", - "options": [], - "query": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "America/Los_Angeles", - "title": "Traces", - "uid": "8klBUGfVk", - "version": 2, - "weekStart": "" -} \ No newline at end of file + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 21, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 9, + "panels": [], + "title": "Traces", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "sql", + "format": 0, + "meta": { + "builderOptions": { + "aggregates": [ + { + "aggregateType": "count", + "column": "" + } + ], + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "groupBy": ["ServiceName"], + "limit": 10000, + "mode": "trend", + "orderBy": [], + "queryType": "timeseries", + "table": "otel_traces" + } + }, + "pluginVersion": "4.0.6", + "queryType": "timeseries", + "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n ServiceName,\r\n count() as ` `\r\nFROM otel.otel_traces\r\nWHERE\r\n ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\nGROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000\r\n", + "refId": "A" + } + ], + "title": "Traces per Service", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Trace ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 77 + }, + { + "id": "links", + "value": [ + { + "title": "__value.raw", + "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${service_name:queryparam}&var-trace_id=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Service Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + }, + { + "id": "links", + "value": [ + { + "title": "__value.raw", + "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${trace_id:queryparam}&var-serviceName=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Duration" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge", + "valueDisplayMode": "text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 216 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Service Tags" + }, + "properties": [ + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 248 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 234 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": " " + }, + "properties": [ + { + "id": "custom.width", + "value": 49 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Span Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 140 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Ts" + }, + "properties": [ + { + "id": "custom.width", + "value": 168 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "URI/workflow_id/actor_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 920 + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Duration" + } + ] + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "limit": 100, + "mode": "list", + "orderBy": [], + "queryType": "table", + "table": "otel_traces" + } + }, + "pluginVersion": "4.9.0", + "queryType": "table", + "rawSql": "SELECT\r\n (argMin(StatusCode, Timestamp) = 'Error' ? '⚠️' : '') as ` `,\r\n min(Timestamp) as Ts,\r\n TraceId as `Trace ID`,\r\n argMin(ServiceName, Timestamp) as `Service Name`,\r\n argMin(SpanName, Timestamp) as `Span Name`,\r\n argMin(coalesce(NULLIF(SpanAttributes['uri'], ''), NULLIF(SpanAttributes['workflow_id'], ''), SpanAttributes['actor_id']), Timestamp) as `URI/workflow_id/actor_id`,\r\n divide(max(Duration), 1000000) as Duration\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\n AND ServiceName != 'loadgenerator'\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\nGROUP BY TraceId\r\nORDER BY Duration DESC\r\nLIMIT 100\r\n", + "refId": "A" + } + ], + "title": "Traces", + "type": "table" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 6, + "options": { + "spanFilters": { + "criticalPathOnly": false, + "matchesOnly": false, + "serviceNameOperator": "=", + "spanNameOperator": "=" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "builderOptions": { + "columns": [ + { + "hint": "trace_id", + "name": "TraceId" + }, + { + "hint": "trace_span_id", + "name": "SpanId" + }, + { + "hint": "trace_parent_span_id", + "name": "ParentSpanId" + }, + { + "hint": "trace_service_name", + "name": "ServiceName" + }, + { + "hint": "trace_operation_name", + "name": "SpanName" + }, + { + "hint": "time", + "name": "Timestamp" + }, + { + "hint": "trace_duration_time", + "name": "Duration" + }, + { + "hint": "trace_tags", + "name": "SpanAttributes" + }, + { + "hint": "trace_service_tags", + "name": "ResourceAttributes" + }, + { + "hint": "trace_status_code", + "name": "StatusCode" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "datetime" + }, + { + "condition": "AND", + "filterType": "custom", + "hint": "trace_duration_time", + "key": "", + "operator": ">", + "type": "UInt64", + "value": 0 + }, + { + "condition": "AND", + "filterType": "custom", + "hint": "trace_service_name", + "key": "", + "operator": "IS ANYTHING", + "type": "string", + "value": "" + } + ], + "limit": 1000, + "meta": { + "isTraceIdMode": true, + "otelEnabled": true, + "otelVersion": "latest", + "traceDurationUnit": "nanoseconds", + "traceId": "${trace_id}" + }, + "mode": "list", + "orderBy": [ + { + "default": true, + "dir": "DESC", + "hint": "time", + "name": "" + }, + { + "default": true, + "dir": "DESC", + "hint": "trace_duration_time", + "name": "" + } + ], + "queryType": "traces", + "table": "otel_traces" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "builder", + "format": 3, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 100, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.0.6", + "queryType": "traces", + "rawSql": "WITH\n\t(SELECT min(Start) FROM otel.otel_traces_trace_id_ts WHERE $__conditionalAll(TraceId, $trace_id)) as trace_start,\n\t(SELECT max(End) + 1 FROM otel.otel_traces_trace_id_ts WHERE $__conditionalAll(TraceId, $trace_id)) as trace_end\nSELECT\n\tTraceId as traceID,\n\tSpanId as spanID,\n\tParentSpanId as parentSpanID,\n\tServiceName as serviceName,\n\tSpanName as operationName, Timestamp as startTime,\n\tmultiply(Duration, 0.000001) as duration,\n\tarrayMap(key -> map('key', key, 'value', SpanAttributes[key]), mapKeys(SpanAttributes)) as tags,\n\tarrayMap(key -> map('key', key, 'value', ResourceAttributes[key]), mapKeys(ResourceAttributes)) as serviceTags,\n\tarrayMap((name, timestamp, attributes) -> tuple(name, toString(multiply(toUnixTimestamp64Nano(timestamp), 0.000001)), arrayMap( key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(name String, timestamp String, fields Array(Map(String, String))), `Events.Name`, `Events.Timestamp`, `Events.Attributes`) AS logs,\n\tarrayMap((traceID, spanID, attributes) -> tuple(traceID, spanID, arrayMap(key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(traceID String, spanID String, tags Array(Map(String, String))), `Links.TraceId`, `Links.SpanId`, `Links.Attributes`) AS references\nFROM otel.otel_traces\nWHERE\n\t$__conditionalAll(traceID, $trace_id) AND startTime >= trace_start AND startTime <= trace_end AND ( Duration > 0 )\nORDER BY Timestamp DESC, Duration DESC\nLIMIT 1000", + "refId": "A" + } + ], + "title": "Trace Details", + "type": "traces" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 24, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "sql", + "format": 0, + "meta": { + "builderOptions": { + "aggregates": [ + { + "aggregateType": "count", + "column": "" + } + ], + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "groupBy": ["ServiceName"], + "limit": 10000, + "mode": "trend", + "orderBy": [], + "queryType": "timeseries", + "table": "otel_traces" + } + }, + "pluginVersion": "4.0.6", + "queryType": "timeseries", + "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n count(*) as ` `,\r\n ServiceName\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(TraceId, $trace_id)\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id)\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\n AND StatusCode IN ('Error', 'STATUS_CODE_ERROR')\r\n AND ServiceName != 'loadgenerator' GROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000", + "refId": "A" + } + ], + "title": "Error rates", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 10, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 11, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "builderOptions": { + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + }, + { + "hint": "log_level", + "name": "SeverityText", + "type": "LowCardinality(String)" + }, + { + "hint": "log_message", + "name": "Body", + "type": "String" + } + ], + "database": "otel", + "filters": [], + "limit": 1000, + "meta": { + "logMessageLike": "", + "otelVersion": "latest" + }, + "mode": "list", + "orderBy": [], + "queryType": "logs", + "table": "otel_logs" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorType": "builder", + "format": 2, + "pluginVersion": "4.0.6", + "rawSql": "SELECT Timestamp as timestamp, Body as body, SeverityText as level FROM \"otel\".\"otel_logs\" LIMIT 1000", + "refId": "A" + } + ], + "title": "Trace Logs", + "type": "logs" + } + ], + "title": "Logs", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "ClickHouse", + "value": "ClickHouse" + }, + "includeAll": false, + "label": "ClickHouse instance", + "name": "datasource", + "options": [], + "query": "grafana-clickhouse-datasource", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ServiceName FROM otel.otel_traces", + "includeAll": true, + "label": "Service Name", + "multi": true, + "name": "service_name", + "options": [], + "query": "SELECT DISTINCT ServiceName FROM otel.otel_traces", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", + "includeAll": true, + "label": "Trace ID", + "name": "trace_id", + "options": [], + "query": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Span", + "multi": true, + "name": "span_name", + "options": [], + "query": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "'All'", + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Ray ID", + "multi": true, + "name": "ray_id", + "options": [], + "query": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "'All'", + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Workflow ID", + "multi": true, + "name": "workflow_id", + "options": [], + "query": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "America/Los_Angeles", + "title": "Traces", + "uid": "8klBUGfVk", + "version": 2, + "weekStart": "" +} diff --git a/docker/dev-multidc-multinode/core/grafana/dashboards/api.json b/docker/dev-multidc-multinode/core/grafana/dashboards/api.json index 34004bf134..4ad455621b 100644 --- a/docker/dev-multidc-multinode/core/grafana/dashboards/api.json +++ b/docker/dev-multidc-multinode/core/grafana/dashboards/api.json @@ -1,1172 +1,1260 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 7, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 24, - "panels": [], - "title": "API", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 26, - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_api_request_pending'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Requests Pending", - "transformations": [ - { - "id": "groupingToMatrix", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 32, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING sum(Count) > 0\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (avg)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 33, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label, bounds, idx\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (p95)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "bounds" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 34, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n Attributes['method'] as method,\n Attributes['path'] as path,\n histogramQuantile(arrayZip(ExplicitBounds, BucketCounts), 0.99) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['watch'] = '0'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, method, path\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (p99)", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\n -- (\n -- SELECT\n -- sum(Value) / $__interval_ms * 1000 as value\n -- FROM otel.otel_metrics_sum\n -- WHERE MetricName = 'rivet_api_request_total'\n -- AND $__conditionalAll(Attributes['path'], $path)\n -- AND $__conditionalAll(Attributes['method'], $method)\n -- AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n -- AND $__timeFilter(TimeUnix)\n -- ) as prev_value\n -- lagInFrame(value, 1, value) OVER (ORDER BY time) as previous_value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_total'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Request Rate", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 28, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['status'], ' (',\n Attributes['error_code'], ')'\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (4xx)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['error_code'], ' (',\n Attributes['status'], ')'\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND Attributes['error_code']-- NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (5xx)", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 35, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200' OR Attributes['status'] LIKE '5%')\n AND Attributes['error_code']-- NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, method, path, status, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "200 vs 5xx (4xx excluded)", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", - "includeAll": true, - "label": "Datacenter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", - "includeAll": true, - "label": "Path", - "multi": true, - "name": "path", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", - "includeAll": true, - "label": "Method", - "multi": true, - "name": "method", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "API", - "uid": "feg8y72gpm2o0c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "API", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 26, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n concat(bounds[idx-1], 's - ', bounds[idx], 's') as label,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, label\nORDER BY Time", + "refId": "A" + } + ], + "title": "Request Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\label" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_pending'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Requests Pending", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": {} + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 32, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING sum(Count) > 0\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (avg)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 33, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING value >= (\n SELECT quantile(0.95)(avg_value)\n FROM (\n SELECT sum(Sum) / sum(Count) as avg_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\n GROUP BY \n $__timeInterval(TimeUnix),\n ResourceAttributes['datacenter_id'],\n Attributes['method'],\n Attributes['path']\n )\n)\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (p95)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 34, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING value >= (\n SELECT quantile(0.99)(avg_value)\n FROM (\n SELECT sum(Sum) / sum(Count) as avg_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\n GROUP BY \n $__timeInterval(TimeUnix),\n ResourceAttributes['datacenter_id'],\n Attributes['method'],\n Attributes['path']\n )\n)\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (p99)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 25, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_total'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Request Rate", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 28, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Name", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['status'], ' (',\n Attributes['error_code'], ')'\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Error Rate (4xx)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 31, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['error_code'], ' (',\n Attributes['status'], ')'\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Error Rate (5xx)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": {} + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 0, + "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "timeseries", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n -- Attributes['method'] as method,\n -- Attributes['path'] as path,\n Attributes['status'] as status,\n -- Attributes['error_code'] as error_code,\n sum(Count) / 30 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200 OK' OR Attributes['status'] LIKE '5%')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, status\nORDER BY time", + "refId": "A" + } + ], + "title": "200 vs 5xx (4xx excluded)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "time": 0, + "value 200 OK": 2, + "value 500 Internal Server Error": 1 + }, + "renameByName": { + "200 OK": "200", + "500 Internal Server Error": "500", + "time": "time", + "value 200 OK": "200", + "value 500 Internal Server Error": "500" + } + } + } + ], + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", + "includeAll": true, + "label": "Datacenter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", + "includeAll": true, + "label": "Path", + "multi": true, + "name": "path", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", + "includeAll": true, + "label": "Method", + "multi": true, + "name": "method", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "API", + "uid": "feg8y72gpm2o0c", + "version": 1, + "weekStart": "" } diff --git a/docker/dev-multidc-multinode/core/grafana/dashboards/cache.json b/docker/dev-multidc-multinode/core/grafana/dashboards/cache.json index f5806c749e..222196172e 100644 --- a/docker/dev-multidc-multinode/core/grafana/dashboards/cache.json +++ b/docker/dev-multidc-multinode/core/grafana/dashboards/cache.json @@ -1,1176 +1,1174 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 10, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n CASE\n WHEN ResourceAttributes['datacenter_id'] != '' AND ResourceAttributes['server_id'] != '' THEN concat(ResourceAttributes['datacenter_id'], ' - ', ResourceAttributes['server_id'])\n ELSE 'Route Cache Size'\n END as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_route_cache_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Route Cache Size", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 11, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Rate Limiters", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "In-Flight Counters", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 8 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active TCP Connections", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "req/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 8 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_tcp_connection'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "TCP Connection Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 8 - }, - "id": 1, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "TCP Connection Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Proxy Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "req/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_proxy_request'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Proxy Request Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 16 - }, - "id": 9, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Proxy Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 24 - }, - "id": 6, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Resolve Route Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", - "description": "", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", - "description": "", - "includeAll": true, - "label": "Dataceter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", - "description": "", - "includeAll": true, - "label": "Server ID", - "multi": true, - "name": "server_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Rivet Guard", - "uid": "cen785ige8fswd2", - "version": 1, - "weekStart": "" -} \ No newline at end of file + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n CASE\n WHEN ResourceAttributes['datacenter_id'] != '' AND ResourceAttributes['server_id'] != '' THEN concat(ResourceAttributes['datacenter_id'], ' - ', ResourceAttributes['server_id'])\n ELSE 'Route Cache Size'\n END as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_route_cache_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Route Cache Size", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "In-Flight Counters", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_tcp_connection'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "TCP Connection Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 1, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "TCP Connection Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active Proxy Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_proxy_request'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Proxy Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 9, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Proxy Request Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 6, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Resolve Route Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", + "description": "", + "includeAll": true, + "label": "Cluster ID", + "multi": true, + "name": "cluster_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", + "description": "", + "includeAll": true, + "label": "Dataceter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", + "description": "", + "includeAll": true, + "label": "Server ID", + "multi": true, + "name": "server_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Rivet Guard", + "uid": "cen785ige8fswd2", + "version": 1, + "weekStart": "" +} diff --git a/docker/dev-multidc-multinode/core/grafana/dashboards/futures.json b/docker/dev-multidc-multinode/core/grafana/dashboards/futures.json index 0c992c4bcb..34d0c27571 100644 --- a/docker/dev-multidc-multinode/core/grafana/dashboards/futures.json +++ b/docker/dev-multidc-multinode/core/grafana/dashboards/futures.json @@ -140,12 +140,8 @@ "list": [ { "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": ["All"], + "value": ["$__all"] }, "datasource": { "type": "grafana-clickhouse-datasource", @@ -168,12 +164,8 @@ }, { "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": ["All"], + "value": ["$__all"] }, "datasource": { "type": "grafana-clickhouse-datasource", @@ -193,6 +185,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, @@ -205,4 +209,4 @@ "title": "Futures", "version": 0, "weekStart": "" -} \ No newline at end of file +} diff --git a/docker/dev-multidc-multinode/core/grafana/dashboards/gasoline.json b/docker/dev-multidc-multinode/core/grafana/dashboards/gasoline.json index bac8c7b6da..6a2fc3a3d6 100644 --- a/docker/dev-multidc-multinode/core/grafana/dashboards/gasoline.json +++ b/docker/dev-multidc-multinode/core/grafana/dashboards/gasoline.json @@ -1,2583 +1,2563 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 3, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Running Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Sleeping Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Dead Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "({{workflow_name}}) {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n Attributes['error_code'] as error_code,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "Dead Workflow Errors", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "__auto", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Workers", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "datacenter_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\datacenter_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{signal_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as signal_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, signal_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Pending Signals", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "signal_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\signal_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size", - "value": "" - }, - "yBuckets": { - "mode": "size", - "scale": { - "log": 2, - "type": "log" - }, - "value": "" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "hide": false, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Signal Receive Lag", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY\n Time, le\nORDER BY\n Time, le", - "refId": "A" - } - ], - "title": "Pull Signals Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "instant": false, - "legendFormat": "{{worker_instance_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Last Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "worker_instance_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\worker_instance_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "instant": false, - "legendFormat": "{{worker_instance_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Last Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "worker_instance_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\worker_instance_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 16, - "panels": [], - "title": "Steps", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 12, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Activity Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 9, - "interval": "15s", - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{activity_name}}: {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['activity_name'] as activity_name,\n Attributes['error_code'] as error_code,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, activity_name, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "Activity Error Rate", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "activity_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\activity_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 21, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Loop Upsert Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ev/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 25, - "interval": "15s", - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "legendFormat": "{{workflow_name}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND Attributes['workflow_name'] LIKE '${workflow_name:regex}'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Loop Events Per Second", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 17, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Message Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 20, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Signal Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 22, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Find Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 76 - }, - "id": 19, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Sub Workflow Dispatch Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", - "includeAll": true, - "label": "Datacenter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", - "includeAll": true, - "label": "Workflow Name", - "multi": true, - "name": "workflow_name", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Gasoline", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Running Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Sleeping Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Dead Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "({{workflow_name}}) {{error_code}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n Attributes['error_code'] as error_code,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name, error_code\nORDER BY time", + "refId": "A" + } + ], + "title": "Dead Workflow Errors", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "__auto", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active Workers", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "datacenter_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\datacenter_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{signal_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as signal_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, signal_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Pending Signals", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "signal_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\signal_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size", + "value": "" + }, + "yBuckets": { + "mode": "size", + "scale": { + "log": 2, + "type": "log" + }, + "value": "" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "hide": false, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Signal Receive Lag", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY\n Time, le\nORDER BY\n Time, le", + "refId": "A" + } + ], + "title": "Pull Signals Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 13, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "instant": false, + "legendFormat": "{{worker_instance_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Last Pull Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "worker_instance_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\worker_instance_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 14, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "instant": false, + "legendFormat": "{{worker_instance_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Last Pull Workflows History Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "worker_instance_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\worker_instance_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Pull Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 24, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Pull Workflows History Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 16, + "panels": [], + "title": "Steps", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 12, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Activity Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{activity_name}}: {{error_code}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['activity_name'] as activity_name,\n Attributes['error_code'] as error_code,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, activity_name, error_code\nORDER BY time", + "refId": "A" + } + ], + "title": "Activity Error Rate", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "activity_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\activity_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 21, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Loop Upsert Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ev/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 25, + "interval": "15s", + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "legendFormat": "{{workflow_name}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND Attributes['workflow_name'] LIKE '${workflow_name:regex}'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Loop Events Per Second", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 17, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Message Send Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 20, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Signal Send Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 22, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Find Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 19, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Sub Workflow Dispatch Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", + "includeAll": true, + "label": "Cluster ID", + "multi": true, + "name": "cluster_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", + "includeAll": true, + "label": "Datacenter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", + "includeAll": true, + "label": "Workflow Name", + "multi": true, + "name": "workflow_name", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Gasoline", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", + "version": 1, + "weekStart": "" } diff --git a/docker/dev-multidc-multinode/core/grafana/dashboards/guard.json b/docker/dev-multidc-multinode/core/grafana/dashboards/guard.json index fb657d441b..b4d37655bc 100644 --- a/docker/dev-multidc-multinode/core/grafana/dashboards/guard.json +++ b/docker/dev-multidc-multinode/core/grafana/dashboards/guard.json @@ -1134,6 +1134,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, diff --git a/docker/dev-multidc-multinode/core/grafana/dashboards/tokio.json b/docker/dev-multidc-multinode/core/grafana/dashboards/tokio.json index efa5e4b49f..45bd84de2a 100644 --- a/docker/dev-multidc-multinode/core/grafana/dashboards/tokio.json +++ b/docker/dev-multidc-multinode/core/grafana/dashboards/tokio.json @@ -993,6 +993,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, @@ -1006,4 +1018,4 @@ "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/docker/dev-multidc-multinode/core/grafana/dashboards/traces.json b/docker/dev-multidc-multinode/core/grafana/dashboards/traces.json index c4c2c5a62f..54c2d0aefb 100644 --- a/docker/dev-multidc-multinode/core/grafana/dashboards/traces.json +++ b/docker/dev-multidc-multinode/core/grafana/dashboards/traces.json @@ -1,979 +1,969 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 21, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 9, - "panels": [], - "title": "Traces", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 0, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "sql", - "format": 0, - "meta": { - "builderOptions": { - "aggregates": [ - { - "aggregateType": "count", - "column": "" - } - ], - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "groupBy": [ - "ServiceName" - ], - "limit": 10000, - "mode": "trend", - "orderBy": [], - "queryType": "timeseries", - "table": "otel_traces" - } - }, - "pluginVersion": "4.0.6", - "queryType": "timeseries", - "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n ServiceName,\r\n count() as ` `\r\nFROM otel.otel_traces\r\nWHERE\r\n ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\nGROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000\r\n", - "refId": "A" - } - ], - "title": "Traces per Service", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Trace ID" - }, - "properties": [ - { - "id": "custom.width", - "value": 77 - }, - { - "id": "links", - "value": [ - { - "title": "__value.raw", - "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${service_name:queryparam}&var-trace_id=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Service Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 130 - }, - { - "id": "links", - "value": [ - { - "title": "__value.raw", - "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${trace_id:queryparam}&var-serviceName=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Duration" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "lcd", - "type": "gauge", - "valueDisplayMode": "text" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 216 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Service Tags" - }, - "properties": [ - { - "id": "custom.inspect", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 248 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 234 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": " " - }, - "properties": [ - { - "id": "custom.width", - "value": 49 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Span Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 140 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Ts" - }, - "properties": [ - { - "id": "custom.width", - "value": 168 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "URI/workflow_id/actor_id" - }, - "properties": [ - { - "id": "custom.width", - "value": 920 - } - ] - } - ] - }, - "gridPos": { - "h": 13, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 4, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Duration" - } - ] - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorType": "sql", - "format": 1, - "meta": { - "builderOptions": { - "columns": [], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "limit": 100, - "mode": "list", - "orderBy": [], - "queryType": "table", - "table": "otel_traces" - } - }, - "pluginVersion": "4.9.0", - "queryType": "table", - "rawSql": "SELECT\r\n (argMin(StatusCode, Timestamp) = 'Error' ? '⚠️' : '') as ` `,\r\n min(Timestamp) as Ts,\r\n TraceId as `Trace ID`,\r\n argMin(ServiceName, Timestamp) as `Service Name`,\r\n argMin(SpanName, Timestamp) as `Span Name`,\r\n argMin(coalesce(NULLIF(SpanAttributes['uri'], ''), NULLIF(SpanAttributes['workflow_id'], ''), SpanAttributes['actor_id']), Timestamp) as `URI/workflow_id/actor_id`,\r\n divide(max(Duration), 1000000) as Duration\r\nFROM otel.otel_traces\r\nWHERE\r\n ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\n AND ServiceName != 'loadgenerator'\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\nGROUP BY TraceId\r\nORDER BY Duration DESC\r\nLIMIT 100\r\n", - "refId": "A" - } - ], - "title": "Traces", - "type": "table" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 17, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 6, - "options": { - "spanFilters": { - "criticalPathOnly": false, - "matchesOnly": false, - "serviceNameOperator": "=", - "spanNameOperator": "=" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "builderOptions": { - "columns": [ - { - "hint": "trace_id", - "name": "TraceId" - }, - { - "hint": "trace_span_id", - "name": "SpanId" - }, - { - "hint": "trace_parent_span_id", - "name": "ParentSpanId" - }, - { - "hint": "trace_service_name", - "name": "ServiceName" - }, - { - "hint": "trace_operation_name", - "name": "SpanName" - }, - { - "hint": "time", - "name": "Timestamp" - }, - { - "hint": "trace_duration_time", - "name": "Duration" - }, - { - "hint": "trace_tags", - "name": "SpanAttributes" - }, - { - "hint": "trace_service_tags", - "name": "ResourceAttributes" - }, - { - "hint": "trace_status_code", - "name": "StatusCode" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "", - "operator": "WITH IN DASHBOARD TIME RANGE", - "type": "datetime" - }, - { - "condition": "AND", - "filterType": "custom", - "hint": "trace_duration_time", - "key": "", - "operator": ">", - "type": "UInt64", - "value": 0 - }, - { - "condition": "AND", - "filterType": "custom", - "hint": "trace_service_name", - "key": "", - "operator": "IS ANYTHING", - "type": "string", - "value": "" - } - ], - "limit": 1000, - "meta": { - "isTraceIdMode": true, - "otelEnabled": true, - "otelVersion": "latest", - "traceDurationUnit": "nanoseconds", - "traceId": "${trace_id}" - }, - "mode": "list", - "orderBy": [ - { - "default": true, - "dir": "DESC", - "hint": "time", - "name": "" - }, - { - "default": true, - "dir": "DESC", - "hint": "trace_duration_time", - "name": "" - } - ], - "queryType": "traces", - "table": "otel_traces" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "builder", - "format": 3, - "meta": { - "builderOptions": { - "columns": [], - "database": "", - "limit": 100, - "mode": "list", - "queryType": "table", - "table": "" - } - }, - "pluginVersion": "4.0.6", - "queryType": "traces", - "rawSql": "WITH\n\t'${trace_id}' as trace_id,\n\t(SELECT min(Start) FROM otel.otel_traces_trace_id_ts WHERE TraceId = trace_id) as trace_start,\n\t(SELECT max(End) + 1 FROM otel.otel_traces_trace_id_ts WHERE TraceId = trace_id) as trace_end\nSELECT\n\tTraceId as traceID,\n\tSpanId as spanID,\n\tParentSpanId as parentSpanID,\n\tServiceName as serviceName,\n\tSpanName as operationName, Timestamp as startTime,\n\tmultiply(Duration, 0.000001) as duration,\n\tarrayMap(key -> map('key', key, 'value', SpanAttributes[key]), mapKeys(SpanAttributes)) as tags,\n\tarrayMap(key -> map('key', key, 'value', ResourceAttributes[key]), mapKeys(ResourceAttributes)) as serviceTags,\n\tarrayMap((name, timestamp, attributes) -> tuple(name, toString(multiply(toUnixTimestamp64Nano(timestamp), 0.000001)), arrayMap( key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(name String, timestamp String, fields Array(Map(String, String))), `Events.Name`, `Events.Timestamp`, `Events.Attributes`) AS logs,\n\tarrayMap((traceID, spanID, attributes) -> tuple(traceID, spanID, arrayMap(key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(traceID String, spanID String, tags Array(Map(String, String))), `Links.TraceId`, `Links.SpanId`, `Links.Attributes`) AS references\nFROM otel.otel_traces\nWHERE\n\ttraceID = trace_id AND startTime >= trace_start AND startTime <= trace_end AND ( Duration > 0 )\nORDER BY Timestamp DESC, Duration DESC\nLIMIT 1000", - "refId": "A" - } - ], - "title": "Trace Details", - "type": "traces" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 24, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "sql", - "format": 0, - "meta": { - "builderOptions": { - "aggregates": [ - { - "aggregateType": "count", - "column": "" - } - ], - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "groupBy": [ - "ServiceName" - ], - "limit": 10000, - "mode": "trend", - "orderBy": [], - "queryType": "timeseries", - "table": "otel_traces" - } - }, - "pluginVersion": "4.0.6", - "queryType": "timeseries", - "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n count(*) as ` `,\r\n ServiceName\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(TraceId IN (${trace_id:singlequote}), $trace_id)\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\n AND StatusCode IN ('Error', 'STATUS_CODE_ERROR')\r\n AND ServiceName != 'loadgenerator' GROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000", - "refId": "A" - } - ], - "title": "Error rates", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 49 - }, - "id": 10, - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 61 - }, - "id": 11, - "options": { - "dedupStrategy": "none", - "enableInfiniteScrolling": false, - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "builderOptions": { - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - }, - { - "hint": "log_level", - "name": "SeverityText", - "type": "LowCardinality(String)" - }, - { - "hint": "log_message", - "name": "Body", - "type": "String" - } - ], - "database": "otel", - "filters": [], - "limit": 1000, - "meta": { - "logMessageLike": "", - "otelVersion": "latest" - }, - "mode": "list", - "orderBy": [], - "queryType": "logs", - "table": "otel_logs" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorType": "builder", - "format": 2, - "pluginVersion": "4.0.6", - "rawSql": "SELECT Timestamp as timestamp, Body as body, SeverityText as level FROM \"otel\".\"otel_logs\" LIMIT 1000", - "refId": "A" - } - ], - "title": "Trace Logs", - "type": "logs" - } - ], - "title": "Logs", - "type": "row" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "ClickHouse", - "value": "ClickHouse" - }, - "includeAll": false, - "label": "ClickHouse instance", - "name": "datasource", - "options": [], - "query": "grafana-clickhouse-datasource", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ServiceName FROM otel.otel_traces", - "includeAll": true, - "label": "Service Name", - "multi": true, - "name": "service_name", - "options": [], - "query": "SELECT DISTINCT ServiceName FROM otel.otel_traces", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", - "includeAll": true, - "label": "Trace ID", - "name": "trace_id", - "options": [], - "query": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Span", - "multi": true, - "name": "span_name", - "options": [], - "query": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "allValue": "'All'", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Ray ID", - "multi": true, - "name": "ray_id", - "options": [], - "query": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "allValue": "'All'", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Workflow ID", - "multi": true, - "name": "workflow_id", - "options": [], - "query": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "America/Los_Angeles", - "title": "Traces", - "uid": "8klBUGfVk", - "version": 2, - "weekStart": "" -} \ No newline at end of file + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 21, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 9, + "panels": [], + "title": "Traces", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "sql", + "format": 0, + "meta": { + "builderOptions": { + "aggregates": [ + { + "aggregateType": "count", + "column": "" + } + ], + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "groupBy": ["ServiceName"], + "limit": 10000, + "mode": "trend", + "orderBy": [], + "queryType": "timeseries", + "table": "otel_traces" + } + }, + "pluginVersion": "4.0.6", + "queryType": "timeseries", + "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n ServiceName,\r\n count() as ` `\r\nFROM otel.otel_traces\r\nWHERE\r\n ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\nGROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000\r\n", + "refId": "A" + } + ], + "title": "Traces per Service", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Trace ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 77 + }, + { + "id": "links", + "value": [ + { + "title": "__value.raw", + "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${service_name:queryparam}&var-trace_id=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Service Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + }, + { + "id": "links", + "value": [ + { + "title": "__value.raw", + "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${trace_id:queryparam}&var-serviceName=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Duration" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge", + "valueDisplayMode": "text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 216 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Service Tags" + }, + "properties": [ + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 248 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 234 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": " " + }, + "properties": [ + { + "id": "custom.width", + "value": 49 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Span Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 140 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Ts" + }, + "properties": [ + { + "id": "custom.width", + "value": 168 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "URI/workflow_id/actor_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 920 + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Duration" + } + ] + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "limit": 100, + "mode": "list", + "orderBy": [], + "queryType": "table", + "table": "otel_traces" + } + }, + "pluginVersion": "4.9.0", + "queryType": "table", + "rawSql": "SELECT\r\n (argMin(StatusCode, Timestamp) = 'Error' ? '⚠️' : '') as ` `,\r\n min(Timestamp) as Ts,\r\n TraceId as `Trace ID`,\r\n argMin(ServiceName, Timestamp) as `Service Name`,\r\n argMin(SpanName, Timestamp) as `Span Name`,\r\n argMin(coalesce(NULLIF(SpanAttributes['uri'], ''), NULLIF(SpanAttributes['workflow_id'], ''), SpanAttributes['actor_id']), Timestamp) as `URI/workflow_id/actor_id`,\r\n divide(max(Duration), 1000000) as Duration\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\n AND ServiceName != 'loadgenerator'\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\nGROUP BY TraceId\r\nORDER BY Duration DESC\r\nLIMIT 100\r\n", + "refId": "A" + } + ], + "title": "Traces", + "type": "table" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 6, + "options": { + "spanFilters": { + "criticalPathOnly": false, + "matchesOnly": false, + "serviceNameOperator": "=", + "spanNameOperator": "=" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "builderOptions": { + "columns": [ + { + "hint": "trace_id", + "name": "TraceId" + }, + { + "hint": "trace_span_id", + "name": "SpanId" + }, + { + "hint": "trace_parent_span_id", + "name": "ParentSpanId" + }, + { + "hint": "trace_service_name", + "name": "ServiceName" + }, + { + "hint": "trace_operation_name", + "name": "SpanName" + }, + { + "hint": "time", + "name": "Timestamp" + }, + { + "hint": "trace_duration_time", + "name": "Duration" + }, + { + "hint": "trace_tags", + "name": "SpanAttributes" + }, + { + "hint": "trace_service_tags", + "name": "ResourceAttributes" + }, + { + "hint": "trace_status_code", + "name": "StatusCode" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "datetime" + }, + { + "condition": "AND", + "filterType": "custom", + "hint": "trace_duration_time", + "key": "", + "operator": ">", + "type": "UInt64", + "value": 0 + }, + { + "condition": "AND", + "filterType": "custom", + "hint": "trace_service_name", + "key": "", + "operator": "IS ANYTHING", + "type": "string", + "value": "" + } + ], + "limit": 1000, + "meta": { + "isTraceIdMode": true, + "otelEnabled": true, + "otelVersion": "latest", + "traceDurationUnit": "nanoseconds", + "traceId": "${trace_id}" + }, + "mode": "list", + "orderBy": [ + { + "default": true, + "dir": "DESC", + "hint": "time", + "name": "" + }, + { + "default": true, + "dir": "DESC", + "hint": "trace_duration_time", + "name": "" + } + ], + "queryType": "traces", + "table": "otel_traces" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "builder", + "format": 3, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 100, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.0.6", + "queryType": "traces", + "rawSql": "WITH\n\t(SELECT min(Start) FROM otel.otel_traces_trace_id_ts WHERE $__conditionalAll(TraceId, $trace_id)) as trace_start,\n\t(SELECT max(End) + 1 FROM otel.otel_traces_trace_id_ts WHERE $__conditionalAll(TraceId, $trace_id)) as trace_end\nSELECT\n\tTraceId as traceID,\n\tSpanId as spanID,\n\tParentSpanId as parentSpanID,\n\tServiceName as serviceName,\n\tSpanName as operationName, Timestamp as startTime,\n\tmultiply(Duration, 0.000001) as duration,\n\tarrayMap(key -> map('key', key, 'value', SpanAttributes[key]), mapKeys(SpanAttributes)) as tags,\n\tarrayMap(key -> map('key', key, 'value', ResourceAttributes[key]), mapKeys(ResourceAttributes)) as serviceTags,\n\tarrayMap((name, timestamp, attributes) -> tuple(name, toString(multiply(toUnixTimestamp64Nano(timestamp), 0.000001)), arrayMap( key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(name String, timestamp String, fields Array(Map(String, String))), `Events.Name`, `Events.Timestamp`, `Events.Attributes`) AS logs,\n\tarrayMap((traceID, spanID, attributes) -> tuple(traceID, spanID, arrayMap(key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(traceID String, spanID String, tags Array(Map(String, String))), `Links.TraceId`, `Links.SpanId`, `Links.Attributes`) AS references\nFROM otel.otel_traces\nWHERE\n\t$__conditionalAll(traceID, $trace_id) AND startTime >= trace_start AND startTime <= trace_end AND ( Duration > 0 )\nORDER BY Timestamp DESC, Duration DESC\nLIMIT 1000", + "refId": "A" + } + ], + "title": "Trace Details", + "type": "traces" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 24, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "sql", + "format": 0, + "meta": { + "builderOptions": { + "aggregates": [ + { + "aggregateType": "count", + "column": "" + } + ], + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "groupBy": ["ServiceName"], + "limit": 10000, + "mode": "trend", + "orderBy": [], + "queryType": "timeseries", + "table": "otel_traces" + } + }, + "pluginVersion": "4.0.6", + "queryType": "timeseries", + "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n count(*) as ` `,\r\n ServiceName\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(TraceId, $trace_id)\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id)\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\n AND StatusCode IN ('Error', 'STATUS_CODE_ERROR')\r\n AND ServiceName != 'loadgenerator' GROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000", + "refId": "A" + } + ], + "title": "Error rates", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 10, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 11, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "builderOptions": { + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + }, + { + "hint": "log_level", + "name": "SeverityText", + "type": "LowCardinality(String)" + }, + { + "hint": "log_message", + "name": "Body", + "type": "String" + } + ], + "database": "otel", + "filters": [], + "limit": 1000, + "meta": { + "logMessageLike": "", + "otelVersion": "latest" + }, + "mode": "list", + "orderBy": [], + "queryType": "logs", + "table": "otel_logs" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorType": "builder", + "format": 2, + "pluginVersion": "4.0.6", + "rawSql": "SELECT Timestamp as timestamp, Body as body, SeverityText as level FROM \"otel\".\"otel_logs\" LIMIT 1000", + "refId": "A" + } + ], + "title": "Trace Logs", + "type": "logs" + } + ], + "title": "Logs", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "ClickHouse", + "value": "ClickHouse" + }, + "includeAll": false, + "label": "ClickHouse instance", + "name": "datasource", + "options": [], + "query": "grafana-clickhouse-datasource", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ServiceName FROM otel.otel_traces", + "includeAll": true, + "label": "Service Name", + "multi": true, + "name": "service_name", + "options": [], + "query": "SELECT DISTINCT ServiceName FROM otel.otel_traces", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", + "includeAll": true, + "label": "Trace ID", + "name": "trace_id", + "options": [], + "query": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Span", + "multi": true, + "name": "span_name", + "options": [], + "query": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "'All'", + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Ray ID", + "multi": true, + "name": "ray_id", + "options": [], + "query": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "'All'", + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Workflow ID", + "multi": true, + "name": "workflow_id", + "options": [], + "query": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "America/Los_Angeles", + "title": "Traces", + "uid": "8klBUGfVk", + "version": 2, + "weekStart": "" +} diff --git a/docker/dev-multidc/core/grafana/dashboards/api.json b/docker/dev-multidc/core/grafana/dashboards/api.json index 34004bf134..4ad455621b 100644 --- a/docker/dev-multidc/core/grafana/dashboards/api.json +++ b/docker/dev-multidc/core/grafana/dashboards/api.json @@ -1,1172 +1,1260 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 7, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 24, - "panels": [], - "title": "API", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 26, - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_api_request_pending'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Requests Pending", - "transformations": [ - { - "id": "groupingToMatrix", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 32, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING sum(Count) > 0\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (avg)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 33, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label, bounds, idx\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (p95)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "bounds" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 34, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n Attributes['method'] as method,\n Attributes['path'] as path,\n histogramQuantile(arrayZip(ExplicitBounds, BucketCounts), 0.99) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['watch'] = '0'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, method, path\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (p99)", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\n -- (\n -- SELECT\n -- sum(Value) / $__interval_ms * 1000 as value\n -- FROM otel.otel_metrics_sum\n -- WHERE MetricName = 'rivet_api_request_total'\n -- AND $__conditionalAll(Attributes['path'], $path)\n -- AND $__conditionalAll(Attributes['method'], $method)\n -- AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n -- AND $__timeFilter(TimeUnix)\n -- ) as prev_value\n -- lagInFrame(value, 1, value) OVER (ORDER BY time) as previous_value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_total'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Request Rate", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 28, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['status'], ' (',\n Attributes['error_code'], ')'\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (4xx)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['error_code'], ' (',\n Attributes['status'], ')'\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND Attributes['error_code']-- NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (5xx)", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 35, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200' OR Attributes['status'] LIKE '5%')\n AND Attributes['error_code']-- NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, method, path, status, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "200 vs 5xx (4xx excluded)", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", - "includeAll": true, - "label": "Datacenter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", - "includeAll": true, - "label": "Path", - "multi": true, - "name": "path", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", - "includeAll": true, - "label": "Method", - "multi": true, - "name": "method", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "API", - "uid": "feg8y72gpm2o0c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "API", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 26, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n concat(bounds[idx-1], 's - ', bounds[idx], 's') as label,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, label\nORDER BY Time", + "refId": "A" + } + ], + "title": "Request Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\label" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_pending'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Requests Pending", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": {} + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 32, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING sum(Count) > 0\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (avg)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 33, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING value >= (\n SELECT quantile(0.95)(avg_value)\n FROM (\n SELECT sum(Sum) / sum(Count) as avg_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\n GROUP BY \n $__timeInterval(TimeUnix),\n ResourceAttributes['datacenter_id'],\n Attributes['method'],\n Attributes['path']\n )\n)\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (p95)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 34, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING value >= (\n SELECT quantile(0.99)(avg_value)\n FROM (\n SELECT sum(Sum) / sum(Count) as avg_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\n GROUP BY \n $__timeInterval(TimeUnix),\n ResourceAttributes['datacenter_id'],\n Attributes['method'],\n Attributes['path']\n )\n)\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (p99)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 25, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_total'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Request Rate", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 28, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Name", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['status'], ' (',\n Attributes['error_code'], ')'\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Error Rate (4xx)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 31, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['error_code'], ' (',\n Attributes['status'], ')'\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Error Rate (5xx)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": {} + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 0, + "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "timeseries", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n -- Attributes['method'] as method,\n -- Attributes['path'] as path,\n Attributes['status'] as status,\n -- Attributes['error_code'] as error_code,\n sum(Count) / 30 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200 OK' OR Attributes['status'] LIKE '5%')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, status\nORDER BY time", + "refId": "A" + } + ], + "title": "200 vs 5xx (4xx excluded)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "time": 0, + "value 200 OK": 2, + "value 500 Internal Server Error": 1 + }, + "renameByName": { + "200 OK": "200", + "500 Internal Server Error": "500", + "time": "time", + "value 200 OK": "200", + "value 500 Internal Server Error": "500" + } + } + } + ], + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", + "includeAll": true, + "label": "Datacenter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", + "includeAll": true, + "label": "Path", + "multi": true, + "name": "path", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", + "includeAll": true, + "label": "Method", + "multi": true, + "name": "method", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "API", + "uid": "feg8y72gpm2o0c", + "version": 1, + "weekStart": "" } diff --git a/docker/dev-multidc/core/grafana/dashboards/cache.json b/docker/dev-multidc/core/grafana/dashboards/cache.json index f5806c749e..222196172e 100644 --- a/docker/dev-multidc/core/grafana/dashboards/cache.json +++ b/docker/dev-multidc/core/grafana/dashboards/cache.json @@ -1,1176 +1,1174 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 10, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n CASE\n WHEN ResourceAttributes['datacenter_id'] != '' AND ResourceAttributes['server_id'] != '' THEN concat(ResourceAttributes['datacenter_id'], ' - ', ResourceAttributes['server_id'])\n ELSE 'Route Cache Size'\n END as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_route_cache_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Route Cache Size", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 11, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Rate Limiters", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "In-Flight Counters", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 8 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active TCP Connections", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "req/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 8 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_tcp_connection'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "TCP Connection Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 8 - }, - "id": 1, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "TCP Connection Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Proxy Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "req/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_proxy_request'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Proxy Request Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 16 - }, - "id": 9, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Proxy Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 24 - }, - "id": 6, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Resolve Route Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", - "description": "", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", - "description": "", - "includeAll": true, - "label": "Dataceter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", - "description": "", - "includeAll": true, - "label": "Server ID", - "multi": true, - "name": "server_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Rivet Guard", - "uid": "cen785ige8fswd2", - "version": 1, - "weekStart": "" -} \ No newline at end of file + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n CASE\n WHEN ResourceAttributes['datacenter_id'] != '' AND ResourceAttributes['server_id'] != '' THEN concat(ResourceAttributes['datacenter_id'], ' - ', ResourceAttributes['server_id'])\n ELSE 'Route Cache Size'\n END as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_route_cache_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Route Cache Size", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "In-Flight Counters", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_tcp_connection'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "TCP Connection Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 1, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "TCP Connection Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active Proxy Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_proxy_request'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Proxy Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 9, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Proxy Request Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 6, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Resolve Route Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", + "description": "", + "includeAll": true, + "label": "Cluster ID", + "multi": true, + "name": "cluster_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", + "description": "", + "includeAll": true, + "label": "Dataceter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", + "description": "", + "includeAll": true, + "label": "Server ID", + "multi": true, + "name": "server_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Rivet Guard", + "uid": "cen785ige8fswd2", + "version": 1, + "weekStart": "" +} diff --git a/docker/dev-multidc/core/grafana/dashboards/futures.json b/docker/dev-multidc/core/grafana/dashboards/futures.json index 0c992c4bcb..34d0c27571 100644 --- a/docker/dev-multidc/core/grafana/dashboards/futures.json +++ b/docker/dev-multidc/core/grafana/dashboards/futures.json @@ -140,12 +140,8 @@ "list": [ { "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": ["All"], + "value": ["$__all"] }, "datasource": { "type": "grafana-clickhouse-datasource", @@ -168,12 +164,8 @@ }, { "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": ["All"], + "value": ["$__all"] }, "datasource": { "type": "grafana-clickhouse-datasource", @@ -193,6 +185,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, @@ -205,4 +209,4 @@ "title": "Futures", "version": 0, "weekStart": "" -} \ No newline at end of file +} diff --git a/docker/dev-multidc/core/grafana/dashboards/gasoline.json b/docker/dev-multidc/core/grafana/dashboards/gasoline.json index bac8c7b6da..6a2fc3a3d6 100644 --- a/docker/dev-multidc/core/grafana/dashboards/gasoline.json +++ b/docker/dev-multidc/core/grafana/dashboards/gasoline.json @@ -1,2583 +1,2563 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 3, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Running Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Sleeping Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Dead Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "({{workflow_name}}) {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n Attributes['error_code'] as error_code,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "Dead Workflow Errors", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "__auto", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Workers", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "datacenter_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\datacenter_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{signal_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as signal_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, signal_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Pending Signals", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "signal_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\signal_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size", - "value": "" - }, - "yBuckets": { - "mode": "size", - "scale": { - "log": 2, - "type": "log" - }, - "value": "" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "hide": false, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Signal Receive Lag", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY\n Time, le\nORDER BY\n Time, le", - "refId": "A" - } - ], - "title": "Pull Signals Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "instant": false, - "legendFormat": "{{worker_instance_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Last Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "worker_instance_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\worker_instance_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "instant": false, - "legendFormat": "{{worker_instance_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Last Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "worker_instance_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\worker_instance_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 16, - "panels": [], - "title": "Steps", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 12, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Activity Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 9, - "interval": "15s", - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{activity_name}}: {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['activity_name'] as activity_name,\n Attributes['error_code'] as error_code,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, activity_name, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "Activity Error Rate", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "activity_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\activity_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 21, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Loop Upsert Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ev/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 25, - "interval": "15s", - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "legendFormat": "{{workflow_name}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND Attributes['workflow_name'] LIKE '${workflow_name:regex}'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Loop Events Per Second", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 17, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Message Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 20, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Signal Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 22, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Find Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 76 - }, - "id": 19, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Sub Workflow Dispatch Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", - "includeAll": true, - "label": "Datacenter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", - "includeAll": true, - "label": "Workflow Name", - "multi": true, - "name": "workflow_name", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Gasoline", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Running Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Sleeping Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Dead Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "({{workflow_name}}) {{error_code}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n Attributes['error_code'] as error_code,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name, error_code\nORDER BY time", + "refId": "A" + } + ], + "title": "Dead Workflow Errors", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "__auto", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active Workers", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "datacenter_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\datacenter_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{signal_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as signal_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, signal_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Pending Signals", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "signal_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\signal_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size", + "value": "" + }, + "yBuckets": { + "mode": "size", + "scale": { + "log": 2, + "type": "log" + }, + "value": "" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "hide": false, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Signal Receive Lag", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY\n Time, le\nORDER BY\n Time, le", + "refId": "A" + } + ], + "title": "Pull Signals Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 13, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "instant": false, + "legendFormat": "{{worker_instance_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Last Pull Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "worker_instance_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\worker_instance_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 14, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "instant": false, + "legendFormat": "{{worker_instance_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Last Pull Workflows History Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "worker_instance_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\worker_instance_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Pull Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 24, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Pull Workflows History Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 16, + "panels": [], + "title": "Steps", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 12, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Activity Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{activity_name}}: {{error_code}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['activity_name'] as activity_name,\n Attributes['error_code'] as error_code,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, activity_name, error_code\nORDER BY time", + "refId": "A" + } + ], + "title": "Activity Error Rate", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "activity_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\activity_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 21, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Loop Upsert Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ev/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 25, + "interval": "15s", + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "legendFormat": "{{workflow_name}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND Attributes['workflow_name'] LIKE '${workflow_name:regex}'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Loop Events Per Second", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 17, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Message Send Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 20, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Signal Send Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 22, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Find Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 19, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Sub Workflow Dispatch Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", + "includeAll": true, + "label": "Cluster ID", + "multi": true, + "name": "cluster_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", + "includeAll": true, + "label": "Datacenter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", + "includeAll": true, + "label": "Workflow Name", + "multi": true, + "name": "workflow_name", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Gasoline", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", + "version": 1, + "weekStart": "" } diff --git a/docker/dev-multidc/core/grafana/dashboards/guard.json b/docker/dev-multidc/core/grafana/dashboards/guard.json index fb657d441b..b4d37655bc 100644 --- a/docker/dev-multidc/core/grafana/dashboards/guard.json +++ b/docker/dev-multidc/core/grafana/dashboards/guard.json @@ -1134,6 +1134,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, diff --git a/docker/dev-multidc/core/grafana/dashboards/tokio.json b/docker/dev-multidc/core/grafana/dashboards/tokio.json index efa5e4b49f..45bd84de2a 100644 --- a/docker/dev-multidc/core/grafana/dashboards/tokio.json +++ b/docker/dev-multidc/core/grafana/dashboards/tokio.json @@ -993,6 +993,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, @@ -1006,4 +1018,4 @@ "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/docker/dev-multidc/core/grafana/dashboards/traces.json b/docker/dev-multidc/core/grafana/dashboards/traces.json index c4c2c5a62f..54c2d0aefb 100644 --- a/docker/dev-multidc/core/grafana/dashboards/traces.json +++ b/docker/dev-multidc/core/grafana/dashboards/traces.json @@ -1,979 +1,969 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 21, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 9, - "panels": [], - "title": "Traces", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 0, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "sql", - "format": 0, - "meta": { - "builderOptions": { - "aggregates": [ - { - "aggregateType": "count", - "column": "" - } - ], - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "groupBy": [ - "ServiceName" - ], - "limit": 10000, - "mode": "trend", - "orderBy": [], - "queryType": "timeseries", - "table": "otel_traces" - } - }, - "pluginVersion": "4.0.6", - "queryType": "timeseries", - "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n ServiceName,\r\n count() as ` `\r\nFROM otel.otel_traces\r\nWHERE\r\n ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\nGROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000\r\n", - "refId": "A" - } - ], - "title": "Traces per Service", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Trace ID" - }, - "properties": [ - { - "id": "custom.width", - "value": 77 - }, - { - "id": "links", - "value": [ - { - "title": "__value.raw", - "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${service_name:queryparam}&var-trace_id=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Service Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 130 - }, - { - "id": "links", - "value": [ - { - "title": "__value.raw", - "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${trace_id:queryparam}&var-serviceName=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Duration" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "lcd", - "type": "gauge", - "valueDisplayMode": "text" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 216 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Service Tags" - }, - "properties": [ - { - "id": "custom.inspect", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 248 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 234 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": " " - }, - "properties": [ - { - "id": "custom.width", - "value": 49 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Span Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 140 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Ts" - }, - "properties": [ - { - "id": "custom.width", - "value": 168 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "URI/workflow_id/actor_id" - }, - "properties": [ - { - "id": "custom.width", - "value": 920 - } - ] - } - ] - }, - "gridPos": { - "h": 13, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 4, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Duration" - } - ] - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorType": "sql", - "format": 1, - "meta": { - "builderOptions": { - "columns": [], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "limit": 100, - "mode": "list", - "orderBy": [], - "queryType": "table", - "table": "otel_traces" - } - }, - "pluginVersion": "4.9.0", - "queryType": "table", - "rawSql": "SELECT\r\n (argMin(StatusCode, Timestamp) = 'Error' ? '⚠️' : '') as ` `,\r\n min(Timestamp) as Ts,\r\n TraceId as `Trace ID`,\r\n argMin(ServiceName, Timestamp) as `Service Name`,\r\n argMin(SpanName, Timestamp) as `Span Name`,\r\n argMin(coalesce(NULLIF(SpanAttributes['uri'], ''), NULLIF(SpanAttributes['workflow_id'], ''), SpanAttributes['actor_id']), Timestamp) as `URI/workflow_id/actor_id`,\r\n divide(max(Duration), 1000000) as Duration\r\nFROM otel.otel_traces\r\nWHERE\r\n ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\n AND ServiceName != 'loadgenerator'\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\nGROUP BY TraceId\r\nORDER BY Duration DESC\r\nLIMIT 100\r\n", - "refId": "A" - } - ], - "title": "Traces", - "type": "table" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 17, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 6, - "options": { - "spanFilters": { - "criticalPathOnly": false, - "matchesOnly": false, - "serviceNameOperator": "=", - "spanNameOperator": "=" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "builderOptions": { - "columns": [ - { - "hint": "trace_id", - "name": "TraceId" - }, - { - "hint": "trace_span_id", - "name": "SpanId" - }, - { - "hint": "trace_parent_span_id", - "name": "ParentSpanId" - }, - { - "hint": "trace_service_name", - "name": "ServiceName" - }, - { - "hint": "trace_operation_name", - "name": "SpanName" - }, - { - "hint": "time", - "name": "Timestamp" - }, - { - "hint": "trace_duration_time", - "name": "Duration" - }, - { - "hint": "trace_tags", - "name": "SpanAttributes" - }, - { - "hint": "trace_service_tags", - "name": "ResourceAttributes" - }, - { - "hint": "trace_status_code", - "name": "StatusCode" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "", - "operator": "WITH IN DASHBOARD TIME RANGE", - "type": "datetime" - }, - { - "condition": "AND", - "filterType": "custom", - "hint": "trace_duration_time", - "key": "", - "operator": ">", - "type": "UInt64", - "value": 0 - }, - { - "condition": "AND", - "filterType": "custom", - "hint": "trace_service_name", - "key": "", - "operator": "IS ANYTHING", - "type": "string", - "value": "" - } - ], - "limit": 1000, - "meta": { - "isTraceIdMode": true, - "otelEnabled": true, - "otelVersion": "latest", - "traceDurationUnit": "nanoseconds", - "traceId": "${trace_id}" - }, - "mode": "list", - "orderBy": [ - { - "default": true, - "dir": "DESC", - "hint": "time", - "name": "" - }, - { - "default": true, - "dir": "DESC", - "hint": "trace_duration_time", - "name": "" - } - ], - "queryType": "traces", - "table": "otel_traces" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "builder", - "format": 3, - "meta": { - "builderOptions": { - "columns": [], - "database": "", - "limit": 100, - "mode": "list", - "queryType": "table", - "table": "" - } - }, - "pluginVersion": "4.0.6", - "queryType": "traces", - "rawSql": "WITH\n\t'${trace_id}' as trace_id,\n\t(SELECT min(Start) FROM otel.otel_traces_trace_id_ts WHERE TraceId = trace_id) as trace_start,\n\t(SELECT max(End) + 1 FROM otel.otel_traces_trace_id_ts WHERE TraceId = trace_id) as trace_end\nSELECT\n\tTraceId as traceID,\n\tSpanId as spanID,\n\tParentSpanId as parentSpanID,\n\tServiceName as serviceName,\n\tSpanName as operationName, Timestamp as startTime,\n\tmultiply(Duration, 0.000001) as duration,\n\tarrayMap(key -> map('key', key, 'value', SpanAttributes[key]), mapKeys(SpanAttributes)) as tags,\n\tarrayMap(key -> map('key', key, 'value', ResourceAttributes[key]), mapKeys(ResourceAttributes)) as serviceTags,\n\tarrayMap((name, timestamp, attributes) -> tuple(name, toString(multiply(toUnixTimestamp64Nano(timestamp), 0.000001)), arrayMap( key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(name String, timestamp String, fields Array(Map(String, String))), `Events.Name`, `Events.Timestamp`, `Events.Attributes`) AS logs,\n\tarrayMap((traceID, spanID, attributes) -> tuple(traceID, spanID, arrayMap(key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(traceID String, spanID String, tags Array(Map(String, String))), `Links.TraceId`, `Links.SpanId`, `Links.Attributes`) AS references\nFROM otel.otel_traces\nWHERE\n\ttraceID = trace_id AND startTime >= trace_start AND startTime <= trace_end AND ( Duration > 0 )\nORDER BY Timestamp DESC, Duration DESC\nLIMIT 1000", - "refId": "A" - } - ], - "title": "Trace Details", - "type": "traces" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 24, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "sql", - "format": 0, - "meta": { - "builderOptions": { - "aggregates": [ - { - "aggregateType": "count", - "column": "" - } - ], - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "groupBy": [ - "ServiceName" - ], - "limit": 10000, - "mode": "trend", - "orderBy": [], - "queryType": "timeseries", - "table": "otel_traces" - } - }, - "pluginVersion": "4.0.6", - "queryType": "timeseries", - "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n count(*) as ` `,\r\n ServiceName\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(TraceId IN (${trace_id:singlequote}), $trace_id)\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\n AND StatusCode IN ('Error', 'STATUS_CODE_ERROR')\r\n AND ServiceName != 'loadgenerator' GROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000", - "refId": "A" - } - ], - "title": "Error rates", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 49 - }, - "id": 10, - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 61 - }, - "id": 11, - "options": { - "dedupStrategy": "none", - "enableInfiniteScrolling": false, - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "builderOptions": { - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - }, - { - "hint": "log_level", - "name": "SeverityText", - "type": "LowCardinality(String)" - }, - { - "hint": "log_message", - "name": "Body", - "type": "String" - } - ], - "database": "otel", - "filters": [], - "limit": 1000, - "meta": { - "logMessageLike": "", - "otelVersion": "latest" - }, - "mode": "list", - "orderBy": [], - "queryType": "logs", - "table": "otel_logs" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorType": "builder", - "format": 2, - "pluginVersion": "4.0.6", - "rawSql": "SELECT Timestamp as timestamp, Body as body, SeverityText as level FROM \"otel\".\"otel_logs\" LIMIT 1000", - "refId": "A" - } - ], - "title": "Trace Logs", - "type": "logs" - } - ], - "title": "Logs", - "type": "row" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "ClickHouse", - "value": "ClickHouse" - }, - "includeAll": false, - "label": "ClickHouse instance", - "name": "datasource", - "options": [], - "query": "grafana-clickhouse-datasource", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ServiceName FROM otel.otel_traces", - "includeAll": true, - "label": "Service Name", - "multi": true, - "name": "service_name", - "options": [], - "query": "SELECT DISTINCT ServiceName FROM otel.otel_traces", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", - "includeAll": true, - "label": "Trace ID", - "name": "trace_id", - "options": [], - "query": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Span", - "multi": true, - "name": "span_name", - "options": [], - "query": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "allValue": "'All'", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Ray ID", - "multi": true, - "name": "ray_id", - "options": [], - "query": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "allValue": "'All'", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Workflow ID", - "multi": true, - "name": "workflow_id", - "options": [], - "query": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "America/Los_Angeles", - "title": "Traces", - "uid": "8klBUGfVk", - "version": 2, - "weekStart": "" -} \ No newline at end of file + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 21, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 9, + "panels": [], + "title": "Traces", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "sql", + "format": 0, + "meta": { + "builderOptions": { + "aggregates": [ + { + "aggregateType": "count", + "column": "" + } + ], + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "groupBy": ["ServiceName"], + "limit": 10000, + "mode": "trend", + "orderBy": [], + "queryType": "timeseries", + "table": "otel_traces" + } + }, + "pluginVersion": "4.0.6", + "queryType": "timeseries", + "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n ServiceName,\r\n count() as ` `\r\nFROM otel.otel_traces\r\nWHERE\r\n ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\nGROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000\r\n", + "refId": "A" + } + ], + "title": "Traces per Service", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Trace ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 77 + }, + { + "id": "links", + "value": [ + { + "title": "__value.raw", + "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${service_name:queryparam}&var-trace_id=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Service Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + }, + { + "id": "links", + "value": [ + { + "title": "__value.raw", + "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${trace_id:queryparam}&var-serviceName=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Duration" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge", + "valueDisplayMode": "text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 216 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Service Tags" + }, + "properties": [ + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 248 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 234 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": " " + }, + "properties": [ + { + "id": "custom.width", + "value": 49 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Span Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 140 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Ts" + }, + "properties": [ + { + "id": "custom.width", + "value": 168 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "URI/workflow_id/actor_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 920 + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Duration" + } + ] + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "limit": 100, + "mode": "list", + "orderBy": [], + "queryType": "table", + "table": "otel_traces" + } + }, + "pluginVersion": "4.9.0", + "queryType": "table", + "rawSql": "SELECT\r\n (argMin(StatusCode, Timestamp) = 'Error' ? '⚠️' : '') as ` `,\r\n min(Timestamp) as Ts,\r\n TraceId as `Trace ID`,\r\n argMin(ServiceName, Timestamp) as `Service Name`,\r\n argMin(SpanName, Timestamp) as `Span Name`,\r\n argMin(coalesce(NULLIF(SpanAttributes['uri'], ''), NULLIF(SpanAttributes['workflow_id'], ''), SpanAttributes['actor_id']), Timestamp) as `URI/workflow_id/actor_id`,\r\n divide(max(Duration), 1000000) as Duration\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\n AND ServiceName != 'loadgenerator'\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\nGROUP BY TraceId\r\nORDER BY Duration DESC\r\nLIMIT 100\r\n", + "refId": "A" + } + ], + "title": "Traces", + "type": "table" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 6, + "options": { + "spanFilters": { + "criticalPathOnly": false, + "matchesOnly": false, + "serviceNameOperator": "=", + "spanNameOperator": "=" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "builderOptions": { + "columns": [ + { + "hint": "trace_id", + "name": "TraceId" + }, + { + "hint": "trace_span_id", + "name": "SpanId" + }, + { + "hint": "trace_parent_span_id", + "name": "ParentSpanId" + }, + { + "hint": "trace_service_name", + "name": "ServiceName" + }, + { + "hint": "trace_operation_name", + "name": "SpanName" + }, + { + "hint": "time", + "name": "Timestamp" + }, + { + "hint": "trace_duration_time", + "name": "Duration" + }, + { + "hint": "trace_tags", + "name": "SpanAttributes" + }, + { + "hint": "trace_service_tags", + "name": "ResourceAttributes" + }, + { + "hint": "trace_status_code", + "name": "StatusCode" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "datetime" + }, + { + "condition": "AND", + "filterType": "custom", + "hint": "trace_duration_time", + "key": "", + "operator": ">", + "type": "UInt64", + "value": 0 + }, + { + "condition": "AND", + "filterType": "custom", + "hint": "trace_service_name", + "key": "", + "operator": "IS ANYTHING", + "type": "string", + "value": "" + } + ], + "limit": 1000, + "meta": { + "isTraceIdMode": true, + "otelEnabled": true, + "otelVersion": "latest", + "traceDurationUnit": "nanoseconds", + "traceId": "${trace_id}" + }, + "mode": "list", + "orderBy": [ + { + "default": true, + "dir": "DESC", + "hint": "time", + "name": "" + }, + { + "default": true, + "dir": "DESC", + "hint": "trace_duration_time", + "name": "" + } + ], + "queryType": "traces", + "table": "otel_traces" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "builder", + "format": 3, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 100, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.0.6", + "queryType": "traces", + "rawSql": "WITH\n\t(SELECT min(Start) FROM otel.otel_traces_trace_id_ts WHERE $__conditionalAll(TraceId, $trace_id)) as trace_start,\n\t(SELECT max(End) + 1 FROM otel.otel_traces_trace_id_ts WHERE $__conditionalAll(TraceId, $trace_id)) as trace_end\nSELECT\n\tTraceId as traceID,\n\tSpanId as spanID,\n\tParentSpanId as parentSpanID,\n\tServiceName as serviceName,\n\tSpanName as operationName, Timestamp as startTime,\n\tmultiply(Duration, 0.000001) as duration,\n\tarrayMap(key -> map('key', key, 'value', SpanAttributes[key]), mapKeys(SpanAttributes)) as tags,\n\tarrayMap(key -> map('key', key, 'value', ResourceAttributes[key]), mapKeys(ResourceAttributes)) as serviceTags,\n\tarrayMap((name, timestamp, attributes) -> tuple(name, toString(multiply(toUnixTimestamp64Nano(timestamp), 0.000001)), arrayMap( key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(name String, timestamp String, fields Array(Map(String, String))), `Events.Name`, `Events.Timestamp`, `Events.Attributes`) AS logs,\n\tarrayMap((traceID, spanID, attributes) -> tuple(traceID, spanID, arrayMap(key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(traceID String, spanID String, tags Array(Map(String, String))), `Links.TraceId`, `Links.SpanId`, `Links.Attributes`) AS references\nFROM otel.otel_traces\nWHERE\n\t$__conditionalAll(traceID, $trace_id) AND startTime >= trace_start AND startTime <= trace_end AND ( Duration > 0 )\nORDER BY Timestamp DESC, Duration DESC\nLIMIT 1000", + "refId": "A" + } + ], + "title": "Trace Details", + "type": "traces" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 24, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "sql", + "format": 0, + "meta": { + "builderOptions": { + "aggregates": [ + { + "aggregateType": "count", + "column": "" + } + ], + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "groupBy": ["ServiceName"], + "limit": 10000, + "mode": "trend", + "orderBy": [], + "queryType": "timeseries", + "table": "otel_traces" + } + }, + "pluginVersion": "4.0.6", + "queryType": "timeseries", + "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n count(*) as ` `,\r\n ServiceName\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(TraceId, $trace_id)\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id)\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\n AND StatusCode IN ('Error', 'STATUS_CODE_ERROR')\r\n AND ServiceName != 'loadgenerator' GROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000", + "refId": "A" + } + ], + "title": "Error rates", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 10, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 11, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "builderOptions": { + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + }, + { + "hint": "log_level", + "name": "SeverityText", + "type": "LowCardinality(String)" + }, + { + "hint": "log_message", + "name": "Body", + "type": "String" + } + ], + "database": "otel", + "filters": [], + "limit": 1000, + "meta": { + "logMessageLike": "", + "otelVersion": "latest" + }, + "mode": "list", + "orderBy": [], + "queryType": "logs", + "table": "otel_logs" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorType": "builder", + "format": 2, + "pluginVersion": "4.0.6", + "rawSql": "SELECT Timestamp as timestamp, Body as body, SeverityText as level FROM \"otel\".\"otel_logs\" LIMIT 1000", + "refId": "A" + } + ], + "title": "Trace Logs", + "type": "logs" + } + ], + "title": "Logs", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "ClickHouse", + "value": "ClickHouse" + }, + "includeAll": false, + "label": "ClickHouse instance", + "name": "datasource", + "options": [], + "query": "grafana-clickhouse-datasource", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ServiceName FROM otel.otel_traces", + "includeAll": true, + "label": "Service Name", + "multi": true, + "name": "service_name", + "options": [], + "query": "SELECT DISTINCT ServiceName FROM otel.otel_traces", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", + "includeAll": true, + "label": "Trace ID", + "name": "trace_id", + "options": [], + "query": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Span", + "multi": true, + "name": "span_name", + "options": [], + "query": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "'All'", + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Ray ID", + "multi": true, + "name": "ray_id", + "options": [], + "query": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "'All'", + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Workflow ID", + "multi": true, + "name": "workflow_id", + "options": [], + "query": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "America/Los_Angeles", + "title": "Traces", + "uid": "8klBUGfVk", + "version": 2, + "weekStart": "" +} diff --git a/docker/dev-multinode/grafana/dashboards/api.json b/docker/dev-multinode/grafana/dashboards/api.json index 34004bf134..4ad455621b 100644 --- a/docker/dev-multinode/grafana/dashboards/api.json +++ b/docker/dev-multinode/grafana/dashboards/api.json @@ -1,1172 +1,1260 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 7, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 24, - "panels": [], - "title": "API", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 26, - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_api_request_pending'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Requests Pending", - "transformations": [ - { - "id": "groupingToMatrix", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 32, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING sum(Count) > 0\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (avg)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 33, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label, bounds, idx\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (p95)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "bounds" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 34, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n Attributes['method'] as method,\n Attributes['path'] as path,\n histogramQuantile(arrayZip(ExplicitBounds, BucketCounts), 0.99) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['watch'] = '0'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, method, path\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (p99)", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\n -- (\n -- SELECT\n -- sum(Value) / $__interval_ms * 1000 as value\n -- FROM otel.otel_metrics_sum\n -- WHERE MetricName = 'rivet_api_request_total'\n -- AND $__conditionalAll(Attributes['path'], $path)\n -- AND $__conditionalAll(Attributes['method'], $method)\n -- AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n -- AND $__timeFilter(TimeUnix)\n -- ) as prev_value\n -- lagInFrame(value, 1, value) OVER (ORDER BY time) as previous_value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_total'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Request Rate", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 28, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['status'], ' (',\n Attributes['error_code'], ')'\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (4xx)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['error_code'], ' (',\n Attributes['status'], ')'\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND Attributes['error_code']-- NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (5xx)", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 35, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200' OR Attributes['status'] LIKE '5%')\n AND Attributes['error_code']-- NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, method, path, status, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "200 vs 5xx (4xx excluded)", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", - "includeAll": true, - "label": "Datacenter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", - "includeAll": true, - "label": "Path", - "multi": true, - "name": "path", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", - "includeAll": true, - "label": "Method", - "multi": true, - "name": "method", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "API", - "uid": "feg8y72gpm2o0c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "API", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 26, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n concat(bounds[idx-1], 's - ', bounds[idx], 's') as label,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, label\nORDER BY Time", + "refId": "A" + } + ], + "title": "Request Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\label" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_pending'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Requests Pending", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": {} + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 32, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING sum(Count) > 0\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (avg)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 33, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING value >= (\n SELECT quantile(0.95)(avg_value)\n FROM (\n SELECT sum(Sum) / sum(Count) as avg_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\n GROUP BY \n $__timeInterval(TimeUnix),\n ResourceAttributes['datacenter_id'],\n Attributes['method'],\n Attributes['path']\n )\n)\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (p95)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 34, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING value >= (\n SELECT quantile(0.99)(avg_value)\n FROM (\n SELECT sum(Sum) / sum(Count) as avg_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\n GROUP BY \n $__timeInterval(TimeUnix),\n ResourceAttributes['datacenter_id'],\n Attributes['method'],\n Attributes['path']\n )\n)\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (p99)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 25, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_total'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Request Rate", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 28, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Name", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['status'], ' (',\n Attributes['error_code'], ')'\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Error Rate (4xx)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 31, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['error_code'], ' (',\n Attributes['status'], ')'\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Error Rate (5xx)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": {} + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 0, + "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "timeseries", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n -- Attributes['method'] as method,\n -- Attributes['path'] as path,\n Attributes['status'] as status,\n -- Attributes['error_code'] as error_code,\n sum(Count) / 30 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200 OK' OR Attributes['status'] LIKE '5%')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, status\nORDER BY time", + "refId": "A" + } + ], + "title": "200 vs 5xx (4xx excluded)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "time": 0, + "value 200 OK": 2, + "value 500 Internal Server Error": 1 + }, + "renameByName": { + "200 OK": "200", + "500 Internal Server Error": "500", + "time": "time", + "value 200 OK": "200", + "value 500 Internal Server Error": "500" + } + } + } + ], + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", + "includeAll": true, + "label": "Datacenter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", + "includeAll": true, + "label": "Path", + "multi": true, + "name": "path", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", + "includeAll": true, + "label": "Method", + "multi": true, + "name": "method", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "API", + "uid": "feg8y72gpm2o0c", + "version": 1, + "weekStart": "" } diff --git a/docker/dev-multinode/grafana/dashboards/cache.json b/docker/dev-multinode/grafana/dashboards/cache.json index f5806c749e..222196172e 100644 --- a/docker/dev-multinode/grafana/dashboards/cache.json +++ b/docker/dev-multinode/grafana/dashboards/cache.json @@ -1,1176 +1,1174 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 10, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n CASE\n WHEN ResourceAttributes['datacenter_id'] != '' AND ResourceAttributes['server_id'] != '' THEN concat(ResourceAttributes['datacenter_id'], ' - ', ResourceAttributes['server_id'])\n ELSE 'Route Cache Size'\n END as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_route_cache_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Route Cache Size", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 11, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Rate Limiters", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "In-Flight Counters", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 8 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active TCP Connections", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "req/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 8 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_tcp_connection'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "TCP Connection Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 8 - }, - "id": 1, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "TCP Connection Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Proxy Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "req/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_proxy_request'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Proxy Request Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 16 - }, - "id": 9, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Proxy Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 24 - }, - "id": 6, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Resolve Route Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", - "description": "", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", - "description": "", - "includeAll": true, - "label": "Dataceter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", - "description": "", - "includeAll": true, - "label": "Server ID", - "multi": true, - "name": "server_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Rivet Guard", - "uid": "cen785ige8fswd2", - "version": 1, - "weekStart": "" -} \ No newline at end of file + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n CASE\n WHEN ResourceAttributes['datacenter_id'] != '' AND ResourceAttributes['server_id'] != '' THEN concat(ResourceAttributes['datacenter_id'], ' - ', ResourceAttributes['server_id'])\n ELSE 'Route Cache Size'\n END as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_route_cache_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Route Cache Size", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "In-Flight Counters", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_tcp_connection'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "TCP Connection Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 1, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "TCP Connection Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active Proxy Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_proxy_request'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Proxy Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 9, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Proxy Request Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 6, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Resolve Route Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", + "description": "", + "includeAll": true, + "label": "Cluster ID", + "multi": true, + "name": "cluster_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", + "description": "", + "includeAll": true, + "label": "Dataceter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", + "description": "", + "includeAll": true, + "label": "Server ID", + "multi": true, + "name": "server_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Rivet Guard", + "uid": "cen785ige8fswd2", + "version": 1, + "weekStart": "" +} diff --git a/docker/dev-multinode/grafana/dashboards/futures.json b/docker/dev-multinode/grafana/dashboards/futures.json index 0c992c4bcb..34d0c27571 100644 --- a/docker/dev-multinode/grafana/dashboards/futures.json +++ b/docker/dev-multinode/grafana/dashboards/futures.json @@ -140,12 +140,8 @@ "list": [ { "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": ["All"], + "value": ["$__all"] }, "datasource": { "type": "grafana-clickhouse-datasource", @@ -168,12 +164,8 @@ }, { "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": ["All"], + "value": ["$__all"] }, "datasource": { "type": "grafana-clickhouse-datasource", @@ -193,6 +185,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, @@ -205,4 +209,4 @@ "title": "Futures", "version": 0, "weekStart": "" -} \ No newline at end of file +} diff --git a/docker/dev-multinode/grafana/dashboards/gasoline.json b/docker/dev-multinode/grafana/dashboards/gasoline.json index bac8c7b6da..6a2fc3a3d6 100644 --- a/docker/dev-multinode/grafana/dashboards/gasoline.json +++ b/docker/dev-multinode/grafana/dashboards/gasoline.json @@ -1,2583 +1,2563 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 3, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Running Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Sleeping Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Dead Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "({{workflow_name}}) {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n Attributes['error_code'] as error_code,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "Dead Workflow Errors", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "__auto", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Workers", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "datacenter_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\datacenter_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{signal_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as signal_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, signal_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Pending Signals", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "signal_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\signal_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size", - "value": "" - }, - "yBuckets": { - "mode": "size", - "scale": { - "log": 2, - "type": "log" - }, - "value": "" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "hide": false, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Signal Receive Lag", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY\n Time, le\nORDER BY\n Time, le", - "refId": "A" - } - ], - "title": "Pull Signals Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "instant": false, - "legendFormat": "{{worker_instance_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Last Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "worker_instance_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\worker_instance_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "instant": false, - "legendFormat": "{{worker_instance_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Last Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "worker_instance_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\worker_instance_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 16, - "panels": [], - "title": "Steps", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 12, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Activity Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 9, - "interval": "15s", - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{activity_name}}: {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['activity_name'] as activity_name,\n Attributes['error_code'] as error_code,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, activity_name, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "Activity Error Rate", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "activity_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\activity_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 21, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Loop Upsert Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ev/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 25, - "interval": "15s", - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "legendFormat": "{{workflow_name}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND Attributes['workflow_name'] LIKE '${workflow_name:regex}'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Loop Events Per Second", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 17, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Message Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 20, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Signal Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 22, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Find Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 76 - }, - "id": 19, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Sub Workflow Dispatch Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", - "includeAll": true, - "label": "Datacenter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", - "includeAll": true, - "label": "Workflow Name", - "multi": true, - "name": "workflow_name", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Gasoline", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Running Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Sleeping Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Dead Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "({{workflow_name}}) {{error_code}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n Attributes['error_code'] as error_code,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name, error_code\nORDER BY time", + "refId": "A" + } + ], + "title": "Dead Workflow Errors", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "__auto", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active Workers", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "datacenter_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\datacenter_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{signal_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as signal_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, signal_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Pending Signals", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "signal_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\signal_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size", + "value": "" + }, + "yBuckets": { + "mode": "size", + "scale": { + "log": 2, + "type": "log" + }, + "value": "" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "hide": false, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Signal Receive Lag", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY\n Time, le\nORDER BY\n Time, le", + "refId": "A" + } + ], + "title": "Pull Signals Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 13, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "instant": false, + "legendFormat": "{{worker_instance_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Last Pull Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "worker_instance_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\worker_instance_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 14, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "instant": false, + "legendFormat": "{{worker_instance_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Last Pull Workflows History Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "worker_instance_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\worker_instance_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Pull Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 24, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Pull Workflows History Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 16, + "panels": [], + "title": "Steps", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 12, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Activity Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{activity_name}}: {{error_code}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['activity_name'] as activity_name,\n Attributes['error_code'] as error_code,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, activity_name, error_code\nORDER BY time", + "refId": "A" + } + ], + "title": "Activity Error Rate", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "activity_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\activity_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 21, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Loop Upsert Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ev/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 25, + "interval": "15s", + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "legendFormat": "{{workflow_name}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND Attributes['workflow_name'] LIKE '${workflow_name:regex}'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Loop Events Per Second", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 17, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Message Send Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 20, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Signal Send Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 22, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Find Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 19, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Sub Workflow Dispatch Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", + "includeAll": true, + "label": "Cluster ID", + "multi": true, + "name": "cluster_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", + "includeAll": true, + "label": "Datacenter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", + "includeAll": true, + "label": "Workflow Name", + "multi": true, + "name": "workflow_name", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Gasoline", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", + "version": 1, + "weekStart": "" } diff --git a/docker/dev-multinode/grafana/dashboards/guard.json b/docker/dev-multinode/grafana/dashboards/guard.json index fb657d441b..b4d37655bc 100644 --- a/docker/dev-multinode/grafana/dashboards/guard.json +++ b/docker/dev-multinode/grafana/dashboards/guard.json @@ -1134,6 +1134,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, diff --git a/docker/dev-multinode/grafana/dashboards/tokio.json b/docker/dev-multinode/grafana/dashboards/tokio.json index efa5e4b49f..45bd84de2a 100644 --- a/docker/dev-multinode/grafana/dashboards/tokio.json +++ b/docker/dev-multinode/grafana/dashboards/tokio.json @@ -993,6 +993,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, @@ -1006,4 +1018,4 @@ "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/docker/dev-multinode/grafana/dashboards/traces.json b/docker/dev-multinode/grafana/dashboards/traces.json index c4c2c5a62f..54c2d0aefb 100644 --- a/docker/dev-multinode/grafana/dashboards/traces.json +++ b/docker/dev-multinode/grafana/dashboards/traces.json @@ -1,979 +1,969 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 21, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 9, - "panels": [], - "title": "Traces", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 0, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "sql", - "format": 0, - "meta": { - "builderOptions": { - "aggregates": [ - { - "aggregateType": "count", - "column": "" - } - ], - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "groupBy": [ - "ServiceName" - ], - "limit": 10000, - "mode": "trend", - "orderBy": [], - "queryType": "timeseries", - "table": "otel_traces" - } - }, - "pluginVersion": "4.0.6", - "queryType": "timeseries", - "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n ServiceName,\r\n count() as ` `\r\nFROM otel.otel_traces\r\nWHERE\r\n ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\nGROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000\r\n", - "refId": "A" - } - ], - "title": "Traces per Service", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Trace ID" - }, - "properties": [ - { - "id": "custom.width", - "value": 77 - }, - { - "id": "links", - "value": [ - { - "title": "__value.raw", - "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${service_name:queryparam}&var-trace_id=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Service Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 130 - }, - { - "id": "links", - "value": [ - { - "title": "__value.raw", - "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${trace_id:queryparam}&var-serviceName=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Duration" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "lcd", - "type": "gauge", - "valueDisplayMode": "text" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 216 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Service Tags" - }, - "properties": [ - { - "id": "custom.inspect", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 248 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 234 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": " " - }, - "properties": [ - { - "id": "custom.width", - "value": 49 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Span Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 140 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Ts" - }, - "properties": [ - { - "id": "custom.width", - "value": 168 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "URI/workflow_id/actor_id" - }, - "properties": [ - { - "id": "custom.width", - "value": 920 - } - ] - } - ] - }, - "gridPos": { - "h": 13, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 4, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Duration" - } - ] - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorType": "sql", - "format": 1, - "meta": { - "builderOptions": { - "columns": [], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "limit": 100, - "mode": "list", - "orderBy": [], - "queryType": "table", - "table": "otel_traces" - } - }, - "pluginVersion": "4.9.0", - "queryType": "table", - "rawSql": "SELECT\r\n (argMin(StatusCode, Timestamp) = 'Error' ? '⚠️' : '') as ` `,\r\n min(Timestamp) as Ts,\r\n TraceId as `Trace ID`,\r\n argMin(ServiceName, Timestamp) as `Service Name`,\r\n argMin(SpanName, Timestamp) as `Span Name`,\r\n argMin(coalesce(NULLIF(SpanAttributes['uri'], ''), NULLIF(SpanAttributes['workflow_id'], ''), SpanAttributes['actor_id']), Timestamp) as `URI/workflow_id/actor_id`,\r\n divide(max(Duration), 1000000) as Duration\r\nFROM otel.otel_traces\r\nWHERE\r\n ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\n AND ServiceName != 'loadgenerator'\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\nGROUP BY TraceId\r\nORDER BY Duration DESC\r\nLIMIT 100\r\n", - "refId": "A" - } - ], - "title": "Traces", - "type": "table" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 17, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 6, - "options": { - "spanFilters": { - "criticalPathOnly": false, - "matchesOnly": false, - "serviceNameOperator": "=", - "spanNameOperator": "=" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "builderOptions": { - "columns": [ - { - "hint": "trace_id", - "name": "TraceId" - }, - { - "hint": "trace_span_id", - "name": "SpanId" - }, - { - "hint": "trace_parent_span_id", - "name": "ParentSpanId" - }, - { - "hint": "trace_service_name", - "name": "ServiceName" - }, - { - "hint": "trace_operation_name", - "name": "SpanName" - }, - { - "hint": "time", - "name": "Timestamp" - }, - { - "hint": "trace_duration_time", - "name": "Duration" - }, - { - "hint": "trace_tags", - "name": "SpanAttributes" - }, - { - "hint": "trace_service_tags", - "name": "ResourceAttributes" - }, - { - "hint": "trace_status_code", - "name": "StatusCode" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "", - "operator": "WITH IN DASHBOARD TIME RANGE", - "type": "datetime" - }, - { - "condition": "AND", - "filterType": "custom", - "hint": "trace_duration_time", - "key": "", - "operator": ">", - "type": "UInt64", - "value": 0 - }, - { - "condition": "AND", - "filterType": "custom", - "hint": "trace_service_name", - "key": "", - "operator": "IS ANYTHING", - "type": "string", - "value": "" - } - ], - "limit": 1000, - "meta": { - "isTraceIdMode": true, - "otelEnabled": true, - "otelVersion": "latest", - "traceDurationUnit": "nanoseconds", - "traceId": "${trace_id}" - }, - "mode": "list", - "orderBy": [ - { - "default": true, - "dir": "DESC", - "hint": "time", - "name": "" - }, - { - "default": true, - "dir": "DESC", - "hint": "trace_duration_time", - "name": "" - } - ], - "queryType": "traces", - "table": "otel_traces" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "builder", - "format": 3, - "meta": { - "builderOptions": { - "columns": [], - "database": "", - "limit": 100, - "mode": "list", - "queryType": "table", - "table": "" - } - }, - "pluginVersion": "4.0.6", - "queryType": "traces", - "rawSql": "WITH\n\t'${trace_id}' as trace_id,\n\t(SELECT min(Start) FROM otel.otel_traces_trace_id_ts WHERE TraceId = trace_id) as trace_start,\n\t(SELECT max(End) + 1 FROM otel.otel_traces_trace_id_ts WHERE TraceId = trace_id) as trace_end\nSELECT\n\tTraceId as traceID,\n\tSpanId as spanID,\n\tParentSpanId as parentSpanID,\n\tServiceName as serviceName,\n\tSpanName as operationName, Timestamp as startTime,\n\tmultiply(Duration, 0.000001) as duration,\n\tarrayMap(key -> map('key', key, 'value', SpanAttributes[key]), mapKeys(SpanAttributes)) as tags,\n\tarrayMap(key -> map('key', key, 'value', ResourceAttributes[key]), mapKeys(ResourceAttributes)) as serviceTags,\n\tarrayMap((name, timestamp, attributes) -> tuple(name, toString(multiply(toUnixTimestamp64Nano(timestamp), 0.000001)), arrayMap( key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(name String, timestamp String, fields Array(Map(String, String))), `Events.Name`, `Events.Timestamp`, `Events.Attributes`) AS logs,\n\tarrayMap((traceID, spanID, attributes) -> tuple(traceID, spanID, arrayMap(key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(traceID String, spanID String, tags Array(Map(String, String))), `Links.TraceId`, `Links.SpanId`, `Links.Attributes`) AS references\nFROM otel.otel_traces\nWHERE\n\ttraceID = trace_id AND startTime >= trace_start AND startTime <= trace_end AND ( Duration > 0 )\nORDER BY Timestamp DESC, Duration DESC\nLIMIT 1000", - "refId": "A" - } - ], - "title": "Trace Details", - "type": "traces" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 24, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "sql", - "format": 0, - "meta": { - "builderOptions": { - "aggregates": [ - { - "aggregateType": "count", - "column": "" - } - ], - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "groupBy": [ - "ServiceName" - ], - "limit": 10000, - "mode": "trend", - "orderBy": [], - "queryType": "timeseries", - "table": "otel_traces" - } - }, - "pluginVersion": "4.0.6", - "queryType": "timeseries", - "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n count(*) as ` `,\r\n ServiceName\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(TraceId IN (${trace_id:singlequote}), $trace_id)\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\n AND StatusCode IN ('Error', 'STATUS_CODE_ERROR')\r\n AND ServiceName != 'loadgenerator' GROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000", - "refId": "A" - } - ], - "title": "Error rates", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 49 - }, - "id": 10, - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 61 - }, - "id": 11, - "options": { - "dedupStrategy": "none", - "enableInfiniteScrolling": false, - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "builderOptions": { - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - }, - { - "hint": "log_level", - "name": "SeverityText", - "type": "LowCardinality(String)" - }, - { - "hint": "log_message", - "name": "Body", - "type": "String" - } - ], - "database": "otel", - "filters": [], - "limit": 1000, - "meta": { - "logMessageLike": "", - "otelVersion": "latest" - }, - "mode": "list", - "orderBy": [], - "queryType": "logs", - "table": "otel_logs" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorType": "builder", - "format": 2, - "pluginVersion": "4.0.6", - "rawSql": "SELECT Timestamp as timestamp, Body as body, SeverityText as level FROM \"otel\".\"otel_logs\" LIMIT 1000", - "refId": "A" - } - ], - "title": "Trace Logs", - "type": "logs" - } - ], - "title": "Logs", - "type": "row" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "ClickHouse", - "value": "ClickHouse" - }, - "includeAll": false, - "label": "ClickHouse instance", - "name": "datasource", - "options": [], - "query": "grafana-clickhouse-datasource", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ServiceName FROM otel.otel_traces", - "includeAll": true, - "label": "Service Name", - "multi": true, - "name": "service_name", - "options": [], - "query": "SELECT DISTINCT ServiceName FROM otel.otel_traces", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", - "includeAll": true, - "label": "Trace ID", - "name": "trace_id", - "options": [], - "query": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Span", - "multi": true, - "name": "span_name", - "options": [], - "query": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "allValue": "'All'", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Ray ID", - "multi": true, - "name": "ray_id", - "options": [], - "query": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "allValue": "'All'", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Workflow ID", - "multi": true, - "name": "workflow_id", - "options": [], - "query": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "America/Los_Angeles", - "title": "Traces", - "uid": "8klBUGfVk", - "version": 2, - "weekStart": "" -} \ No newline at end of file + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 21, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 9, + "panels": [], + "title": "Traces", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "sql", + "format": 0, + "meta": { + "builderOptions": { + "aggregates": [ + { + "aggregateType": "count", + "column": "" + } + ], + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "groupBy": ["ServiceName"], + "limit": 10000, + "mode": "trend", + "orderBy": [], + "queryType": "timeseries", + "table": "otel_traces" + } + }, + "pluginVersion": "4.0.6", + "queryType": "timeseries", + "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n ServiceName,\r\n count() as ` `\r\nFROM otel.otel_traces\r\nWHERE\r\n ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\nGROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000\r\n", + "refId": "A" + } + ], + "title": "Traces per Service", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Trace ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 77 + }, + { + "id": "links", + "value": [ + { + "title": "__value.raw", + "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${service_name:queryparam}&var-trace_id=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Service Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + }, + { + "id": "links", + "value": [ + { + "title": "__value.raw", + "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${trace_id:queryparam}&var-serviceName=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Duration" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge", + "valueDisplayMode": "text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 216 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Service Tags" + }, + "properties": [ + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 248 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 234 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": " " + }, + "properties": [ + { + "id": "custom.width", + "value": 49 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Span Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 140 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Ts" + }, + "properties": [ + { + "id": "custom.width", + "value": 168 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "URI/workflow_id/actor_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 920 + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Duration" + } + ] + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "limit": 100, + "mode": "list", + "orderBy": [], + "queryType": "table", + "table": "otel_traces" + } + }, + "pluginVersion": "4.9.0", + "queryType": "table", + "rawSql": "SELECT\r\n (argMin(StatusCode, Timestamp) = 'Error' ? '⚠️' : '') as ` `,\r\n min(Timestamp) as Ts,\r\n TraceId as `Trace ID`,\r\n argMin(ServiceName, Timestamp) as `Service Name`,\r\n argMin(SpanName, Timestamp) as `Span Name`,\r\n argMin(coalesce(NULLIF(SpanAttributes['uri'], ''), NULLIF(SpanAttributes['workflow_id'], ''), SpanAttributes['actor_id']), Timestamp) as `URI/workflow_id/actor_id`,\r\n divide(max(Duration), 1000000) as Duration\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\n AND ServiceName != 'loadgenerator'\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\nGROUP BY TraceId\r\nORDER BY Duration DESC\r\nLIMIT 100\r\n", + "refId": "A" + } + ], + "title": "Traces", + "type": "table" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 6, + "options": { + "spanFilters": { + "criticalPathOnly": false, + "matchesOnly": false, + "serviceNameOperator": "=", + "spanNameOperator": "=" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "builderOptions": { + "columns": [ + { + "hint": "trace_id", + "name": "TraceId" + }, + { + "hint": "trace_span_id", + "name": "SpanId" + }, + { + "hint": "trace_parent_span_id", + "name": "ParentSpanId" + }, + { + "hint": "trace_service_name", + "name": "ServiceName" + }, + { + "hint": "trace_operation_name", + "name": "SpanName" + }, + { + "hint": "time", + "name": "Timestamp" + }, + { + "hint": "trace_duration_time", + "name": "Duration" + }, + { + "hint": "trace_tags", + "name": "SpanAttributes" + }, + { + "hint": "trace_service_tags", + "name": "ResourceAttributes" + }, + { + "hint": "trace_status_code", + "name": "StatusCode" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "datetime" + }, + { + "condition": "AND", + "filterType": "custom", + "hint": "trace_duration_time", + "key": "", + "operator": ">", + "type": "UInt64", + "value": 0 + }, + { + "condition": "AND", + "filterType": "custom", + "hint": "trace_service_name", + "key": "", + "operator": "IS ANYTHING", + "type": "string", + "value": "" + } + ], + "limit": 1000, + "meta": { + "isTraceIdMode": true, + "otelEnabled": true, + "otelVersion": "latest", + "traceDurationUnit": "nanoseconds", + "traceId": "${trace_id}" + }, + "mode": "list", + "orderBy": [ + { + "default": true, + "dir": "DESC", + "hint": "time", + "name": "" + }, + { + "default": true, + "dir": "DESC", + "hint": "trace_duration_time", + "name": "" + } + ], + "queryType": "traces", + "table": "otel_traces" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "builder", + "format": 3, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 100, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.0.6", + "queryType": "traces", + "rawSql": "WITH\n\t(SELECT min(Start) FROM otel.otel_traces_trace_id_ts WHERE $__conditionalAll(TraceId, $trace_id)) as trace_start,\n\t(SELECT max(End) + 1 FROM otel.otel_traces_trace_id_ts WHERE $__conditionalAll(TraceId, $trace_id)) as trace_end\nSELECT\n\tTraceId as traceID,\n\tSpanId as spanID,\n\tParentSpanId as parentSpanID,\n\tServiceName as serviceName,\n\tSpanName as operationName, Timestamp as startTime,\n\tmultiply(Duration, 0.000001) as duration,\n\tarrayMap(key -> map('key', key, 'value', SpanAttributes[key]), mapKeys(SpanAttributes)) as tags,\n\tarrayMap(key -> map('key', key, 'value', ResourceAttributes[key]), mapKeys(ResourceAttributes)) as serviceTags,\n\tarrayMap((name, timestamp, attributes) -> tuple(name, toString(multiply(toUnixTimestamp64Nano(timestamp), 0.000001)), arrayMap( key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(name String, timestamp String, fields Array(Map(String, String))), `Events.Name`, `Events.Timestamp`, `Events.Attributes`) AS logs,\n\tarrayMap((traceID, spanID, attributes) -> tuple(traceID, spanID, arrayMap(key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(traceID String, spanID String, tags Array(Map(String, String))), `Links.TraceId`, `Links.SpanId`, `Links.Attributes`) AS references\nFROM otel.otel_traces\nWHERE\n\t$__conditionalAll(traceID, $trace_id) AND startTime >= trace_start AND startTime <= trace_end AND ( Duration > 0 )\nORDER BY Timestamp DESC, Duration DESC\nLIMIT 1000", + "refId": "A" + } + ], + "title": "Trace Details", + "type": "traces" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 24, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "sql", + "format": 0, + "meta": { + "builderOptions": { + "aggregates": [ + { + "aggregateType": "count", + "column": "" + } + ], + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "groupBy": ["ServiceName"], + "limit": 10000, + "mode": "trend", + "orderBy": [], + "queryType": "timeseries", + "table": "otel_traces" + } + }, + "pluginVersion": "4.0.6", + "queryType": "timeseries", + "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n count(*) as ` `,\r\n ServiceName\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(TraceId, $trace_id)\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id)\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\n AND StatusCode IN ('Error', 'STATUS_CODE_ERROR')\r\n AND ServiceName != 'loadgenerator' GROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000", + "refId": "A" + } + ], + "title": "Error rates", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 10, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 11, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "builderOptions": { + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + }, + { + "hint": "log_level", + "name": "SeverityText", + "type": "LowCardinality(String)" + }, + { + "hint": "log_message", + "name": "Body", + "type": "String" + } + ], + "database": "otel", + "filters": [], + "limit": 1000, + "meta": { + "logMessageLike": "", + "otelVersion": "latest" + }, + "mode": "list", + "orderBy": [], + "queryType": "logs", + "table": "otel_logs" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorType": "builder", + "format": 2, + "pluginVersion": "4.0.6", + "rawSql": "SELECT Timestamp as timestamp, Body as body, SeverityText as level FROM \"otel\".\"otel_logs\" LIMIT 1000", + "refId": "A" + } + ], + "title": "Trace Logs", + "type": "logs" + } + ], + "title": "Logs", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "ClickHouse", + "value": "ClickHouse" + }, + "includeAll": false, + "label": "ClickHouse instance", + "name": "datasource", + "options": [], + "query": "grafana-clickhouse-datasource", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ServiceName FROM otel.otel_traces", + "includeAll": true, + "label": "Service Name", + "multi": true, + "name": "service_name", + "options": [], + "query": "SELECT DISTINCT ServiceName FROM otel.otel_traces", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", + "includeAll": true, + "label": "Trace ID", + "name": "trace_id", + "options": [], + "query": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Span", + "multi": true, + "name": "span_name", + "options": [], + "query": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "'All'", + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Ray ID", + "multi": true, + "name": "ray_id", + "options": [], + "query": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "'All'", + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Workflow ID", + "multi": true, + "name": "workflow_id", + "options": [], + "query": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "America/Los_Angeles", + "title": "Traces", + "uid": "8klBUGfVk", + "version": 2, + "weekStart": "" +} diff --git a/docker/dev/grafana/dashboards/api.json b/docker/dev/grafana/dashboards/api.json index 34004bf134..4ad455621b 100644 --- a/docker/dev/grafana/dashboards/api.json +++ b/docker/dev/grafana/dashboards/api.json @@ -1,1172 +1,1260 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 7, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 24, - "panels": [], - "title": "API", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 26, - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_api_request_pending'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Requests Pending", - "transformations": [ - { - "id": "groupingToMatrix", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 32, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING sum(Count) > 0\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (avg)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 33, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label, bounds, idx\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (p95)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "bounds" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 34, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n Attributes['method'] as method,\n Attributes['path'] as path,\n histogramQuantile(arrayZip(ExplicitBounds, BucketCounts), 0.99) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['watch'] = '0'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, method, path\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (p99)", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\n -- (\n -- SELECT\n -- sum(Value) / $__interval_ms * 1000 as value\n -- FROM otel.otel_metrics_sum\n -- WHERE MetricName = 'rivet_api_request_total'\n -- AND $__conditionalAll(Attributes['path'], $path)\n -- AND $__conditionalAll(Attributes['method'], $method)\n -- AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n -- AND $__timeFilter(TimeUnix)\n -- ) as prev_value\n -- lagInFrame(value, 1, value) OVER (ORDER BY time) as previous_value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_total'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Request Rate", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 28, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['status'], ' (',\n Attributes['error_code'], ')'\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (4xx)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['error_code'], ' (',\n Attributes['status'], ')'\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND Attributes['error_code']-- NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (5xx)", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 35, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200' OR Attributes['status'] LIKE '5%')\n AND Attributes['error_code']-- NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, method, path, status, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "200 vs 5xx (4xx excluded)", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", - "includeAll": true, - "label": "Datacenter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", - "includeAll": true, - "label": "Path", - "multi": true, - "name": "path", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", - "includeAll": true, - "label": "Method", - "multi": true, - "name": "method", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "API", - "uid": "feg8y72gpm2o0c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "API", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 26, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n concat(bounds[idx-1], 's - ', bounds[idx], 's') as label,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, label\nORDER BY Time", + "refId": "A" + } + ], + "title": "Request Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\label" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_pending'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Requests Pending", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": {} + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 32, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING sum(Count) > 0\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (avg)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 33, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING value >= (\n SELECT quantile(0.95)(avg_value)\n FROM (\n SELECT sum(Sum) / sum(Count) as avg_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\n GROUP BY \n $__timeInterval(TimeUnix),\n ResourceAttributes['datacenter_id'],\n Attributes['method'],\n Attributes['path']\n )\n)\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (p95)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 34, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING value >= (\n SELECT quantile(0.99)(avg_value)\n FROM (\n SELECT sum(Sum) / sum(Count) as avg_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\n GROUP BY \n $__timeInterval(TimeUnix),\n ResourceAttributes['datacenter_id'],\n Attributes['method'],\n Attributes['path']\n )\n)\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (p99)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 25, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_total'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Request Rate", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 28, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Name", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['status'], ' (',\n Attributes['error_code'], ')'\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Error Rate (4xx)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 31, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['error_code'], ' (',\n Attributes['status'], ')'\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Error Rate (5xx)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": {} + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 0, + "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "timeseries", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n -- Attributes['method'] as method,\n -- Attributes['path'] as path,\n Attributes['status'] as status,\n -- Attributes['error_code'] as error_code,\n sum(Count) / 30 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200 OK' OR Attributes['status'] LIKE '5%')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, status\nORDER BY time", + "refId": "A" + } + ], + "title": "200 vs 5xx (4xx excluded)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "time": 0, + "value 200 OK": 2, + "value 500 Internal Server Error": 1 + }, + "renameByName": { + "200 OK": "200", + "500 Internal Server Error": "500", + "time": "time", + "value 200 OK": "200", + "value 500 Internal Server Error": "500" + } + } + } + ], + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", + "includeAll": true, + "label": "Datacenter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", + "includeAll": true, + "label": "Path", + "multi": true, + "name": "path", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", + "includeAll": true, + "label": "Method", + "multi": true, + "name": "method", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "API", + "uid": "feg8y72gpm2o0c", + "version": 1, + "weekStart": "" } diff --git a/docker/dev/grafana/dashboards/cache.json b/docker/dev/grafana/dashboards/cache.json index f5806c749e..222196172e 100644 --- a/docker/dev/grafana/dashboards/cache.json +++ b/docker/dev/grafana/dashboards/cache.json @@ -1,1176 +1,1174 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 10, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n CASE\n WHEN ResourceAttributes['datacenter_id'] != '' AND ResourceAttributes['server_id'] != '' THEN concat(ResourceAttributes['datacenter_id'], ' - ', ResourceAttributes['server_id'])\n ELSE 'Route Cache Size'\n END as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_route_cache_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Route Cache Size", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 11, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Rate Limiters", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "In-Flight Counters", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 8 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active TCP Connections", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "req/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 8 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_tcp_connection'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "TCP Connection Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 8 - }, - "id": 1, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "TCP Connection Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Proxy Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "req/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_proxy_request'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Proxy Request Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 16 - }, - "id": 9, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Proxy Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 24 - }, - "id": 6, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Resolve Route Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", - "description": "", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", - "description": "", - "includeAll": true, - "label": "Dataceter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", - "description": "", - "includeAll": true, - "label": "Server ID", - "multi": true, - "name": "server_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Rivet Guard", - "uid": "cen785ige8fswd2", - "version": 1, - "weekStart": "" -} \ No newline at end of file + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n CASE\n WHEN ResourceAttributes['datacenter_id'] != '' AND ResourceAttributes['server_id'] != '' THEN concat(ResourceAttributes['datacenter_id'], ' - ', ResourceAttributes['server_id'])\n ELSE 'Route Cache Size'\n END as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_route_cache_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Route Cache Size", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "In-Flight Counters", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_tcp_connection'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "TCP Connection Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 1, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "TCP Connection Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active Proxy Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_proxy_request'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Proxy Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 9, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Proxy Request Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 6, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Resolve Route Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", + "description": "", + "includeAll": true, + "label": "Cluster ID", + "multi": true, + "name": "cluster_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", + "description": "", + "includeAll": true, + "label": "Dataceter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", + "description": "", + "includeAll": true, + "label": "Server ID", + "multi": true, + "name": "server_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Rivet Guard", + "uid": "cen785ige8fswd2", + "version": 1, + "weekStart": "" +} diff --git a/docker/dev/grafana/dashboards/futures.json b/docker/dev/grafana/dashboards/futures.json index 0c992c4bcb..34d0c27571 100644 --- a/docker/dev/grafana/dashboards/futures.json +++ b/docker/dev/grafana/dashboards/futures.json @@ -140,12 +140,8 @@ "list": [ { "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": ["All"], + "value": ["$__all"] }, "datasource": { "type": "grafana-clickhouse-datasource", @@ -168,12 +164,8 @@ }, { "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": ["All"], + "value": ["$__all"] }, "datasource": { "type": "grafana-clickhouse-datasource", @@ -193,6 +185,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, @@ -205,4 +209,4 @@ "title": "Futures", "version": 0, "weekStart": "" -} \ No newline at end of file +} diff --git a/docker/dev/grafana/dashboards/gasoline.json b/docker/dev/grafana/dashboards/gasoline.json index bac8c7b6da..6a2fc3a3d6 100644 --- a/docker/dev/grafana/dashboards/gasoline.json +++ b/docker/dev/grafana/dashboards/gasoline.json @@ -1,2583 +1,2563 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 3, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Running Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Sleeping Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Dead Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "({{workflow_name}}) {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n Attributes['error_code'] as error_code,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "Dead Workflow Errors", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "__auto", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Workers", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "datacenter_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\datacenter_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{signal_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as signal_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, signal_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Pending Signals", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "signal_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\signal_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size", - "value": "" - }, - "yBuckets": { - "mode": "size", - "scale": { - "log": 2, - "type": "log" - }, - "value": "" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "hide": false, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Signal Receive Lag", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY\n Time, le\nORDER BY\n Time, le", - "refId": "A" - } - ], - "title": "Pull Signals Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "instant": false, - "legendFormat": "{{worker_instance_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Last Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "worker_instance_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\worker_instance_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "instant": false, - "legendFormat": "{{worker_instance_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Last Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "worker_instance_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\worker_instance_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 16, - "panels": [], - "title": "Steps", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 12, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Activity Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 9, - "interval": "15s", - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{activity_name}}: {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['activity_name'] as activity_name,\n Attributes['error_code'] as error_code,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, activity_name, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "Activity Error Rate", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "activity_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\activity_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 21, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Loop Upsert Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ev/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 25, - "interval": "15s", - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "legendFormat": "{{workflow_name}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND Attributes['workflow_name'] LIKE '${workflow_name:regex}'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Loop Events Per Second", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 17, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Message Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 20, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Signal Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 22, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Find Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 76 - }, - "id": 19, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Sub Workflow Dispatch Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", - "includeAll": true, - "label": "Datacenter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", - "includeAll": true, - "label": "Workflow Name", - "multi": true, - "name": "workflow_name", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Gasoline", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Running Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Sleeping Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Dead Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "({{workflow_name}}) {{error_code}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n Attributes['error_code'] as error_code,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name, error_code\nORDER BY time", + "refId": "A" + } + ], + "title": "Dead Workflow Errors", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "__auto", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active Workers", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "datacenter_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\datacenter_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{signal_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as signal_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, signal_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Pending Signals", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "signal_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\signal_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size", + "value": "" + }, + "yBuckets": { + "mode": "size", + "scale": { + "log": 2, + "type": "log" + }, + "value": "" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "hide": false, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Signal Receive Lag", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY\n Time, le\nORDER BY\n Time, le", + "refId": "A" + } + ], + "title": "Pull Signals Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 13, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "instant": false, + "legendFormat": "{{worker_instance_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Last Pull Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "worker_instance_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\worker_instance_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 14, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "instant": false, + "legendFormat": "{{worker_instance_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Last Pull Workflows History Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "worker_instance_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\worker_instance_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Pull Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 24, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Pull Workflows History Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 16, + "panels": [], + "title": "Steps", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 12, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Activity Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{activity_name}}: {{error_code}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['activity_name'] as activity_name,\n Attributes['error_code'] as error_code,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, activity_name, error_code\nORDER BY time", + "refId": "A" + } + ], + "title": "Activity Error Rate", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "activity_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\activity_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 21, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Loop Upsert Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ev/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 25, + "interval": "15s", + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "legendFormat": "{{workflow_name}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND Attributes['workflow_name'] LIKE '${workflow_name:regex}'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Loop Events Per Second", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 17, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Message Send Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 20, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Signal Send Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 22, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Find Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 19, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Sub Workflow Dispatch Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", + "includeAll": true, + "label": "Cluster ID", + "multi": true, + "name": "cluster_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", + "includeAll": true, + "label": "Datacenter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", + "includeAll": true, + "label": "Workflow Name", + "multi": true, + "name": "workflow_name", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Gasoline", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", + "version": 1, + "weekStart": "" } diff --git a/docker/dev/grafana/dashboards/guard.json b/docker/dev/grafana/dashboards/guard.json index fb657d441b..b4d37655bc 100644 --- a/docker/dev/grafana/dashboards/guard.json +++ b/docker/dev/grafana/dashboards/guard.json @@ -1134,6 +1134,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, diff --git a/docker/dev/grafana/dashboards/tokio.json b/docker/dev/grafana/dashboards/tokio.json index efa5e4b49f..45bd84de2a 100644 --- a/docker/dev/grafana/dashboards/tokio.json +++ b/docker/dev/grafana/dashboards/tokio.json @@ -993,6 +993,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, @@ -1006,4 +1018,4 @@ "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/docker/dev/grafana/dashboards/traces.json b/docker/dev/grafana/dashboards/traces.json index c4c2c5a62f..54c2d0aefb 100644 --- a/docker/dev/grafana/dashboards/traces.json +++ b/docker/dev/grafana/dashboards/traces.json @@ -1,979 +1,969 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 21, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 9, - "panels": [], - "title": "Traces", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 0, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "sql", - "format": 0, - "meta": { - "builderOptions": { - "aggregates": [ - { - "aggregateType": "count", - "column": "" - } - ], - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "groupBy": [ - "ServiceName" - ], - "limit": 10000, - "mode": "trend", - "orderBy": [], - "queryType": "timeseries", - "table": "otel_traces" - } - }, - "pluginVersion": "4.0.6", - "queryType": "timeseries", - "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n ServiceName,\r\n count() as ` `\r\nFROM otel.otel_traces\r\nWHERE\r\n ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\nGROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000\r\n", - "refId": "A" - } - ], - "title": "Traces per Service", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Trace ID" - }, - "properties": [ - { - "id": "custom.width", - "value": 77 - }, - { - "id": "links", - "value": [ - { - "title": "__value.raw", - "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${service_name:queryparam}&var-trace_id=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Service Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 130 - }, - { - "id": "links", - "value": [ - { - "title": "__value.raw", - "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${trace_id:queryparam}&var-serviceName=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Duration" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "lcd", - "type": "gauge", - "valueDisplayMode": "text" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 216 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Service Tags" - }, - "properties": [ - { - "id": "custom.inspect", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 248 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 234 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": " " - }, - "properties": [ - { - "id": "custom.width", - "value": 49 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Span Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 140 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Ts" - }, - "properties": [ - { - "id": "custom.width", - "value": 168 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "URI/workflow_id/actor_id" - }, - "properties": [ - { - "id": "custom.width", - "value": 920 - } - ] - } - ] - }, - "gridPos": { - "h": 13, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 4, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Duration" - } - ] - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorType": "sql", - "format": 1, - "meta": { - "builderOptions": { - "columns": [], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "limit": 100, - "mode": "list", - "orderBy": [], - "queryType": "table", - "table": "otel_traces" - } - }, - "pluginVersion": "4.9.0", - "queryType": "table", - "rawSql": "SELECT\r\n (argMin(StatusCode, Timestamp) = 'Error' ? '⚠️' : '') as ` `,\r\n min(Timestamp) as Ts,\r\n TraceId as `Trace ID`,\r\n argMin(ServiceName, Timestamp) as `Service Name`,\r\n argMin(SpanName, Timestamp) as `Span Name`,\r\n argMin(coalesce(NULLIF(SpanAttributes['uri'], ''), NULLIF(SpanAttributes['workflow_id'], ''), SpanAttributes['actor_id']), Timestamp) as `URI/workflow_id/actor_id`,\r\n divide(max(Duration), 1000000) as Duration\r\nFROM otel.otel_traces\r\nWHERE\r\n ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\n AND ServiceName != 'loadgenerator'\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\nGROUP BY TraceId\r\nORDER BY Duration DESC\r\nLIMIT 100\r\n", - "refId": "A" - } - ], - "title": "Traces", - "type": "table" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 17, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 6, - "options": { - "spanFilters": { - "criticalPathOnly": false, - "matchesOnly": false, - "serviceNameOperator": "=", - "spanNameOperator": "=" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "builderOptions": { - "columns": [ - { - "hint": "trace_id", - "name": "TraceId" - }, - { - "hint": "trace_span_id", - "name": "SpanId" - }, - { - "hint": "trace_parent_span_id", - "name": "ParentSpanId" - }, - { - "hint": "trace_service_name", - "name": "ServiceName" - }, - { - "hint": "trace_operation_name", - "name": "SpanName" - }, - { - "hint": "time", - "name": "Timestamp" - }, - { - "hint": "trace_duration_time", - "name": "Duration" - }, - { - "hint": "trace_tags", - "name": "SpanAttributes" - }, - { - "hint": "trace_service_tags", - "name": "ResourceAttributes" - }, - { - "hint": "trace_status_code", - "name": "StatusCode" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "", - "operator": "WITH IN DASHBOARD TIME RANGE", - "type": "datetime" - }, - { - "condition": "AND", - "filterType": "custom", - "hint": "trace_duration_time", - "key": "", - "operator": ">", - "type": "UInt64", - "value": 0 - }, - { - "condition": "AND", - "filterType": "custom", - "hint": "trace_service_name", - "key": "", - "operator": "IS ANYTHING", - "type": "string", - "value": "" - } - ], - "limit": 1000, - "meta": { - "isTraceIdMode": true, - "otelEnabled": true, - "otelVersion": "latest", - "traceDurationUnit": "nanoseconds", - "traceId": "${trace_id}" - }, - "mode": "list", - "orderBy": [ - { - "default": true, - "dir": "DESC", - "hint": "time", - "name": "" - }, - { - "default": true, - "dir": "DESC", - "hint": "trace_duration_time", - "name": "" - } - ], - "queryType": "traces", - "table": "otel_traces" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "builder", - "format": 3, - "meta": { - "builderOptions": { - "columns": [], - "database": "", - "limit": 100, - "mode": "list", - "queryType": "table", - "table": "" - } - }, - "pluginVersion": "4.0.6", - "queryType": "traces", - "rawSql": "WITH\n\t'${trace_id}' as trace_id,\n\t(SELECT min(Start) FROM otel.otel_traces_trace_id_ts WHERE TraceId = trace_id) as trace_start,\n\t(SELECT max(End) + 1 FROM otel.otel_traces_trace_id_ts WHERE TraceId = trace_id) as trace_end\nSELECT\n\tTraceId as traceID,\n\tSpanId as spanID,\n\tParentSpanId as parentSpanID,\n\tServiceName as serviceName,\n\tSpanName as operationName, Timestamp as startTime,\n\tmultiply(Duration, 0.000001) as duration,\n\tarrayMap(key -> map('key', key, 'value', SpanAttributes[key]), mapKeys(SpanAttributes)) as tags,\n\tarrayMap(key -> map('key', key, 'value', ResourceAttributes[key]), mapKeys(ResourceAttributes)) as serviceTags,\n\tarrayMap((name, timestamp, attributes) -> tuple(name, toString(multiply(toUnixTimestamp64Nano(timestamp), 0.000001)), arrayMap( key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(name String, timestamp String, fields Array(Map(String, String))), `Events.Name`, `Events.Timestamp`, `Events.Attributes`) AS logs,\n\tarrayMap((traceID, spanID, attributes) -> tuple(traceID, spanID, arrayMap(key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(traceID String, spanID String, tags Array(Map(String, String))), `Links.TraceId`, `Links.SpanId`, `Links.Attributes`) AS references\nFROM otel.otel_traces\nWHERE\n\ttraceID = trace_id AND startTime >= trace_start AND startTime <= trace_end AND ( Duration > 0 )\nORDER BY Timestamp DESC, Duration DESC\nLIMIT 1000", - "refId": "A" - } - ], - "title": "Trace Details", - "type": "traces" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 24, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "sql", - "format": 0, - "meta": { - "builderOptions": { - "aggregates": [ - { - "aggregateType": "count", - "column": "" - } - ], - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "groupBy": [ - "ServiceName" - ], - "limit": 10000, - "mode": "trend", - "orderBy": [], - "queryType": "timeseries", - "table": "otel_traces" - } - }, - "pluginVersion": "4.0.6", - "queryType": "timeseries", - "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n count(*) as ` `,\r\n ServiceName\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(TraceId IN (${trace_id:singlequote}), $trace_id)\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\n AND StatusCode IN ('Error', 'STATUS_CODE_ERROR')\r\n AND ServiceName != 'loadgenerator' GROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000", - "refId": "A" - } - ], - "title": "Error rates", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 49 - }, - "id": 10, - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 61 - }, - "id": 11, - "options": { - "dedupStrategy": "none", - "enableInfiniteScrolling": false, - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "builderOptions": { - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - }, - { - "hint": "log_level", - "name": "SeverityText", - "type": "LowCardinality(String)" - }, - { - "hint": "log_message", - "name": "Body", - "type": "String" - } - ], - "database": "otel", - "filters": [], - "limit": 1000, - "meta": { - "logMessageLike": "", - "otelVersion": "latest" - }, - "mode": "list", - "orderBy": [], - "queryType": "logs", - "table": "otel_logs" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorType": "builder", - "format": 2, - "pluginVersion": "4.0.6", - "rawSql": "SELECT Timestamp as timestamp, Body as body, SeverityText as level FROM \"otel\".\"otel_logs\" LIMIT 1000", - "refId": "A" - } - ], - "title": "Trace Logs", - "type": "logs" - } - ], - "title": "Logs", - "type": "row" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "ClickHouse", - "value": "ClickHouse" - }, - "includeAll": false, - "label": "ClickHouse instance", - "name": "datasource", - "options": [], - "query": "grafana-clickhouse-datasource", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ServiceName FROM otel.otel_traces", - "includeAll": true, - "label": "Service Name", - "multi": true, - "name": "service_name", - "options": [], - "query": "SELECT DISTINCT ServiceName FROM otel.otel_traces", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", - "includeAll": true, - "label": "Trace ID", - "name": "trace_id", - "options": [], - "query": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Span", - "multi": true, - "name": "span_name", - "options": [], - "query": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "allValue": "'All'", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Ray ID", - "multi": true, - "name": "ray_id", - "options": [], - "query": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "allValue": "'All'", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Workflow ID", - "multi": true, - "name": "workflow_id", - "options": [], - "query": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "America/Los_Angeles", - "title": "Traces", - "uid": "8klBUGfVk", - "version": 2, - "weekStart": "" -} \ No newline at end of file + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 21, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 9, + "panels": [], + "title": "Traces", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "sql", + "format": 0, + "meta": { + "builderOptions": { + "aggregates": [ + { + "aggregateType": "count", + "column": "" + } + ], + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "groupBy": ["ServiceName"], + "limit": 10000, + "mode": "trend", + "orderBy": [], + "queryType": "timeseries", + "table": "otel_traces" + } + }, + "pluginVersion": "4.0.6", + "queryType": "timeseries", + "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n ServiceName,\r\n count() as ` `\r\nFROM otel.otel_traces\r\nWHERE\r\n ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\nGROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000\r\n", + "refId": "A" + } + ], + "title": "Traces per Service", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Trace ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 77 + }, + { + "id": "links", + "value": [ + { + "title": "__value.raw", + "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${service_name:queryparam}&var-trace_id=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Service Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + }, + { + "id": "links", + "value": [ + { + "title": "__value.raw", + "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${trace_id:queryparam}&var-serviceName=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Duration" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge", + "valueDisplayMode": "text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 216 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Service Tags" + }, + "properties": [ + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 248 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 234 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": " " + }, + "properties": [ + { + "id": "custom.width", + "value": 49 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Span Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 140 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Ts" + }, + "properties": [ + { + "id": "custom.width", + "value": 168 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "URI/workflow_id/actor_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 920 + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Duration" + } + ] + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "limit": 100, + "mode": "list", + "orderBy": [], + "queryType": "table", + "table": "otel_traces" + } + }, + "pluginVersion": "4.9.0", + "queryType": "table", + "rawSql": "SELECT\r\n (argMin(StatusCode, Timestamp) = 'Error' ? '⚠️' : '') as ` `,\r\n min(Timestamp) as Ts,\r\n TraceId as `Trace ID`,\r\n argMin(ServiceName, Timestamp) as `Service Name`,\r\n argMin(SpanName, Timestamp) as `Span Name`,\r\n argMin(coalesce(NULLIF(SpanAttributes['uri'], ''), NULLIF(SpanAttributes['workflow_id'], ''), SpanAttributes['actor_id']), Timestamp) as `URI/workflow_id/actor_id`,\r\n divide(max(Duration), 1000000) as Duration\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\n AND ServiceName != 'loadgenerator'\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\nGROUP BY TraceId\r\nORDER BY Duration DESC\r\nLIMIT 100\r\n", + "refId": "A" + } + ], + "title": "Traces", + "type": "table" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 6, + "options": { + "spanFilters": { + "criticalPathOnly": false, + "matchesOnly": false, + "serviceNameOperator": "=", + "spanNameOperator": "=" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "builderOptions": { + "columns": [ + { + "hint": "trace_id", + "name": "TraceId" + }, + { + "hint": "trace_span_id", + "name": "SpanId" + }, + { + "hint": "trace_parent_span_id", + "name": "ParentSpanId" + }, + { + "hint": "trace_service_name", + "name": "ServiceName" + }, + { + "hint": "trace_operation_name", + "name": "SpanName" + }, + { + "hint": "time", + "name": "Timestamp" + }, + { + "hint": "trace_duration_time", + "name": "Duration" + }, + { + "hint": "trace_tags", + "name": "SpanAttributes" + }, + { + "hint": "trace_service_tags", + "name": "ResourceAttributes" + }, + { + "hint": "trace_status_code", + "name": "StatusCode" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "datetime" + }, + { + "condition": "AND", + "filterType": "custom", + "hint": "trace_duration_time", + "key": "", + "operator": ">", + "type": "UInt64", + "value": 0 + }, + { + "condition": "AND", + "filterType": "custom", + "hint": "trace_service_name", + "key": "", + "operator": "IS ANYTHING", + "type": "string", + "value": "" + } + ], + "limit": 1000, + "meta": { + "isTraceIdMode": true, + "otelEnabled": true, + "otelVersion": "latest", + "traceDurationUnit": "nanoseconds", + "traceId": "${trace_id}" + }, + "mode": "list", + "orderBy": [ + { + "default": true, + "dir": "DESC", + "hint": "time", + "name": "" + }, + { + "default": true, + "dir": "DESC", + "hint": "trace_duration_time", + "name": "" + } + ], + "queryType": "traces", + "table": "otel_traces" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "builder", + "format": 3, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 100, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.0.6", + "queryType": "traces", + "rawSql": "WITH\n\t(SELECT min(Start) FROM otel.otel_traces_trace_id_ts WHERE $__conditionalAll(TraceId, $trace_id)) as trace_start,\n\t(SELECT max(End) + 1 FROM otel.otel_traces_trace_id_ts WHERE $__conditionalAll(TraceId, $trace_id)) as trace_end\nSELECT\n\tTraceId as traceID,\n\tSpanId as spanID,\n\tParentSpanId as parentSpanID,\n\tServiceName as serviceName,\n\tSpanName as operationName, Timestamp as startTime,\n\tmultiply(Duration, 0.000001) as duration,\n\tarrayMap(key -> map('key', key, 'value', SpanAttributes[key]), mapKeys(SpanAttributes)) as tags,\n\tarrayMap(key -> map('key', key, 'value', ResourceAttributes[key]), mapKeys(ResourceAttributes)) as serviceTags,\n\tarrayMap((name, timestamp, attributes) -> tuple(name, toString(multiply(toUnixTimestamp64Nano(timestamp), 0.000001)), arrayMap( key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(name String, timestamp String, fields Array(Map(String, String))), `Events.Name`, `Events.Timestamp`, `Events.Attributes`) AS logs,\n\tarrayMap((traceID, spanID, attributes) -> tuple(traceID, spanID, arrayMap(key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(traceID String, spanID String, tags Array(Map(String, String))), `Links.TraceId`, `Links.SpanId`, `Links.Attributes`) AS references\nFROM otel.otel_traces\nWHERE\n\t$__conditionalAll(traceID, $trace_id) AND startTime >= trace_start AND startTime <= trace_end AND ( Duration > 0 )\nORDER BY Timestamp DESC, Duration DESC\nLIMIT 1000", + "refId": "A" + } + ], + "title": "Trace Details", + "type": "traces" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 24, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "sql", + "format": 0, + "meta": { + "builderOptions": { + "aggregates": [ + { + "aggregateType": "count", + "column": "" + } + ], + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "groupBy": ["ServiceName"], + "limit": 10000, + "mode": "trend", + "orderBy": [], + "queryType": "timeseries", + "table": "otel_traces" + } + }, + "pluginVersion": "4.0.6", + "queryType": "timeseries", + "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n count(*) as ` `,\r\n ServiceName\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(TraceId, $trace_id)\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id)\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\n AND StatusCode IN ('Error', 'STATUS_CODE_ERROR')\r\n AND ServiceName != 'loadgenerator' GROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000", + "refId": "A" + } + ], + "title": "Error rates", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 10, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 11, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "builderOptions": { + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + }, + { + "hint": "log_level", + "name": "SeverityText", + "type": "LowCardinality(String)" + }, + { + "hint": "log_message", + "name": "Body", + "type": "String" + } + ], + "database": "otel", + "filters": [], + "limit": 1000, + "meta": { + "logMessageLike": "", + "otelVersion": "latest" + }, + "mode": "list", + "orderBy": [], + "queryType": "logs", + "table": "otel_logs" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorType": "builder", + "format": 2, + "pluginVersion": "4.0.6", + "rawSql": "SELECT Timestamp as timestamp, Body as body, SeverityText as level FROM \"otel\".\"otel_logs\" LIMIT 1000", + "refId": "A" + } + ], + "title": "Trace Logs", + "type": "logs" + } + ], + "title": "Logs", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "ClickHouse", + "value": "ClickHouse" + }, + "includeAll": false, + "label": "ClickHouse instance", + "name": "datasource", + "options": [], + "query": "grafana-clickhouse-datasource", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ServiceName FROM otel.otel_traces", + "includeAll": true, + "label": "Service Name", + "multi": true, + "name": "service_name", + "options": [], + "query": "SELECT DISTINCT ServiceName FROM otel.otel_traces", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", + "includeAll": true, + "label": "Trace ID", + "name": "trace_id", + "options": [], + "query": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Span", + "multi": true, + "name": "span_name", + "options": [], + "query": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "'All'", + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Ray ID", + "multi": true, + "name": "ray_id", + "options": [], + "query": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "'All'", + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Workflow ID", + "multi": true, + "name": "workflow_id", + "options": [], + "query": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "America/Los_Angeles", + "title": "Traces", + "uid": "8klBUGfVk", + "version": 2, + "weekStart": "" +} diff --git a/docker/template/grafana-dashboards/api.json b/docker/template/grafana-dashboards/api.json index 34004bf134..4ad455621b 100644 --- a/docker/template/grafana-dashboards/api.json +++ b/docker/template/grafana-dashboards/api.json @@ -1,1172 +1,1260 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 7, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 24, - "panels": [], - "title": "API", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 26, - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_api_request_pending'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Requests Pending", - "transformations": [ - { - "id": "groupingToMatrix", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 32, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING sum(Count) > 0\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (avg)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 33, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label, bounds, idx\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (p95)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "bounds" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 34, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n Attributes['method'] as method,\n Attributes['path'] as path,\n histogramQuantile(arrayZip(ExplicitBounds, BucketCounts), 0.99) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['watch'] = '0'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, method, path\nORDER BY time", - "refId": "A" - } - ], - "title": "Duration (p99)", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\n -- (\n -- SELECT\n -- sum(Value) / $__interval_ms * 1000 as value\n -- FROM otel.otel_metrics_sum\n -- WHERE MetricName = 'rivet_api_request_total'\n -- AND $__conditionalAll(Attributes['path'], $path)\n -- AND $__conditionalAll(Attributes['method'], $method)\n -- AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n -- AND $__timeFilter(TimeUnix)\n -- ) as prev_value\n -- lagInFrame(value, 1, value) OVER (ORDER BY time) as previous_value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_total'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Request Rate", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": {} - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 28, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['status'], ' (',\n Attributes['error_code'], ')'\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (4xx)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['error_code'], ' (',\n Attributes['status'], ')'\n ) as label,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND Attributes['error_code']-- NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (5xx)", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 35, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200' OR Attributes['status'] LIKE '5%')\n AND Attributes['error_code']-- NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, method, path, status, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "200 vs 5xx (4xx excluded)", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", - "includeAll": true, - "label": "Datacenter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", - "includeAll": true, - "label": "Path", - "multi": true, - "name": "path", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", - "includeAll": true, - "label": "Method", - "multi": true, - "name": "method", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "API", - "uid": "feg8y72gpm2o0c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "API", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 26, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n concat(bounds[idx-1], 's - ', bounds[idx], 's') as label,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, label\nORDER BY Time", + "refId": "A" + } + ], + "title": "Request Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\label" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_pending'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Requests Pending", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": {} + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 32, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING sum(Count) > 0\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (avg)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 33, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING value >= (\n SELECT quantile(0.95)(avg_value)\n FROM (\n SELECT sum(Sum) / sum(Count) as avg_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\n GROUP BY \n $__timeInterval(TimeUnix),\n ResourceAttributes['datacenter_id'],\n Attributes['method'],\n Attributes['path']\n )\n)\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (p95)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 34, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Sum) / sum(Count) as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nHAVING value >= (\n SELECT quantile(0.99)(avg_value)\n FROM (\n SELECT sum(Sum) / sum(Count) as avg_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\n GROUP BY \n $__timeInterval(TimeUnix),\n ResourceAttributes['datacenter_id'],\n Attributes['method'],\n Attributes['path']\n )\n)\nORDER BY time", + "refId": "A" + } + ], + "title": "Duration (p99)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 25, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path']\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_total'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Request Rate", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 28, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Name", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['status'], ' (',\n Attributes['error_code'], ')'\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Error Rate (4xx)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 31, + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{datacenter_id}} {{method}} {{path}}: {{status}} ({{error_code}})", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n concat(\n ResourceAttributes['datacenter_id'], ' ',\n Attributes['method'], ' ',\n Attributes['path'], ': ',\n Attributes['error_code'], ' (',\n Attributes['status'], ')'\n ) as label,\n sum(Value) / $metric_interval as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Error Rate (5xx)", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": {} + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 0, + "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", + "meta": {}, + "pluginVersion": "4.11.1", + "queryType": "timeseries", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n -- Attributes['method'] as method,\n -- Attributes['path'] as path,\n Attributes['status'] as status,\n -- Attributes['error_code'] as error_code,\n sum(Count) / 30 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200 OK' OR Attributes['status'] LIKE '5%')\n AND $__conditionalAll(Attributes['path'], $path)\n AND $__conditionalAll(Attributes['method'], $method)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, status\nORDER BY time", + "refId": "A" + } + ], + "title": "200 vs 5xx (4xx excluded)", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "time": 0, + "value 200 OK": 2, + "value 500 Internal Server Error": 1 + }, + "renameByName": { + "200 OK": "200", + "500 Internal Server Error": "500", + "time": "time", + "value 200 OK": "200", + "value 500 Internal Server Error": "500" + } + } + } + ], + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", + "includeAll": true, + "label": "Datacenter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_errors' ORDER BY datacenter_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", + "includeAll": true, + "label": "Path", + "multi": true, + "name": "path", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) ORDER BY path", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", + "includeAll": true, + "label": "Method", + "multi": true, + "name": "method", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request' AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id) AND $__conditionalAll(Attributes['path'], $path) ORDER BY method", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "API", + "uid": "feg8y72gpm2o0c", + "version": 1, + "weekStart": "" } diff --git a/docker/template/grafana-dashboards/cache.json b/docker/template/grafana-dashboards/cache.json index f5806c749e..222196172e 100644 --- a/docker/template/grafana-dashboards/cache.json +++ b/docker/template/grafana-dashboards/cache.json @@ -1,1176 +1,1174 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 10, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n CASE\n WHEN ResourceAttributes['datacenter_id'] != '' AND ResourceAttributes['server_id'] != '' THEN concat(ResourceAttributes['datacenter_id'], ' - ', ResourceAttributes['server_id'])\n ELSE 'Route Cache Size'\n END as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_route_cache_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", - "refId": "A" - } - ], - "title": "Route Cache Size", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 11, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Rate Limiters", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 12, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "In-Flight Counters", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 8 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active TCP Connections", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "req/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 8 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_tcp_connection'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "TCP Connection Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 8 - }, - "id": 1, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "TCP Connection Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Proxy Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "req/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_proxy_request'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Proxy Request Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 16 - }, - "id": 9, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Proxy Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 24 - }, - "id": 6, - "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Resolve Route Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", - "description": "", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", - "description": "", - "includeAll": true, - "label": "Dataceter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", - "description": "", - "includeAll": true, - "label": "Server ID", - "multi": true, - "name": "server_id", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Rivet Guard", - "uid": "cen785ige8fswd2", - "version": 1, - "weekStart": "" -} \ No newline at end of file + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n CASE\n WHEN ResourceAttributes['datacenter_id'] != '' AND ResourceAttributes['server_id'] != '' THEN concat(ResourceAttributes['datacenter_id'], ' - ', ResourceAttributes['server_id'])\n ELSE 'Route Cache Size'\n END as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_route_cache_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\nORDER BY time", + "refId": "A" + } + ], + "title": "Route Cache Size", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "label", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\label" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "In-Flight Counters", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_tcp_connection'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "TCP Connection Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 1, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "TCP Connection Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active Proxy Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{datacenter_id}} - {{server_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_guard_proxy_request'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Proxy Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 9, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Proxy Request Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 6, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "range": true, + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Resolve Route Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", + "description": "", + "includeAll": true, + "label": "Cluster ID", + "multi": true, + "name": "cluster_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY cluster_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", + "description": "", + "includeAll": true, + "label": "Dataceter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY datacenter_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", + "description": "", + "includeAll": true, + "label": "Server ID", + "multi": true, + "name": "server_id", + "options": [], + "query": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_guard_tcp_connection_pending' ORDER BY server_id", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Rivet Guard", + "uid": "cen785ige8fswd2", + "version": 1, + "weekStart": "" +} diff --git a/docker/template/grafana-dashboards/futures.json b/docker/template/grafana-dashboards/futures.json index 0c992c4bcb..34d0c27571 100644 --- a/docker/template/grafana-dashboards/futures.json +++ b/docker/template/grafana-dashboards/futures.json @@ -140,12 +140,8 @@ "list": [ { "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": ["All"], + "value": ["$__all"] }, "datasource": { "type": "grafana-clickhouse-datasource", @@ -168,12 +164,8 @@ }, { "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": ["All"], + "value": ["$__all"] }, "datasource": { "type": "grafana-clickhouse-datasource", @@ -193,6 +185,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, @@ -205,4 +209,4 @@ "title": "Futures", "version": 0, "weekStart": "" -} \ No newline at end of file +} diff --git a/docker/template/grafana-dashboards/gasoline.json b/docker/template/grafana-dashboards/gasoline.json index bac8c7b6da..6a2fc3a3d6 100644 --- a/docker/template/grafana-dashboards/gasoline.json +++ b/docker/template/grafana-dashboards/gasoline.json @@ -1,2583 +1,2563 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 3, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Running Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.2", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Sleeping Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Dead Workflows", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "({{workflow_name}}) {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n Attributes['error_code'] as error_code,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "Dead Workflow Errors", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "__auto", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Workers", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "datacenter_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\datacenter_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{signal_name}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as signal_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, signal_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Pending Signals", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "signal_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\signal_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size", - "value": "" - }, - "yBuckets": { - "mode": "size", - "scale": { - "log": 2, - "type": "log" - }, - "value": "" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "hide": false, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Signal Receive Lag", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY\n Time, le\nORDER BY\n Time, le", - "refId": "A" - } - ], - "title": "Pull Signals Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "instant": false, - "legendFormat": "{{worker_instance_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Last Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "worker_instance_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\worker_instance_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "instant": false, - "legendFormat": "{{worker_instance_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Last Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "worker_instance_id", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\worker_instance_id" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 16, - "panels": [], - "title": "Steps", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 12, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Activity Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 9, - "interval": "15s", - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{activity_name}}: {{error_code}}", - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['activity_name'] as activity_name,\n Attributes['error_code'] as error_code,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, activity_name, error_code\nORDER BY time", - "refId": "A" - } - ], - "title": "Activity Error Rate", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "activity_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\activity_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 21, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Loop Upsert Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ev/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 25, - "interval": "15s", - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "legendFormat": "{{workflow_name}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND Attributes['workflow_name'] LIKE '${workflow_name:regex}'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", - "refId": "A" - } - ], - "title": "Loop Events Per Second", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "workflow_name", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\workflow_name" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 17, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Message Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 20, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Signal Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 22, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Find Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 76 - }, - "id": 19, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.10.1", - "queryType": "table", - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Sub Workflow Dispatch Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", - "includeAll": true, - "label": "Datacenter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", - "includeAll": true, - "label": "Workflow Name", - "multi": true, - "name": "workflow_name", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Gasoline", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Running Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.2", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Sleeping Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{workflow_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Dead Workflows", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "({{workflow_name}}) {{error_code}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n Attributes['error_code'] as error_code,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name, error_code\nORDER BY time", + "refId": "A" + } + ], + "title": "Dead Workflow Errors", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "__auto", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Active Workers", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "datacenter_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\datacenter_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "instant": false, + "legendFormat": "{{signal_name}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as signal_name,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, signal_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Pending Signals", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "signal_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\signal_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size", + "value": "" + }, + "yBuckets": { + "mode": "size", + "scale": { + "log": 2, + "type": "log" + }, + "value": "" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "hide": false, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Signal Receive Lag", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY\n Time, le\nORDER BY\n Time, le", + "refId": "A" + } + ], + "title": "Pull Signals Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 13, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "instant": false, + "legendFormat": "{{worker_instance_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Last Pull Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "worker_instance_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\worker_instance_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 14, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "instant": false, + "legendFormat": "{{worker_instance_id}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_instance_id'] as worker_instance_id,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, worker_instance_id\nORDER BY time", + "refId": "A" + } + ], + "title": "Last Pull Workflows History Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "worker_instance_id", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\worker_instance_id" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Pull Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 24, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Pull Workflows History Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 16, + "panels": [], + "title": "Steps", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 12, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Activity Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "legendFormat": "{{activity_name}}: {{error_code}}", + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['activity_name'] as activity_name,\n Attributes['error_code'] as error_code,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, activity_name, error_code\nORDER BY time", + "refId": "A" + } + ], + "title": "Activity Error Rate", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "activity_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\activity_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 21, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Loop Upsert Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ev/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 25, + "interval": "15s", + "options": { + "legend": { + "calcs": ["mean"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "legendFormat": "{{workflow_name}}", + "range": true, + "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(Count) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_histogram\nWHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND Attributes['workflow_name'] LIKE '${workflow_name:regex}'\n AND ResourceAttributes['cluster_id'] LIKE '${cluster_id:regex}'\n AND ResourceAttributes['datacenter_id'] LIKE '${datacenter_id:regex}'\n AND $__timeFilter(TimeUnix)\nGROUP BY time, workflow_name\nORDER BY time", + "refId": "A" + } + ], + "title": "Loop Events Per Second", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "workflow_name", + "emptyValue": "zero", + "rowField": "time", + "valueField": "value" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "time\\workflow_name" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 17, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Message Send Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 20, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Signal Send Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 22, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Find Workflows Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 19, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorMode": "code", + "editorType": "sql", + "format": 1, + "meta": {}, + "pluginVersion": "4.10.1", + "queryType": "table", + "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND $__conditionalAll(Attributes['workflow_name'], $workflow_name)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", + "refId": "A" + } + ], + "title": "Sub Workflow Dispatch Duration", + "transformations": [ + { + "id": "groupingToMatrix", + "options": { + "columnField": "le", + "emptyValue": "zero", + "rowField": "Time", + "valueField": "count" + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Time\\le" + } + ], + "fields": {} + } + } + ], + "type": "heatmap" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", + "includeAll": true, + "label": "Cluster ID", + "multi": true, + "name": "cluster_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY cluster_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", + "includeAll": true, + "label": "Datacenter ID", + "multi": true, + "name": "datacenter_id", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_gasoline_worker_last_ping' ORDER BY datacenter_id", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", + "includeAll": true, + "label": "Workflow Name", + "multi": true, + "name": "workflow_name", + "options": [], + "query": { + "qryType": 1, + "rawSql": "SELECT DISTINCT Attributes['workflow_name'] as workflow_name FROM otel.otel_metrics_histogram WHERE MetricName = 'rivet_gasoline_signal_recv_lag' ORDER BY workflow_name", + "refId": "ClickHouseVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Gasoline", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", + "version": 1, + "weekStart": "" } diff --git a/docker/template/grafana-dashboards/guard.json b/docker/template/grafana-dashboards/guard.json index fb657d441b..b4d37655bc 100644 --- a/docker/template/grafana-dashboards/guard.json +++ b/docker/template/grafana-dashboards/guard.json @@ -1134,6 +1134,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, diff --git a/docker/template/grafana-dashboards/tokio.json b/docker/template/grafana-dashboards/tokio.json index efa5e4b49f..45bd84de2a 100644 --- a/docker/template/grafana-dashboards/tokio.json +++ b/docker/template/grafana-dashboards/tokio.json @@ -993,6 +993,18 @@ "refresh": 1, "regex": "", "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" } ] }, @@ -1006,4 +1018,4 @@ "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/docker/template/grafana-dashboards/traces.json b/docker/template/grafana-dashboards/traces.json index c4c2c5a62f..54c2d0aefb 100644 --- a/docker/template/grafana-dashboards/traces.json +++ b/docker/template/grafana-dashboards/traces.json @@ -1,979 +1,969 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 21, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 9, - "panels": [], - "title": "Traces", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 0, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "sql", - "format": 0, - "meta": { - "builderOptions": { - "aggregates": [ - { - "aggregateType": "count", - "column": "" - } - ], - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "groupBy": [ - "ServiceName" - ], - "limit": 10000, - "mode": "trend", - "orderBy": [], - "queryType": "timeseries", - "table": "otel_traces" - } - }, - "pluginVersion": "4.0.6", - "queryType": "timeseries", - "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n ServiceName,\r\n count() as ` `\r\nFROM otel.otel_traces\r\nWHERE\r\n ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\nGROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000\r\n", - "refId": "A" - } - ], - "title": "Traces per Service", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Trace ID" - }, - "properties": [ - { - "id": "custom.width", - "value": 77 - }, - { - "id": "links", - "value": [ - { - "title": "__value.raw", - "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${service_name:queryparam}&var-trace_id=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Service Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 130 - }, - { - "id": "links", - "value": [ - { - "title": "__value.raw", - "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${trace_id:queryparam}&var-serviceName=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Duration" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "lcd", - "type": "gauge", - "valueDisplayMode": "text" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 216 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Service Tags" - }, - "properties": [ - { - "id": "custom.inspect", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 248 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "timestamp" - }, - "properties": [ - { - "id": "custom.width", - "value": 234 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": " " - }, - "properties": [ - { - "id": "custom.width", - "value": 49 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Span Name" - }, - "properties": [ - { - "id": "custom.width", - "value": 140 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Ts" - }, - "properties": [ - { - "id": "custom.width", - "value": 168 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "URI/workflow_id/actor_id" - }, - "properties": [ - { - "id": "custom.width", - "value": 920 - } - ] - } - ] - }, - "gridPos": { - "h": 13, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 4, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Duration" - } - ] - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorType": "sql", - "format": 1, - "meta": { - "builderOptions": { - "columns": [], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "limit": 100, - "mode": "list", - "orderBy": [], - "queryType": "table", - "table": "otel_traces" - } - }, - "pluginVersion": "4.9.0", - "queryType": "table", - "rawSql": "SELECT\r\n (argMin(StatusCode, Timestamp) = 'Error' ? '⚠️' : '') as ` `,\r\n min(Timestamp) as Ts,\r\n TraceId as `Trace ID`,\r\n argMin(ServiceName, Timestamp) as `Service Name`,\r\n argMin(SpanName, Timestamp) as `Span Name`,\r\n argMin(coalesce(NULLIF(SpanAttributes['uri'], ''), NULLIF(SpanAttributes['workflow_id'], ''), SpanAttributes['actor_id']), Timestamp) as `URI/workflow_id/actor_id`,\r\n divide(max(Duration), 1000000) as Duration\r\nFROM otel.otel_traces\r\nWHERE\r\n ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\n AND ServiceName != 'loadgenerator'\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\nGROUP BY TraceId\r\nORDER BY Duration DESC\r\nLIMIT 100\r\n", - "refId": "A" - } - ], - "title": "Traces", - "type": "table" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 17, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 6, - "options": { - "spanFilters": { - "criticalPathOnly": false, - "matchesOnly": false, - "serviceNameOperator": "=", - "spanNameOperator": "=" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "builderOptions": { - "columns": [ - { - "hint": "trace_id", - "name": "TraceId" - }, - { - "hint": "trace_span_id", - "name": "SpanId" - }, - { - "hint": "trace_parent_span_id", - "name": "ParentSpanId" - }, - { - "hint": "trace_service_name", - "name": "ServiceName" - }, - { - "hint": "trace_operation_name", - "name": "SpanName" - }, - { - "hint": "time", - "name": "Timestamp" - }, - { - "hint": "trace_duration_time", - "name": "Duration" - }, - { - "hint": "trace_tags", - "name": "SpanAttributes" - }, - { - "hint": "trace_service_tags", - "name": "ResourceAttributes" - }, - { - "hint": "trace_status_code", - "name": "StatusCode" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "", - "operator": "WITH IN DASHBOARD TIME RANGE", - "type": "datetime" - }, - { - "condition": "AND", - "filterType": "custom", - "hint": "trace_duration_time", - "key": "", - "operator": ">", - "type": "UInt64", - "value": 0 - }, - { - "condition": "AND", - "filterType": "custom", - "hint": "trace_service_name", - "key": "", - "operator": "IS ANYTHING", - "type": "string", - "value": "" - } - ], - "limit": 1000, - "meta": { - "isTraceIdMode": true, - "otelEnabled": true, - "otelVersion": "latest", - "traceDurationUnit": "nanoseconds", - "traceId": "${trace_id}" - }, - "mode": "list", - "orderBy": [ - { - "default": true, - "dir": "DESC", - "hint": "time", - "name": "" - }, - { - "default": true, - "dir": "DESC", - "hint": "trace_duration_time", - "name": "" - } - ], - "queryType": "traces", - "table": "otel_traces" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "builder", - "format": 3, - "meta": { - "builderOptions": { - "columns": [], - "database": "", - "limit": 100, - "mode": "list", - "queryType": "table", - "table": "" - } - }, - "pluginVersion": "4.0.6", - "queryType": "traces", - "rawSql": "WITH\n\t'${trace_id}' as trace_id,\n\t(SELECT min(Start) FROM otel.otel_traces_trace_id_ts WHERE TraceId = trace_id) as trace_start,\n\t(SELECT max(End) + 1 FROM otel.otel_traces_trace_id_ts WHERE TraceId = trace_id) as trace_end\nSELECT\n\tTraceId as traceID,\n\tSpanId as spanID,\n\tParentSpanId as parentSpanID,\n\tServiceName as serviceName,\n\tSpanName as operationName, Timestamp as startTime,\n\tmultiply(Duration, 0.000001) as duration,\n\tarrayMap(key -> map('key', key, 'value', SpanAttributes[key]), mapKeys(SpanAttributes)) as tags,\n\tarrayMap(key -> map('key', key, 'value', ResourceAttributes[key]), mapKeys(ResourceAttributes)) as serviceTags,\n\tarrayMap((name, timestamp, attributes) -> tuple(name, toString(multiply(toUnixTimestamp64Nano(timestamp), 0.000001)), arrayMap( key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(name String, timestamp String, fields Array(Map(String, String))), `Events.Name`, `Events.Timestamp`, `Events.Attributes`) AS logs,\n\tarrayMap((traceID, spanID, attributes) -> tuple(traceID, spanID, arrayMap(key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(traceID String, spanID String, tags Array(Map(String, String))), `Links.TraceId`, `Links.SpanId`, `Links.Attributes`) AS references\nFROM otel.otel_traces\nWHERE\n\ttraceID = trace_id AND startTime >= trace_start AND startTime <= trace_end AND ( Duration > 0 )\nORDER BY Timestamp DESC, Duration DESC\nLIMIT 1000", - "refId": "A" - } - ], - "title": "Trace Details", - "type": "traces" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 24, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse-traces" - }, - "editorType": "sql", - "format": 0, - "meta": { - "builderOptions": { - "aggregates": [ - { - "aggregateType": "count", - "column": "" - } - ], - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - } - ], - "database": "otel", - "filters": [ - { - "condition": "AND", - "filterType": "custom", - "hint": "time", - "key": "Timestamp", - "operator": "WITH IN DASHBOARD TIME RANGE", - "restrictToFields": [ - { - "label": "Timestamp", - "name": "Timestamp", - "picklistValues": [], - "type": "DateTime64(9)" - } - ], - "type": "datetime" - } - ], - "groupBy": [ - "ServiceName" - ], - "limit": 10000, - "mode": "trend", - "orderBy": [], - "queryType": "timeseries", - "table": "otel_traces" - } - }, - "pluginVersion": "4.0.6", - "queryType": "timeseries", - "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n count(*) as ` `,\r\n ServiceName\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(TraceId IN (${trace_id:singlequote}), $trace_id)\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND ServiceName IN (${service_name:singlequote})\r\n AND SpanName IN (${span_name:singlequote})\r\n AND (($ray_id, NULL).1 = 'All' ? true : SpanAttributes['ray_id'] IN (${ray_id:singlequote}))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : SpanAttributes['workflow_id'] IN (${workflow_id:singlequote}))\r\n AND StatusCode IN ('Error', 'STATUS_CODE_ERROR')\r\n AND ServiceName != 'loadgenerator' GROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000", - "refId": "A" - } - ], - "title": "Error rates", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 49 - }, - "id": 10, - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 61 - }, - "id": 11, - "options": { - "dedupStrategy": "none", - "enableInfiniteScrolling": false, - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": true, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "builderOptions": { - "columns": [ - { - "hint": "time", - "name": "Timestamp", - "type": "DateTime64(9)" - }, - { - "hint": "log_level", - "name": "SeverityText", - "type": "LowCardinality(String)" - }, - { - "hint": "log_message", - "name": "Body", - "type": "String" - } - ], - "database": "otel", - "filters": [], - "limit": 1000, - "meta": { - "logMessageLike": "", - "otelVersion": "latest" - }, - "mode": "list", - "orderBy": [], - "queryType": "logs", - "table": "otel_logs" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorType": "builder", - "format": 2, - "pluginVersion": "4.0.6", - "rawSql": "SELECT Timestamp as timestamp, Body as body, SeverityText as level FROM \"otel\".\"otel_logs\" LIMIT 1000", - "refId": "A" - } - ], - "title": "Trace Logs", - "type": "logs" - } - ], - "title": "Logs", - "type": "row" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "ClickHouse", - "value": "ClickHouse" - }, - "includeAll": false, - "label": "ClickHouse instance", - "name": "datasource", - "options": [], - "query": "grafana-clickhouse-datasource", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ServiceName FROM otel.otel_traces", - "includeAll": true, - "label": "Service Name", - "multi": true, - "name": "service_name", - "options": [], - "query": "SELECT DISTINCT ServiceName FROM otel.otel_traces", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", - "includeAll": true, - "label": "Trace ID", - "name": "trace_id", - "options": [], - "query": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Span", - "multi": true, - "name": "span_name", - "options": [], - "query": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "allValue": "'All'", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Ray ID", - "multi": true, - "name": "ray_id", - "options": [], - "query": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "allValue": "'All'", - "current": { - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "description": "", - "includeAll": true, - "label": "Workflow ID", - "multi": true, - "name": "workflow_id", - "options": [], - "query": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND ServiceName IN (${service_name:singlequote}) LIMIT 1000;", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "America/Los_Angeles", - "title": "Traces", - "uid": "8klBUGfVk", - "version": 2, - "weekStart": "" -} \ No newline at end of file + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 21, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 9, + "panels": [], + "title": "Traces", + "type": "row" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "sql", + "format": 0, + "meta": { + "builderOptions": { + "aggregates": [ + { + "aggregateType": "count", + "column": "" + } + ], + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "groupBy": ["ServiceName"], + "limit": 10000, + "mode": "trend", + "orderBy": [], + "queryType": "timeseries", + "table": "otel_traces" + } + }, + "pluginVersion": "4.0.6", + "queryType": "timeseries", + "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n ServiceName,\r\n count() as ` `\r\nFROM otel.otel_traces\r\nWHERE\r\n ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\nGROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000\r\n", + "refId": "A" + } + ], + "title": "Traces per Service", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Trace ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 77 + }, + { + "id": "links", + "value": [ + { + "title": "__value.raw", + "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${service_name:queryparam}&var-trace_id=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Service Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + }, + { + "id": "links", + "value": [ + { + "title": "__value.raw", + "url": "/d/8klBUGfVk/otel-traces?${__url_time_range}&${trace_id:queryparam}&var-serviceName=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Duration" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge", + "valueDisplayMode": "text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 216 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Service Tags" + }, + "properties": [ + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 248 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 234 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": " " + }, + "properties": [ + { + "id": "custom.width", + "value": 49 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Span Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 140 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Ts" + }, + "properties": [ + { + "id": "custom.width", + "value": 168 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "URI/workflow_id/actor_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 920 + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Duration" + } + ] + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "limit": 100, + "mode": "list", + "orderBy": [], + "queryType": "table", + "table": "otel_traces" + } + }, + "pluginVersion": "4.9.0", + "queryType": "table", + "rawSql": "SELECT\r\n (argMin(StatusCode, Timestamp) = 'Error' ? '⚠️' : '') as ` `,\r\n min(Timestamp) as Ts,\r\n TraceId as `Trace ID`,\r\n argMin(ServiceName, Timestamp) as `Service Name`,\r\n argMin(SpanName, Timestamp) as `Span Name`,\r\n argMin(coalesce(NULLIF(SpanAttributes['uri'], ''), NULLIF(SpanAttributes['workflow_id'], ''), SpanAttributes['actor_id']), Timestamp) as `URI/workflow_id/actor_id`,\r\n divide(max(Duration), 1000000) as Duration\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id))\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\n AND ServiceName != 'loadgenerator'\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\nGROUP BY TraceId\r\nORDER BY Duration DESC\r\nLIMIT 100\r\n", + "refId": "A" + } + ], + "title": "Traces", + "type": "table" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 17, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 6, + "options": { + "spanFilters": { + "criticalPathOnly": false, + "matchesOnly": false, + "serviceNameOperator": "=", + "spanNameOperator": "=" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "builderOptions": { + "columns": [ + { + "hint": "trace_id", + "name": "TraceId" + }, + { + "hint": "trace_span_id", + "name": "SpanId" + }, + { + "hint": "trace_parent_span_id", + "name": "ParentSpanId" + }, + { + "hint": "trace_service_name", + "name": "ServiceName" + }, + { + "hint": "trace_operation_name", + "name": "SpanName" + }, + { + "hint": "time", + "name": "Timestamp" + }, + { + "hint": "trace_duration_time", + "name": "Duration" + }, + { + "hint": "trace_tags", + "name": "SpanAttributes" + }, + { + "hint": "trace_service_tags", + "name": "ResourceAttributes" + }, + { + "hint": "trace_status_code", + "name": "StatusCode" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "", + "operator": "WITH IN DASHBOARD TIME RANGE", + "type": "datetime" + }, + { + "condition": "AND", + "filterType": "custom", + "hint": "trace_duration_time", + "key": "", + "operator": ">", + "type": "UInt64", + "value": 0 + }, + { + "condition": "AND", + "filterType": "custom", + "hint": "trace_service_name", + "key": "", + "operator": "IS ANYTHING", + "type": "string", + "value": "" + } + ], + "limit": 1000, + "meta": { + "isTraceIdMode": true, + "otelEnabled": true, + "otelVersion": "latest", + "traceDurationUnit": "nanoseconds", + "traceId": "${trace_id}" + }, + "mode": "list", + "orderBy": [ + { + "default": true, + "dir": "DESC", + "hint": "time", + "name": "" + }, + { + "default": true, + "dir": "DESC", + "hint": "trace_duration_time", + "name": "" + } + ], + "queryType": "traces", + "table": "otel_traces" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "builder", + "format": 3, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 100, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.0.6", + "queryType": "traces", + "rawSql": "WITH\n\t(SELECT min(Start) FROM otel.otel_traces_trace_id_ts WHERE $__conditionalAll(TraceId, $trace_id)) as trace_start,\n\t(SELECT max(End) + 1 FROM otel.otel_traces_trace_id_ts WHERE $__conditionalAll(TraceId, $trace_id)) as trace_end\nSELECT\n\tTraceId as traceID,\n\tSpanId as spanID,\n\tParentSpanId as parentSpanID,\n\tServiceName as serviceName,\n\tSpanName as operationName, Timestamp as startTime,\n\tmultiply(Duration, 0.000001) as duration,\n\tarrayMap(key -> map('key', key, 'value', SpanAttributes[key]), mapKeys(SpanAttributes)) as tags,\n\tarrayMap(key -> map('key', key, 'value', ResourceAttributes[key]), mapKeys(ResourceAttributes)) as serviceTags,\n\tarrayMap((name, timestamp, attributes) -> tuple(name, toString(multiply(toUnixTimestamp64Nano(timestamp), 0.000001)), arrayMap( key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(name String, timestamp String, fields Array(Map(String, String))), `Events.Name`, `Events.Timestamp`, `Events.Attributes`) AS logs,\n\tarrayMap((traceID, spanID, attributes) -> tuple(traceID, spanID, arrayMap(key -> map('key', key, 'value', attributes[key]), mapKeys(attributes)))::Tuple(traceID String, spanID String, tags Array(Map(String, String))), `Links.TraceId`, `Links.SpanId`, `Links.Attributes`) AS references\nFROM otel.otel_traces\nWHERE\n\t$__conditionalAll(traceID, $trace_id) AND startTime >= trace_start AND startTime <= trace_end AND ( Duration > 0 )\nORDER BY Timestamp DESC, Duration DESC\nLIMIT 1000", + "refId": "A" + } + ], + "title": "Trace Details", + "type": "traces" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 24, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse-traces" + }, + "editorType": "sql", + "format": 0, + "meta": { + "builderOptions": { + "aggregates": [ + { + "aggregateType": "count", + "column": "" + } + ], + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + } + ], + "database": "otel", + "filters": [ + { + "condition": "AND", + "filterType": "custom", + "hint": "time", + "key": "Timestamp", + "operator": "WITH IN DASHBOARD TIME RANGE", + "restrictToFields": [ + { + "label": "Timestamp", + "name": "Timestamp", + "picklistValues": [], + "type": "DateTime64(9)" + } + ], + "type": "datetime" + } + ], + "groupBy": ["ServiceName"], + "limit": 10000, + "mode": "trend", + "orderBy": [], + "queryType": "timeseries", + "table": "otel_traces" + } + }, + "pluginVersion": "4.0.6", + "queryType": "timeseries", + "rawSql": "SELECT\r\n $__timeInterval(Timestamp) as time,\r\n count(*) as ` `,\r\n ServiceName\r\nFROM otel.otel_traces\r\nWHERE\r\n $__conditionalAll(TraceId, $trace_id)\r\n AND ( Timestamp >= $__fromTime AND Timestamp <= $__toTime )\r\n AND $__conditionalAll(ServiceName, $service_name)\r\n AND $__conditionalAll(SpanName, $span_name)\r\n AND (($ray_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['ray_id'], $ray_id)\r\n AND (($workflow_id, NULL).1 = 'All' ? true : $__conditionalAll(SpanAttributes['workflow_id'], $workflow_id))\r\n AND StatusCode IN ('Error', 'STATUS_CODE_ERROR')\r\n AND ServiceName != 'loadgenerator' GROUP BY ServiceName, time\r\nORDER BY time ASC\r\nLIMIT 100000", + "refId": "A" + } + ], + "title": "Error rates", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 10, + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 11, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "builderOptions": { + "columns": [ + { + "hint": "time", + "name": "Timestamp", + "type": "DateTime64(9)" + }, + { + "hint": "log_level", + "name": "SeverityText", + "type": "LowCardinality(String)" + }, + { + "hint": "log_message", + "name": "Body", + "type": "String" + } + ], + "database": "otel", + "filters": [], + "limit": 1000, + "meta": { + "logMessageLike": "", + "otelVersion": "latest" + }, + "mode": "list", + "orderBy": [], + "queryType": "logs", + "table": "otel_logs" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "editorType": "builder", + "format": 2, + "pluginVersion": "4.0.6", + "rawSql": "SELECT Timestamp as timestamp, Body as body, SeverityText as level FROM \"otel\".\"otel_logs\" LIMIT 1000", + "refId": "A" + } + ], + "title": "Trace Logs", + "type": "logs" + } + ], + "title": "Logs", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "ClickHouse", + "value": "ClickHouse" + }, + "includeAll": false, + "label": "ClickHouse instance", + "name": "datasource", + "options": [], + "query": "grafana-clickhouse-datasource", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT ServiceName FROM otel.otel_traces", + "includeAll": true, + "label": "Service Name", + "multi": true, + "name": "service_name", + "options": [], + "query": "SELECT DISTINCT ServiceName FROM otel.otel_traces", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", + "includeAll": true, + "label": "Trace ID", + "name": "trace_id", + "options": [], + "query": "SELECT DISTINCT TraceId FROM otel.otel_traces WHERE ParentSpanId = '' LIMIT 100", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Span", + "multi": true, + "name": "span_name", + "options": [], + "query": "SELECT DISTINCT SpanName FROM otel.otel_traces WHERE $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "'All'", + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Ray ID", + "multi": true, + "name": "ray_id", + "options": [], + "query": "SELECT DISTINCT SpanAttributes['ray_id'] FROM otel.otel_traces WHERE SpanAttributes['ray_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "allValue": "'All'", + "current": { + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "clickhouse" + }, + "definition": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "description": "", + "includeAll": true, + "label": "Workflow ID", + "multi": true, + "name": "workflow_id", + "options": [], + "query": "SELECT DISTINCT SpanAttributes['workflow_id'] FROM otel.otel_traces WHERE SpanAttributes['workflow_id'] != '' AND $__conditionalAll(ServiceName, $service_name) LIMIT 1000;", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "30", + "value": "30" + }, + "hide": 2, + "label": "Metric Export Interval (seconds)", + "name": "metric_interval", + "query": "30", + "skipUrlSync": true, + "type": "constant" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "America/Los_Angeles", + "title": "Traces", + "uid": "8klBUGfVk", + "version": 2, + "weekStart": "" +} diff --git a/packages/common/metrics/src/providers.rs b/packages/common/metrics/src/providers.rs index 15dec7201f..db8d62e5c5 100644 --- a/packages/common/metrics/src/providers.rs +++ b/packages/common/metrics/src/providers.rs @@ -1,9 +1,8 @@ // Based off of https://github.com/tokio-rs/tracing-opentelemetry/blob/v0.1.x/examples/opentelemetry-otlp.rs // Based off of https://github.com/tokio-rs/tracing-opentelemetry/blob/v0.1.x/examples/opentelemetry-otlp.rs -use std::sync::{Arc, RwLock, OnceLock}; -use opentelemetry::{KeyValue, global}; use opentelemetry::trace::{SamplingResult, SpanKind}; +use opentelemetry::{KeyValue, global}; use opentelemetry_otlp::WithExportConfig; use opentelemetry_sdk::{ Resource, @@ -12,6 +11,7 @@ use opentelemetry_sdk::{ trace::{RandomIdGenerator, Sampler, SdkTracerProvider}, }; use opentelemetry_semantic_conventions::{SCHEMA_URL, attribute::SERVICE_VERSION}; +use std::sync::{Arc, OnceLock, RwLock}; /// Dynamic sampler that can be updated at runtime #[derive(Clone, Debug)] @@ -133,7 +133,7 @@ fn init_tracer_provider() -> SdkTracerProvider { fn init_meter_provider() -> SdkMeterProvider { let exporter = opentelemetry_otlp::MetricExporter::builder() .with_tonic() - .with_temporality(opentelemetry_sdk::metrics::Temporality::default()) + .with_temporality(opentelemetry_sdk::metrics::Temporality::Delta) .with_protocol(opentelemetry_otlp::Protocol::Grpc) .with_endpoint(otel_grpc_endpoint()) .build() diff --git a/packages/common/runtime/src/traces.rs b/packages/common/runtime/src/traces.rs index cd885392ff..9d10924bef 100644 --- a/packages/common/runtime/src/traces.rs +++ b/packages/common/runtime/src/traces.rs @@ -26,8 +26,8 @@ pub fn init_tracing_subscriber(otel_providers: &Option) { Some(providers) => { let tracer = providers.tracer_provider.tracer("tracing-otel-subscriber"); - let otel_trace_layer = - OpenTelemetryLayer::new(tracer).with_filter(build_filter_from_env_var("RUST_TRACE")); + let otel_trace_layer = OpenTelemetryLayer::new(tracer) + .with_filter(build_filter_from_env_var("RUST_TRACE")); let otel_metric_layer = MetricsLayer::new(providers.meter_provider.clone()) .with_filter(build_filter_from_env_var("RUST_TRACE")); diff --git a/packages/core/api-peer/src/internal.rs b/packages/core/api-peer/src/internal.rs index 679c8a98c1..14ad07d884 100644 --- a/packages/core/api-peer/src/internal.rs +++ b/packages/core/api-peer/src/internal.rs @@ -47,6 +47,7 @@ pub async fn bump_serverless_autoscaler( } #[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct SetTracingConfigRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub filter: Option>, @@ -55,6 +56,7 @@ pub struct SetTracingConfigRequest { } #[derive(Serialize)] +#[serde(deny_unknown_fields)] pub struct SetTracingConfigResponse {} #[tracing::instrument(skip_all)] diff --git a/packages/core/api-public/src/actors/get_or_create.rs b/packages/core/api-public/src/actors/get_or_create.rs index 7b113f564d..8f0403eef3 100644 --- a/packages/core/api-public/src/actors/get_or_create.rs +++ b/packages/core/api-public/src/actors/get_or_create.rs @@ -4,8 +4,8 @@ use axum::{ response::{IntoResponse, Response}, }; use rivet_api_builder::{ - extract::{Extension, Json, Query}, ApiError, + extract::{Extension, Json, Query}, }; use rivet_types::actors::CrashPolicy; use rivet_util::Id; diff --git a/packages/core/api-public/src/actors/utils.rs b/packages/core/api-public/src/actors/utils.rs index b6186fe5c7..3fa74e13f7 100644 --- a/packages/core/api-public/src/actors/utils.rs +++ b/packages/core/api-public/src/actors/utils.rs @@ -10,11 +10,7 @@ use std::collections::HashMap; /// Helper function to fetch an actor by ID, automatically routing to the correct datacenter /// based on the actor ID's label. #[tracing::instrument(skip_all)] -pub async fn fetch_actor_by_id( - ctx: &ApiCtx, - actor_id: Id, - namespace: String, -) -> Result { +pub async fn fetch_actor_by_id(ctx: &ApiCtx, actor_id: Id, namespace: String) -> Result { let list_query = rivet_api_types::actors::list::ListQuery { namespace, actor_ids: Some(actor_id.to_string()), diff --git a/packages/core/api-public/src/namespaces.rs b/packages/core/api-public/src/namespaces.rs index 1afa48b336..385ed54658 100644 --- a/packages/core/api-public/src/namespaces.rs +++ b/packages/core/api-public/src/namespaces.rs @@ -21,10 +21,7 @@ use crate::ctx::ApiCtx; security(("bearer_auth" = [])), )] #[tracing::instrument(skip_all)] -pub async fn list( - Extension(ctx): Extension, - Query(query): Query, -) -> Response { +pub async fn list(Extension(ctx): Extension, Query(query): Query) -> Response { match list_inner(ctx, query).await { Ok(response) => Json(response).into_response(), Err(err) => ApiError::from(err).into_response(), diff --git a/packages/core/api-public/src/runners.rs b/packages/core/api-public/src/runners.rs index 2b19109893..9d752fa3ec 100644 --- a/packages/core/api-public/src/runners.rs +++ b/packages/core/api-public/src/runners.rs @@ -22,10 +22,7 @@ use crate::ctx::ApiCtx; security(("bearer_auth" = [])), )] #[tracing::instrument(skip_all)] -pub async fn list( - Extension(ctx): Extension, - Query(query): Query, -) -> Response { +pub async fn list(Extension(ctx): Extension, Query(query): Query) -> Response { match list_inner(ctx, query).await { Ok(response) => Json(response).into_response(), Err(err) => ApiError::from(err).into_response(), diff --git a/packages/infra/engine/src/commands/start.rs b/packages/infra/engine/src/commands/start.rs index 0ef31b6843..d25735f9b8 100644 --- a/packages/infra/engine/src/commands/start.rs +++ b/packages/infra/engine/src/commands/start.rs @@ -11,6 +11,10 @@ const LOGS_RETENTION: Duration = Duration::from_secs(7 * 24 * 60 * 60); pub struct Opts { #[arg(long, value_enum)] services: Vec, + + /// Exclude the specified services instead of including them + #[arg(long)] + exclude_services: bool, } #[derive(clap::ValueEnum, Clone, PartialEq)] @@ -50,7 +54,7 @@ impl Opts { .await?; } - // Select services t orun + // Select services to run let services = if self.services.is_empty() { // Run all services run_config.services.clone() @@ -62,12 +66,23 @@ impl Opts { .map(|x| x.clone().into()) .collect::>(); - run_config - .services - .iter() - .filter(|x| service_kinds.iter().any(|y| y.eq(&x.kind))) - .cloned() - .collect::>() + if self.exclude_services { + // Exclude specified services + run_config + .services + .iter() + .filter(|x| !service_kinds.iter().any(|y| y.eq(&x.kind))) + .cloned() + .collect::>() + } else { + // Include only specified services + run_config + .services + .iter() + .filter(|x| service_kinds.iter().any(|y| y.eq(&x.kind))) + .cloned() + .collect::>() + } }; // Start server diff --git a/packages/infra/engine/src/run_config.rs b/packages/infra/engine/src/run_config.rs index e1253e7e95..a149e9a257 100644 --- a/packages/infra/engine/src/run_config.rs +++ b/packages/infra/engine/src/run_config.rs @@ -22,9 +22,11 @@ pub fn config(_rivet_config: rivet_config::Config) -> Result { ServiceKind::Standalone, |config, pools| Box::pin(pegboard_serverless::start(config, pools)), ), - Service::new("tracing_reconfigure", ServiceKind::Standalone, |config, pools| { - Box::pin(rivet_tracing_reconfigure::start(config, pools)) - }), + Service::new( + "tracing_reconfigure", + ServiceKind::Standalone, + |config, pools| Box::pin(rivet_tracing_reconfigure::start(config, pools)), + ), ]; Ok(RunConfigData { services }) diff --git a/packages/services/epoxy/src/http_client.rs b/packages/services/epoxy/src/http_client.rs index 5d94b230e1..779670c57c 100644 --- a/packages/services/epoxy/src/http_client.rs +++ b/packages/services/epoxy/src/http_client.rs @@ -5,6 +5,7 @@ use epoxy_protocol::{ versioned, }; use futures_util::{StreamExt, stream::FuturesUnordered}; +use gas::prelude::*; use rivet_api_builder::ApiCtx; use std::future::Future; use vbare::OwnedVersionedData; @@ -24,6 +25,7 @@ fn find_replica_address( .map(|r| r.api_peer_url.clone()) } +#[tracing::instrument(skip_all, fields(%from_replica_id, ?replica_ids, ?quorum_type))] pub async fn fanout_to_replicas( from_replica_id: ReplicaId, replica_ids: &[ReplicaId], @@ -93,6 +95,7 @@ where Ok(successful_responses) } +#[tracing::instrument(skip_all)] pub async fn send_message( ctx: &ApiCtx, config: &protocol::ClusterConfig, @@ -102,6 +105,7 @@ pub async fn send_message( send_message_to_address(ctx, replica_url, request).await } +#[tracing::instrument(skip_all, fields(%replica_url))] pub async fn send_message_to_address( ctx: &ApiCtx, replica_url: String, @@ -116,8 +120,7 @@ pub async fn send_message_to_address( "sending message to replica directly" ); - return crate::replica::message_request::message_request(&ctx, from_replica_id, request) - .await; + return crate::replica::message_request::message_request(&ctx, request).await; } let mut replica_url = url::Url::parse(&replica_url)?; @@ -139,6 +142,7 @@ pub async fn send_message_to_address( .post(replica_url.to_string()) .body(request.serialize()?) .send() + .custom_instrument(tracing::info_span!("http_request")) .await; let response = match response_result { diff --git a/packages/services/epoxy/src/http_routes.rs b/packages/services/epoxy/src/http_routes.rs index 50777228b3..79ba36d00d 100644 --- a/packages/services/epoxy/src/http_routes.rs +++ b/packages/services/epoxy/src/http_routes.rs @@ -28,8 +28,7 @@ pub async fn message(ctx: ApiCtx, path: VersionedPath, _query: (), body: Bytes) ); // Process message directly using ops - let response = - crate::replica::message_request::message_request(&ctx, current_replica_id, request).await?; + let response = crate::replica::message_request::message_request(&ctx, request).await?; versioned::Response::latest(response).serialize(path.version) } diff --git a/packages/services/epoxy/src/ops/kv/get_optimistic.rs b/packages/services/epoxy/src/ops/kv/get_optimistic.rs index a47236ac92..a05a84dd89 100644 --- a/packages/services/epoxy/src/ops/kv/get_optimistic.rs +++ b/packages/services/epoxy/src/ops/kv/get_optimistic.rs @@ -86,9 +86,7 @@ pub async fn epoxy_kv_get_optimistic(ctx: &OperationCtx, input: &Input) -> Resul // Request fanout to other datacenters, return first datacenter with any non-none value let config = ctx - .op(crate::ops::read_cluster_config::Input { - replica_id: input.replica_id, - }) + .op(crate::ops::read_cluster_config::Input {}) .await? .config; diff --git a/packages/services/epoxy/src/ops/read_cluster_config.rs b/packages/services/epoxy/src/ops/read_cluster_config.rs index 3440782a5c..62a23c2092 100644 --- a/packages/services/epoxy/src/ops/read_cluster_config.rs +++ b/packages/services/epoxy/src/ops/read_cluster_config.rs @@ -5,9 +5,7 @@ use gas::prelude::*; use crate::utils; #[derive(Debug)] -pub struct Input { - pub replica_id: ReplicaId, -} +pub struct Input {} #[derive(Debug)] pub struct Output { @@ -18,10 +16,7 @@ pub struct Output { pub async fn epoxy_read_cluster_config(ctx: &OperationCtx, input: &Input) -> Result { let config = ctx .udb()? - .run(|tx| { - let replica_id = input.replica_id; - async move { utils::read_config(&tx, replica_id).await } - }) + .run(|tx| async move { utils::read_config(&tx, ctx.config().epoxy_replica_id()).await }) .custom_instrument(tracing::info_span!("read_cluster_config_tx")) .await?; diff --git a/packages/services/epoxy/src/replica/message_request.rs b/packages/services/epoxy/src/replica/message_request.rs index dd63e9d4b3..4e41d1abf1 100644 --- a/packages/services/epoxy/src/replica/message_request.rs +++ b/packages/services/epoxy/src/replica/message_request.rs @@ -8,9 +8,10 @@ use crate::{ops, replica}; #[tracing::instrument(skip_all)] pub async fn message_request( ctx: &ApiCtx, - replica_id: ReplicaId, request: protocol::Request, ) -> Result { + let current_replica_id = ctx.config().epoxy_replica_id(); + let kind = match request.kind { protocol::RequestKind::UpdateConfigRequest(req) => { tracing::info!( @@ -23,7 +24,7 @@ pub async fn message_request( ctx.udb()? .run(|tx| { let req = req.clone(); - async move { replica::update_config::update_config(&*tx, replica_id, req) } + async move { replica::update_config::update_config(&*tx, current_replica_id, req) } }) .custom_instrument(tracing::info_span!("update_config_tx")) .await?; @@ -35,7 +36,7 @@ pub async fn message_request( .udb()? .run(|tx| { let req = req.clone(); - async move { replica::messages::pre_accept(&*tx, replica_id, req).await } + async move { replica::messages::pre_accept(&*tx, current_replica_id, req).await } }) .custom_instrument(tracing::info_span!("pre_accept_tx")) .await?; @@ -46,7 +47,7 @@ pub async fn message_request( .udb()? .run(|tx| { let req = req.clone(); - async move { replica::messages::accept(&*tx, replica_id, req).await } + async move { replica::messages::accept(&*tx, current_replica_id, req).await } }) .custom_instrument(tracing::info_span!("accept_tx")) .await?; @@ -58,7 +59,7 @@ pub async fn message_request( .run(|tx| { let req = req.clone(); async move { - replica::messages::commit(&*tx, replica_id, req, true).await?; + replica::messages::commit(&*tx, current_replica_id, req, true).await?; Result::Ok(()) } }) @@ -72,7 +73,7 @@ pub async fn message_request( .udb()? .run(|tx| { let req = req.clone(); - async move { replica::messages::prepare(&*tx, replica_id, req).await } + async move { replica::messages::prepare(&*tx, current_replica_id, req).await } }) .custom_instrument(tracing::info_span!("prepare_tx")) .await?; @@ -84,7 +85,9 @@ pub async fn message_request( .udb()? .run(|tx| { let req = req.clone(); - async move { replica::messages::download_instances(&*tx, replica_id, req).await } + async move { + replica::messages::download_instances(&*tx, current_replica_id, req).await + } }) .custom_instrument(tracing::info_span!("download_instances_tx")) .await?; @@ -101,9 +104,9 @@ pub async fn message_request( protocol::RequestKind::CoordinatorUpdateReplicaStatusRequest(req) => { // Send signal to coordinator workflow tracing::info!( - ?replica_id, - update_replica_id = ?req.replica_id, - update_status = ?req.status, + ?current_replica_id, + update_replica_id=?req.replica_id, + update_status=?req.status, "received coordinator update replica status request" ); @@ -113,7 +116,7 @@ pub async fn message_request( }) .bypass_signal_from_workflow_I_KNOW_WHAT_IM_DOING() .to_workflow::() - .tag("replica", replica_id) + .tag("replica", current_replica_id) .send() .await?; @@ -121,17 +124,14 @@ pub async fn message_request( } protocol::RequestKind::BeginLearningRequest(req) => { // Send signal to replica workflow - tracing::info!( - replica_id = ?replica_id, - "received begin learning request" - ); + tracing::info!(?current_replica_id, "received begin learning request"); ctx.signal(crate::workflows::replica::BeginLearning { config: req.config.clone().into(), }) .bypass_signal_from_workflow_I_KNOW_WHAT_IM_DOING() .to_workflow::() - .tag("replica", replica_id) + .tag("replica", current_replica_id) .send() .await?; @@ -141,7 +141,7 @@ pub async fn message_request( // Handle KV get request let result = ctx .op(ops::kv::get_local::Input { - replica_id, + replica_id: current_replica_id, key: req.key.clone(), }) .await?; diff --git a/scripts/decode_audit_entry.js b/scripts/decode_audit_entry.js index 9afa6123db..246e904707 100644 --- a/scripts/decode_audit_entry.js +++ b/scripts/decode_audit_entry.js @@ -13,7 +13,7 @@ const buffer = Buffer.from(hexString, 'hex'); const version = buffer.readUInt16LE(0); const dataBuffer = buffer.slice(2); -console.log('Version:', version); +console.log('Embedded VBARE Version:', version); class BareDecoder { constructor(buffer) { @@ -60,9 +60,17 @@ class BareDecoder { return readFn.call(this); } - readUnion(variants) { + readUnion(variants, tagNames) { const tag = this.readUint(); - return { tag, value: variants[tag].call(this) }; + const value = variants[tag].call(this); + const result = { + tag: tagNames ? tagNames[tag] : tag + }; + // Only include value if it's not void (undefined/null or the string representation) + if (value !== undefined && value !== null && value !== 'Any') { + result.value = value; + } + return result; } } @@ -70,7 +78,6 @@ class BareDecoder { const decoder = new BareDecoder(dataBuffer); console.log('Decoding audit entry from hex:', hexString); -console.log('Buffer length:', dataBuffer.length, 'bytes\n'); // Data struct const data = {}; @@ -83,7 +90,7 @@ const namespace = decoder.readUnion([ () => 'Any', // 0: Any (void) () => decoder.readData(), // 1: Id () => decoder.readData().toString('utf8') // 2: Name -]); +], ['Any', 'Id', 'Name']); data.request.namespace = namespace; // resource: ResourceKind enum @@ -94,7 +101,7 @@ data.request.resource = resourceKinds[decoder.readEnum()]; const target = decoder.readUnion([ () => 'Any', // 0: Any (void) () => decoder.readData() // 1: Id -]); +], ['Any', 'Id']); data.request.target = target; // operation: OperationKind enum