{ "__inputs": [ { "name": "DS_ACE_PROMETHEUS", "label": "ACE Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "4.3.1" }, { "type": "panel", "id": "graph", "name": "Graph", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } ], "annotations": { "list": [] }, "editable": true, "gnetId": null, "graphTooltip": 0, "hideControls": false, "id": null, "links": [], "refresh": false, "rows": [ { "collapse": false, "height": 250, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_ACE_PROMETHEUS}", "fill": 1, "id": 1, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 12, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(ace_content_service_read_duration_seconds_bucket[5m])) by (operation,le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{operation}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "READ duration s (95%)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "READ", "titleSize": "h6" }, { "collapse": false, "height": 250, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_ACE_PROMETHEUS}", "fill": 1, "id": 2, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 12, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(ace_content_service_read_duration_seconds_bucket{operation=~\"getVersion|getRevision\"}[5m])) by (operation,variant,le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{operation}} {{variant}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "READ variant duration s (95%)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "READ Variant", "titleSize": "h6" }, { "collapse": false, "height": 250, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_ACE_PROMETHEUS}", "fill": 1, "id": 3, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 12, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(ace_content_service_write_duration_seconds_bucket[5m])) by (operation,le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{operation}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "WRITE duration s (95%)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "WRITE", "titleSize": "h6" }, { "collapse": false, "height": 250, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_ACE_PROMETHEUS}", "fill": 1, "id": 10, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 12, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(ace_content_service_import_duration_seconds_bucket[5m])) by (operation,le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{operation}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import duration s (95%)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": false, "title": "Dashboard Row", "titleSize": "h6" }, { "collapse": false, "height": 250, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_ACE_PROMETHEUS}", "fill": 1, "id": 11, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 12, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(ace_content_service_meta_duration_seconds_bucket[5m])) by (operation,le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{operation}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Meta read duration s (95%)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": false, "title": "Dashboard Row", "titleSize": "h6" }, { "collapse": true, "height": 250, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_ACE_PROMETHEUS}", "fill": 1, "id": 4, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 12, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(irate(ace_contentstorage_cache_requests_total{cache_status=\"hit\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "hit", "refId": "A", "step": 2 }, { "expr": "sum(irate(ace_contentstorage_cache_requests_total{cache_status=\"miss\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "miss", "refId": "B", "step": 2 }, { "expr": "sum(irate(ace_contentstorage_cache_requests_total{cache_status=\"update\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "update", "refId": "C", "step": 2 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage Cache", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Cache", "titleSize": "h6" }, { "collapse": true, "height": 411, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_ACE_PROMETHEUS}", "fill": 1, "id": 1, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideZero": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": true, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(increase(ace_read_pipeline_duration_seconds_sum[1m])) by (stage)/clamp_min(sum(increase(ace_read_pipeline_duration_seconds_count[1m])) by (stage), 1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{stage}}", "metric": "ace_read_pipeline_duration_seconds_count", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Pipeline stages", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_ACE_PROMETHEUS}", "fill": 1, "id": 2, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "1 - (sum(increase(ace_read_pipeline_duration_seconds_bucket{le=\"1.0\"}[1m])) by (stage)/sum(increase(ace_read_pipeline_duration_seconds_bucket{le=\"+Inf\"}[1m])) by (stage))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{stage}}", "metric": "ace_read_pipeline_duration_seconds_bucket", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "> 1s execute time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Read pipeline", "titleSize": "h6" }, { "collapse": true, "height": 250, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_ACE_PROMETHEUS}", "fill": 1, "id": 3, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.9, sum(rate(ace_read_pipeline_duration_seconds_bucket[1m])) by (variant, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{variant}}", "refId": "A", "step": 2 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "90% read latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_ACE_PROMETHEUS}", "fill": 1, "id": 4, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(ace_read_pipeline_duration_seconds_bucket[10m])) by (variant, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{variant}}", "refId": "A", "step": 2 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% read latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Read Latency", "titleSize": "h6" }, { "collapse": true, "height": 380, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_ACE_PROMETHEUS}", "fill": 1, "id": 8, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideZero": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(increase(ace_write_pipeline_duration_seconds_sum[1m])) by (callback)/clamp_min(sum(increase(ace_write_pipeline_duration_seconds_count[1m])) by (callback), 1)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{callback}}", "metric": "ace_read_pipeline_duration_seconds_count", "refId": "A", "step": 2 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write pipeline stages", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_ACE_PROMETHEUS}", "fill": 1, "id": 9, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "1 - (sum(increase(ace_write_pipeline_duration_seconds_bucket{le=\"1.0\"}[1m])) by (callback)/sum(increase(ace_write_pipeline_duration_seconds_bucket{le=\"+Inf\"}[1m])) by (callback))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{callback}}", "metric": "ace_read_pipeline_duration_seconds_bucket", "refId": "A", "step": 2 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "> 1s execute time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Write Pipeline", "titleSize": "h6" }, { "collapse": true, "height": 250, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_ACE_PROMETHEUS}", "fill": 1, "id": 5, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.9, sum(rate(ace_write_pipeline_duration_seconds_bucket[1m]) > 0.1) by (contentType, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{contentType}}", "refId": "A", "step": 2 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "90% write latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_ACE_PROMETHEUS}", "fill": 1, "id": 6, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(ace_write_pipeline_duration_seconds_bucket[1m]) > 0.1) by (contentType, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{contentType}}", "refId": "A", "step": 2 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% write latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Write Latency", "titleSize": "h6" } ], "schemaVersion": 14, "style": "dark", "tags": [], "templating": { "list": [] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "Content Service", "version": 8 }