diff --git a/deploy/infrabox/grafana/dashboards/api_performance_dashboard.json b/deploy/infrabox/grafana/dashboards/api_performance_dashboard.json new file mode 100644 index 000000000..b4e58a363 --- /dev/null +++ b/deploy/infrabox/grafana/dashboards/api_performance_dashboard.json @@ -0,0 +1,594 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "API performance and project activity overview. Tracks which projects generate the most load and (once Flask metrics are enabled) which endpoints are called most frequently.", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "Project Activity", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "Allocated CPU cores per project across all clusters. Proxy for which projects are driving compute load.", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 2, + "interval": "10s", + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rsc_current_count{cluster=\"'%'\", rsc=\"cpu\", project!=\"'%'\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{project}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Allocation by Project", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": "CPU cores", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "infrabox", + "description": "Most active projects in the selected time window, ranked by job count.", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "links": [], + "pageSize": 15, + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Project", + "mappingType": 1, + "pattern": "name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Builds", + "decimals": 0, + "mappingType": 1, + "pattern": "builds", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Jobs", + "decimals": 0, + "mappingType": 1, + "pattern": "jobs", + "thresholds": [ + "50", + "200" + ], + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Running", + "decimals": 0, + "mappingType": 1, + "pattern": "running", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Queued", + "decimals": 0, + "mappingType": 1, + "pattern": "queued", + "thresholds": [ + "5", + "20" + ], + "colorMode": "cell", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "rawSql": "SELECT p.name,\n count(DISTINCT b.id) AS builds,\n count(j.id) AS jobs,\n count(j.id) FILTER (WHERE j.state = 'running') AS running,\n count(j.id) FILTER (WHERE j.state = 'queued') AS queued\nFROM project p\nJOIN build b ON b.project_id = p.id\nJOIN job j ON j.build_id = b.id\nWHERE j.created_at BETWEEN $__timeFrom() AND $__timeTo()\nGROUP BY p.name\nORDER BY jobs DESC\nLIMIT 20", + "refId": "A" + } + ], + "title": "Most Active Projects", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "infrabox", + "description": "Number of builds submitted per minute per project. High submission rates indicate projects actively triggering CI.", + "fill": 1, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 4, + "interval": null, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "rawSql": "SELECT\n $__timeGroup(j.created_at, '1m') AS time,\n p.name AS metric,\n count(DISTINCT b.id) AS value\nFROM job j\nJOIN build b ON b.id = j.build_id\nJOIN project p ON p.id = b.project_id\nWHERE j.created_at BETWEEN $__timeFrom() AND $__timeTo()\nGROUP BY time, p.name\nORDER BY time", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Build Submission Rate by Project (per minute)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "Builds / min", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 10, + "title": "API Request Metrics (requires Flask middleware \u2014 see DM01-6243)", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "API requests per second per endpoint. Requires infrabox_api_requests_total metric (Flask before_request middleware). Shows 'No data' until middleware is deployed.", + "fill": 1, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 18 + }, + "id": 11, + "interval": "30s", + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(15, sum by (endpoint, method) (rate(infrabox_api_requests_total[5m])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{method}} {{endpoint}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "API Request Rate by Endpoint (req/s)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "reqps", + "label": "Requests/s", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "prometheus", + "description": "Top endpoints by total request count in the selected time window. Requires infrabox_api_requests_total metric.", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 18 + }, + "id": 12, + "links": [], + "pageSize": 15, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Endpoint", + "mappingType": 1, + "pattern": "endpoint", + "type": "string", + "unit": "short" + }, + { + "alias": "Requests", + "decimals": 0, + "mappingType": 1, + "pattern": "Value", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "topk(15, sum by (endpoint) (increase(infrabox_api_requests_total[$__range])))", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Top Endpoints by Volume", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "description": "95th percentile API response latency per endpoint. Requires infrabox_api_request_duration_seconds histogram metric.", + "fill": 1, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 13, + "interval": "30s", + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10, histogram_quantile(0.95, sum by (endpoint, le) (rate(infrabox_api_request_duration_seconds_bucket[5m]))))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p95 {{endpoint}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 1 + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 5 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "API Latency p95 by Endpoint", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": "Response time (s)", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5m", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "infrabox", + "performance", + "api" + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "1h" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d" + ] + }, + "timezone": "browser", + "title": "InfraBox API Performance", + "uid": "infrabox-api-perf", + "version": 1 +} \ No newline at end of file diff --git a/src/db/migrations/00049.sql b/src/db/migrations/00049.sql new file mode 100644 index 000000000..4381d74bb --- /dev/null +++ b/src/db/migrations/00049.sql @@ -0,0 +1,6 @@ +-- Add index on job.created_at to speed up time-range queries. +-- +-- Without this index, any query filtering by created_at (e.g. Grafana +-- dashboards, build activity reports) performs a full sequential scan on the +-- job table. +CREATE INDEX IF NOT EXISTS job_created_at_idx ON job USING btree (created_at);