From 4d00b5e32be16c41f9fa23c5d5d7ff7b10b8b113 Mon Sep 17 00:00:00 2001 From: JSLMPR Date: Fri, 10 Jul 2026 00:14:13 +0200 Subject: [PATCH] I added an importable Grafana dashboard at dashboards/video-clipping-overview-grafana.json:1 and the companion setup note in docs/dashboards.md:1. The dashboard covers queue backlog and age, DLQ and processing counts, API request rate and p95 latency, plus CPU and heap usage using the Prometheus and Micrometer metrics already exposed by the service. --- .idea/vcs.xml | 2 +- .../video-clipping-overview-grafana.json | 642 ++++++++++++++++++ docs/dashboards.md | 33 + docs/grafana-dashboard-creation-prompt.md | 239 +++++++ ...eo-clipping-service-implementation-plan.md | 2 +- .../uploads/upl_367df9ed2414/source | 1 - 6 files changed, 916 insertions(+), 3 deletions(-) create mode 100644 dashboards/video-clipping-overview-grafana.json create mode 100644 docs/dashboards.md create mode 100644 docs/grafana-dashboard-creation-prompt.md delete mode 100644 tmp/ffmpeg-input/uploads/upl_367df9ed2414/source diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 94a25f7..35eb1dd 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -1,6 +1,6 @@ - + \ No newline at end of file diff --git a/dashboards/video-clipping-overview-grafana.json b/dashboards/video-clipping-overview-grafana.json new file mode 100644 index 0000000..a976bd4 --- /dev/null +++ b/dashboards/video-clipping-overview-grafana.json @@ -0,0 +1,642 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(video_clipping_queue_pending)", + "legendFormat": "pending", + "range": true, + "refId": "A" + } + ], + "title": "Pending Queue Depth", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 30 + }, + { + "color": "red", + "value": 120 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max(video_clipping_queue_oldest_pending_age_seconds)", + "legendFormat": "oldest pending age", + "range": true, + "refId": "A" + } + ], + "title": "Oldest Pending Age", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(video_clipping_queue_dlq)", + "legendFormat": "dlq", + "range": true, + "refId": "A" + } + ], + "title": "DLQ Messages", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(video_clipping_queue_processing)", + "legendFormat": "processing", + "range": true, + "refId": "A" + } + ], + "title": "Processing Messages", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(video_clipping_queue_pending)", + "legendFormat": "pending", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(video_clipping_queue_processing)", + "legendFormat": "processing", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(video_clipping_queue_dlq)", + "legendFormat": "dlq", + "range": true, + "refId": "C" + } + ], + "title": "Queue State Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max(video_clipping_queue_oldest_pending_age_seconds)", + "legendFormat": "oldest pending age", + "range": true, + "refId": "A" + } + ], + "title": "Queue Age", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(http_server_requests_seconds_count{uri!~\"/actuator.*\"}[5m])) by (method, status)", + "legendFormat": "{{method}} {{status}}", + "range": true, + "refId": "A" + } + ], + "title": "API Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(http_server_requests_seconds_bucket{uri!~\"/actuator.*\"}[5m])) by (le, uri, method))", + "legendFormat": "{{method}} {{uri}} p95", + "range": true, + "refId": "A" + } + ], + "title": "API p95 Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "system_cpu_usage", + "legendFormat": "system cpu", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "process_cpu_usage", + "legendFormat": "process cpu", + "range": true, + "refId": "B" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "jvm_memory_used_bytes{area=\"heap\"}", + "legendFormat": "{{id}}", + "range": true, + "refId": "A" + } + ], + "title": "Heap Usage", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "video-clipping", + "spring-boot", + "operations" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Video Clipping Overview", + "uid": "video-clipping-overview", + "version": 1, + "weekStart": "" +} diff --git a/docs/dashboards.md b/docs/dashboards.md new file mode 100644 index 0000000..025e562 --- /dev/null +++ b/docs/dashboards.md @@ -0,0 +1,33 @@ +# Dashboards + +This document captures the dashboard artifact for operational readiness plan item `Dashboards`. + +Included artifact: + +- Grafana dashboard JSON: `dashboards/video-clipping-overview-grafana.json` + +Dashboard coverage: + +- Queue backlog: pending depth, oldest pending age, processing count, DLQ count. +- API behavior: request rate and p95 latency for non-Actuator endpoints. +- Runtime health: process CPU, system CPU, and JVM heap usage. + +Data source expectations: + +- Prometheus scrapes `/actuator/prometheus`. +- Database-backed queue deployments expose: +- `video_clipping_queue_pending` +- `video_clipping_queue_processing` +- `video_clipping_queue_dlq` +- `video_clipping_queue_oldest_pending_age_seconds` +- Standard Spring Boot and JVM metrics come from Micrometer Actuator export. + +Import guidance: + +- Create or reuse a Prometheus data source in Grafana. +- Import `dashboards/video-clipping-overview-grafana.json`. +- Set the Grafana data source variable `DS_PROMETHEUS` during import. + +Operational note: + +- If production workers use SQS instead of the database queue, swap the queue panels to native queue metrics while keeping the same dashboard layout and thresholds. diff --git a/docs/grafana-dashboard-creation-prompt.md b/docs/grafana-dashboard-creation-prompt.md new file mode 100644 index 0000000..307a572 --- /dev/null +++ b/docs/grafana-dashboard-creation-prompt.md @@ -0,0 +1,239 @@ +# Grafana Dashboard Creation Prompt + +Use this prompt with the AI instance running on the VPS server. + +```md +You are working on the `video-clipping-service` project. Create the full Grafana dashboard definition for this service as a Grafana dashboard JSON document that can be imported directly into Grafana. + +Your task is to produce the complete dashboard artifact, not a partial sketch. + +## Output requirements + +- Return a single valid Grafana dashboard JSON document. +- The JSON must be importable in Grafana without manual structural edits. +- Use a Prometheus data source variable named `DS_PROMETHEUS`. +- Set the Prometheus datasource on every panel to: + - `"type": "prometheus"` + - `"uid": "${DS_PROMETHEUS}"` +- Use Grafana schema/style compatible with Grafana 10.x. +- Set `"id": null`. +- Make the dashboard editable. +- Do not wrap the final JSON in explanations. Output the JSON only. + +## Dashboard identity + +- Dashboard title: `Video Clipping Overview` +- Service/application name: `video-clipping-service` +- Metrics endpoint scraped by Prometheus: `/actuator/prometheus` +- Spring Boot Actuator and Micrometer are enabled. + +## Service context + +This is a Spring Boot video clipping service. It exposes REST endpoints for: + +- `POST /v1/video-assets` +- `POST /v1/video-assets/{assetId}/uploads:complete` +- `GET /v1/video-assets/{assetId}` +- `GET /v1/video-assets/{assetId}/upload-session` +- `GET /v1/video-assets/{assetId}/clip-jobs` +- `GET /v1/video-assets/{assetId}/events` +- `POST /v1/video-assets/{assetId}/clip-jobs` +- `DELETE /v1/video-assets/{assetId}` +- `GET /v1/clip-jobs/{jobId}` +- `GET /v1/clip-jobs/{jobId}/clips` +- `GET /v1/clip-jobs/{jobId}/events` +- `POST /v1/clip-jobs/{jobId}:cancel` +- `POST /v1/clips/{clipId}/download-url` + +Exclude `/actuator` endpoints from API traffic panels. + +## Available custom queue metrics + +The service emits these Micrometer metrics for the database-backed queue. In Prometheus naming, dots become underscores: + +- `video.clipping.queue.pending` -> `video_clipping_queue_pending` +- `video.clipping.queue.processing` -> `video_clipping_queue_processing` +- `video.clipping.queue.dlq` -> `video_clipping_queue_dlq` +- `video.clipping.queue.oldest.pending.age.seconds` -> `video_clipping_queue_oldest_pending_age_seconds` + +Metric meanings: + +- `video_clipping_queue_pending`: number of pending clip-job queue messages +- `video_clipping_queue_processing`: number of claimed messages currently processing +- `video_clipping_queue_dlq`: number of dead-lettered messages +- `video_clipping_queue_oldest_pending_age_seconds`: age in seconds of the oldest pending message + +## Standard Micrometer / Actuator metrics available + +Use these standard metrics: + +- `http_server_requests_seconds_count` +- `http_server_requests_seconds_bucket` +- `system_cpu_usage` +- `process_cpu_usage` +- `jvm_memory_used_bytes{area="heap"}` + +## Required dashboard layout + +Create exactly 10 panels arranged in a 24-column Grafana grid. + +### Row 1: 4 stat panels, each height 8 and width 6 + +1. Panel title: `Pending Queue Depth` + - Type: `stat` + - Grid: `x=0, y=0, w=6, h=8` + - Query: `sum(video_clipping_queue_pending)` + - Legend: `pending` + - Unit: `none` + - Color mode: background + - Graph mode: area + - Thresholds: + - green: default + - orange at `5` + - red at `20` + +2. Panel title: `Oldest Pending Age` + - Type: `stat` + - Grid: `x=6, y=0, w=6, h=8` + - Query: `max(video_clipping_queue_oldest_pending_age_seconds)` + - Legend: `oldest pending age` + - Unit: `s` + - Color mode: background + - Graph mode: area + - Thresholds: + - green: default + - orange at `30` + - red at `120` + +3. Panel title: `DLQ Messages` + - Type: `stat` + - Grid: `x=12, y=0, w=6, h=8` + - Query: `sum(video_clipping_queue_dlq)` + - Legend: `dlq` + - Unit: `none` + - Color mode: background + - Graph mode: area + - Thresholds: + - green: default + - red at `1` + +4. Panel title: `Processing Messages` + - Type: `stat` + - Grid: `x=18, y=0, w=6, h=8` + - Query: `sum(video_clipping_queue_processing)` + - Legend: `processing` + - Unit: `none` + - Color mode: background + - Graph mode: area + - Thresholds: + - green: default + - orange at `1` + - red at `5` + +### Row 2: 2 time series panels, each height 8 and width 12 + +5. Panel title: `Queue State Over Time` + - Type: `timeseries` + - Grid: `x=0, y=8, w=12, h=8` + - Unit: `none` + - Queries: + - `sum(video_clipping_queue_pending)` legend `pending` + - `sum(video_clipping_queue_processing)` legend `processing` + - `sum(video_clipping_queue_dlq)` legend `dlq` + - Use line draw style, fill opacity around 20, line width 2, show legend at bottom + +6. Panel title: `Queue Age` + - Type: `timeseries` + - Grid: `x=12, y=8, w=12, h=8` + - Unit: `s` + - Query: + - `max(video_clipping_queue_oldest_pending_age_seconds)` legend `oldest pending age` + - Show legend at bottom + +### Row 3: 2 time series panels, each height 8 and width 12 + +7. Panel title: `API Request Rate` + - Type: `timeseries` + - Grid: `x=0, y=16, w=12, h=8` + - Unit: `reqps` + - Query: + - `sum(rate(http_server_requests_seconds_count{uri!~"/actuator.*"}[5m])) by (method, status)` + - Legend format: `{{method}} {{status}}` + - Show legend at bottom + +8. Panel title: `API p95 Latency` + - Type: `timeseries` + - Grid: `x=12, y=16, w=12, h=8` + - Unit: `s` + - Query: + - `histogram_quantile(0.95, sum(rate(http_server_requests_seconds_bucket{uri!~"/actuator.*"}[5m])) by (le, uri, method))` + - Legend format: `{{method}} {{uri}} p95` + - Show legend at bottom + +### Row 4: 2 time series panels, each height 8 and width 12 + +9. Panel title: `CPU Usage` + - Type: `timeseries` + - Grid: `x=0, y=24, w=12, h=8` + - Unit: `percentunit` + - Queries: + - `system_cpu_usage` legend `system cpu` + - `process_cpu_usage` legend `process cpu` + - Show legend at bottom + +10. Panel title: `Heap Usage` + - Type: `timeseries` + - Grid: `x=12, y=24, w=12, h=8` + - Unit: `bytes` + - Query: + - `jvm_memory_used_bytes{area="heap"}` + - Legend format: `{{id}}` + - Show legend at bottom + +## General panel settings + +- For stat panels: + - Reduce calculation: `lastNotNull` + - `values: false` + - `textMode: auto` + - `orientation: auto` + - `justifyMode: auto` + +- For timeseries panels: + - Tooltip mode: `single` + - Tooltip sort: `none` + - Legend display mode: `list` + - Legend placement: `bottom` + - Show legend: `true` + +- For timeseries field defaults where reasonable: + - Color mode: `palette-classic` + - Line width: `2` + - Show points: `never` + - Gradient mode: `none` + - Stacking mode: `none` + +## Dashboard metadata + +Include standard Grafana dashboard metadata such as: + +- annotations with the default Grafana annotations block +- `editable: true` +- `graphTooltip: 0` +- `links: []` +- `liveNow: false` +- `fiscalYearStartMonth: 0` + +## Acceptance criteria + +The generated JSON is only acceptable if: + +- it imports as a full dashboard in Grafana +- all 10 required panels exist +- every panel uses the `DS_PROMETHEUS` datasource variable +- the PromQL expressions exactly match the queries specified above +- the panel titles, units, thresholds, and grid positions match the specification above +- `/actuator` traffic is excluded from the API panels + +Return only the final Grafana dashboard JSON. +``` diff --git a/docs/video-clipping-service-implementation-plan.md b/docs/video-clipping-service-implementation-plan.md index 6541231..9ab4666 100644 --- a/docs/video-clipping-service-implementation-plan.md +++ b/docs/video-clipping-service-implementation-plan.md @@ -719,7 +719,7 @@ Keep the domain independent from Spring framework details. Adapters implement st ### Operational Readiness -1. [ ] Dashboards. +1. [x] Dashboards. 2. [ ] Alerts. 3. [ ] Runbooks. 4. [ ] Backup and restore validation. diff --git a/tmp/ffmpeg-input/uploads/upl_367df9ed2414/source b/tmp/ffmpeg-input/uploads/upl_367df9ed2414/source deleted file mode 100644 index cb3d12a..0000000 --- a/tmp/ffmpeg-input/uploads/upl_367df9ed2414/source +++ /dev/null @@ -1 +0,0 @@ -stub-video-content \ No newline at end of file