diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 94a25f7..35eb1dd 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -1,6 +1,6 @@ - + \ No newline at end of file diff --git a/dashboards/video-clipping-overview-grafana.json b/dashboards/video-clipping-overview-grafana.json new file mode 100644 index 0000000..a976bd4 --- /dev/null +++ b/dashboards/video-clipping-overview-grafana.json @@ -0,0 +1,642 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 20 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(video_clipping_queue_pending)", + "legendFormat": "pending", + "range": true, + "refId": "A" + } + ], + "title": "Pending Queue Depth", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 30 + }, + { + "color": "red", + "value": 120 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max(video_clipping_queue_oldest_pending_age_seconds)", + "legendFormat": "oldest pending age", + "range": true, + "refId": "A" + } + ], + "title": "Oldest Pending Age", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(video_clipping_queue_dlq)", + "legendFormat": "dlq", + "range": true, + "refId": "A" + } + ], + "title": "DLQ Messages", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(video_clipping_queue_processing)", + "legendFormat": "processing", + "range": true, + "refId": "A" + } + ], + "title": "Processing Messages", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(video_clipping_queue_pending)", + "legendFormat": "pending", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "sum(video_clipping_queue_processing)", + "legendFormat": "processing", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "sum(video_clipping_queue_dlq)", + "legendFormat": "dlq", + "range": true, + "refId": "C" + } + ], + "title": "Queue State Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "max(video_clipping_queue_oldest_pending_age_seconds)", + "legendFormat": "oldest pending age", + "range": true, + "refId": "A" + } + ], + "title": "Queue Age", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(http_server_requests_seconds_count{uri!~\"/actuator.*\"}[5m])) by (method, status)", + "legendFormat": "{{method}} {{status}}", + "range": true, + "refId": "A" + } + ], + "title": "API Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(http_server_requests_seconds_bucket{uri!~\"/actuator.*\"}[5m])) by (le, uri, method))", + "legendFormat": "{{method}} {{uri}} p95", + "range": true, + "refId": "A" + } + ], + "title": "API p95 Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "system_cpu_usage", + "legendFormat": "system cpu", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "process_cpu_usage", + "legendFormat": "process cpu", + "range": true, + "refId": "B" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "editorMode": "code", + "expr": "jvm_memory_used_bytes{area=\"heap\"}", + "legendFormat": "{{id}}", + "range": true, + "refId": "A" + } + ], + "title": "Heap Usage", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "video-clipping", + "spring-boot", + "operations" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Video Clipping Overview", + "uid": "video-clipping-overview", + "version": 1, + "weekStart": "" +} diff --git a/docs/dashboards.md b/docs/dashboards.md new file mode 100644 index 0000000..025e562 --- /dev/null +++ b/docs/dashboards.md @@ -0,0 +1,33 @@ +# Dashboards + +This document captures the dashboard artifact for operational readiness plan item `Dashboards`. + +Included artifact: + +- Grafana dashboard JSON: `dashboards/video-clipping-overview-grafana.json` + +Dashboard coverage: + +- Queue backlog: pending depth, oldest pending age, processing count, DLQ count. +- API behavior: request rate and p95 latency for non-Actuator endpoints. +- Runtime health: process CPU, system CPU, and JVM heap usage. + +Data source expectations: + +- Prometheus scrapes `/actuator/prometheus`. +- Database-backed queue deployments expose: +- `video_clipping_queue_pending` +- `video_clipping_queue_processing` +- `video_clipping_queue_dlq` +- `video_clipping_queue_oldest_pending_age_seconds` +- Standard Spring Boot and JVM metrics come from Micrometer Actuator export. + +Import guidance: + +- Create or reuse a Prometheus data source in Grafana. +- Import `dashboards/video-clipping-overview-grafana.json`. +- Set the Grafana data source variable `DS_PROMETHEUS` during import. + +Operational note: + +- If production workers use SQS instead of the database queue, swap the queue panels to native queue metrics while keeping the same dashboard layout and thresholds. diff --git a/docs/grafana-dashboard-creation-prompt.md b/docs/grafana-dashboard-creation-prompt.md new file mode 100644 index 0000000..307a572 --- /dev/null +++ b/docs/grafana-dashboard-creation-prompt.md @@ -0,0 +1,239 @@ +# Grafana Dashboard Creation Prompt + +Use this prompt with the AI instance running on the VPS server. + +```md +You are working on the `video-clipping-service` project. Create the full Grafana dashboard definition for this service as a Grafana dashboard JSON document that can be imported directly into Grafana. + +Your task is to produce the complete dashboard artifact, not a partial sketch. + +## Output requirements + +- Return a single valid Grafana dashboard JSON document. +- The JSON must be importable in Grafana without manual structural edits. +- Use a Prometheus data source variable named `DS_PROMETHEUS`. +- Set the Prometheus datasource on every panel to: + - `"type": "prometheus"` + - `"uid": "${DS_PROMETHEUS}"` +- Use Grafana schema/style compatible with Grafana 10.x. +- Set `"id": null`. +- Make the dashboard editable. +- Do not wrap the final JSON in explanations. Output the JSON only. + +## Dashboard identity + +- Dashboard title: `Video Clipping Overview` +- Service/application name: `video-clipping-service` +- Metrics endpoint scraped by Prometheus: `/actuator/prometheus` +- Spring Boot Actuator and Micrometer are enabled. + +## Service context + +This is a Spring Boot video clipping service. It exposes REST endpoints for: + +- `POST /v1/video-assets` +- `POST /v1/video-assets/{assetId}/uploads:complete` +- `GET /v1/video-assets/{assetId}` +- `GET /v1/video-assets/{assetId}/upload-session` +- `GET /v1/video-assets/{assetId}/clip-jobs` +- `GET /v1/video-assets/{assetId}/events` +- `POST /v1/video-assets/{assetId}/clip-jobs` +- `DELETE /v1/video-assets/{assetId}` +- `GET /v1/clip-jobs/{jobId}` +- `GET /v1/clip-jobs/{jobId}/clips` +- `GET /v1/clip-jobs/{jobId}/events` +- `POST /v1/clip-jobs/{jobId}:cancel` +- `POST /v1/clips/{clipId}/download-url` + +Exclude `/actuator` endpoints from API traffic panels. + +## Available custom queue metrics + +The service emits these Micrometer metrics for the database-backed queue. In Prometheus naming, dots become underscores: + +- `video.clipping.queue.pending` -> `video_clipping_queue_pending` +- `video.clipping.queue.processing` -> `video_clipping_queue_processing` +- `video.clipping.queue.dlq` -> `video_clipping_queue_dlq` +- `video.clipping.queue.oldest.pending.age.seconds` -> `video_clipping_queue_oldest_pending_age_seconds` + +Metric meanings: + +- `video_clipping_queue_pending`: number of pending clip-job queue messages +- `video_clipping_queue_processing`: number of claimed messages currently processing +- `video_clipping_queue_dlq`: number of dead-lettered messages +- `video_clipping_queue_oldest_pending_age_seconds`: age in seconds of the oldest pending message + +## Standard Micrometer / Actuator metrics available + +Use these standard metrics: + +- `http_server_requests_seconds_count` +- `http_server_requests_seconds_bucket` +- `system_cpu_usage` +- `process_cpu_usage` +- `jvm_memory_used_bytes{area="heap"}` + +## Required dashboard layout + +Create exactly 10 panels arranged in a 24-column Grafana grid. + +### Row 1: 4 stat panels, each height 8 and width 6 + +1. Panel title: `Pending Queue Depth` + - Type: `stat` + - Grid: `x=0, y=0, w=6, h=8` + - Query: `sum(video_clipping_queue_pending)` + - Legend: `pending` + - Unit: `none` + - Color mode: background + - Graph mode: area + - Thresholds: + - green: default + - orange at `5` + - red at `20` + +2. Panel title: `Oldest Pending Age` + - Type: `stat` + - Grid: `x=6, y=0, w=6, h=8` + - Query: `max(video_clipping_queue_oldest_pending_age_seconds)` + - Legend: `oldest pending age` + - Unit: `s` + - Color mode: background + - Graph mode: area + - Thresholds: + - green: default + - orange at `30` + - red at `120` + +3. Panel title: `DLQ Messages` + - Type: `stat` + - Grid: `x=12, y=0, w=6, h=8` + - Query: `sum(video_clipping_queue_dlq)` + - Legend: `dlq` + - Unit: `none` + - Color mode: background + - Graph mode: area + - Thresholds: + - green: default + - red at `1` + +4. Panel title: `Processing Messages` + - Type: `stat` + - Grid: `x=18, y=0, w=6, h=8` + - Query: `sum(video_clipping_queue_processing)` + - Legend: `processing` + - Unit: `none` + - Color mode: background + - Graph mode: area + - Thresholds: + - green: default + - orange at `1` + - red at `5` + +### Row 2: 2 time series panels, each height 8 and width 12 + +5. Panel title: `Queue State Over Time` + - Type: `timeseries` + - Grid: `x=0, y=8, w=12, h=8` + - Unit: `none` + - Queries: + - `sum(video_clipping_queue_pending)` legend `pending` + - `sum(video_clipping_queue_processing)` legend `processing` + - `sum(video_clipping_queue_dlq)` legend `dlq` + - Use line draw style, fill opacity around 20, line width 2, show legend at bottom + +6. Panel title: `Queue Age` + - Type: `timeseries` + - Grid: `x=12, y=8, w=12, h=8` + - Unit: `s` + - Query: + - `max(video_clipping_queue_oldest_pending_age_seconds)` legend `oldest pending age` + - Show legend at bottom + +### Row 3: 2 time series panels, each height 8 and width 12 + +7. Panel title: `API Request Rate` + - Type: `timeseries` + - Grid: `x=0, y=16, w=12, h=8` + - Unit: `reqps` + - Query: + - `sum(rate(http_server_requests_seconds_count{uri!~"/actuator.*"}[5m])) by (method, status)` + - Legend format: `{{method}} {{status}}` + - Show legend at bottom + +8. Panel title: `API p95 Latency` + - Type: `timeseries` + - Grid: `x=12, y=16, w=12, h=8` + - Unit: `s` + - Query: + - `histogram_quantile(0.95, sum(rate(http_server_requests_seconds_bucket{uri!~"/actuator.*"}[5m])) by (le, uri, method))` + - Legend format: `{{method}} {{uri}} p95` + - Show legend at bottom + +### Row 4: 2 time series panels, each height 8 and width 12 + +9. Panel title: `CPU Usage` + - Type: `timeseries` + - Grid: `x=0, y=24, w=12, h=8` + - Unit: `percentunit` + - Queries: + - `system_cpu_usage` legend `system cpu` + - `process_cpu_usage` legend `process cpu` + - Show legend at bottom + +10. Panel title: `Heap Usage` + - Type: `timeseries` + - Grid: `x=12, y=24, w=12, h=8` + - Unit: `bytes` + - Query: + - `jvm_memory_used_bytes{area="heap"}` + - Legend format: `{{id}}` + - Show legend at bottom + +## General panel settings + +- For stat panels: + - Reduce calculation: `lastNotNull` + - `values: false` + - `textMode: auto` + - `orientation: auto` + - `justifyMode: auto` + +- For timeseries panels: + - Tooltip mode: `single` + - Tooltip sort: `none` + - Legend display mode: `list` + - Legend placement: `bottom` + - Show legend: `true` + +- For timeseries field defaults where reasonable: + - Color mode: `palette-classic` + - Line width: `2` + - Show points: `never` + - Gradient mode: `none` + - Stacking mode: `none` + +## Dashboard metadata + +Include standard Grafana dashboard metadata such as: + +- annotations with the default Grafana annotations block +- `editable: true` +- `graphTooltip: 0` +- `links: []` +- `liveNow: false` +- `fiscalYearStartMonth: 0` + +## Acceptance criteria + +The generated JSON is only acceptable if: + +- it imports as a full dashboard in Grafana +- all 10 required panels exist +- every panel uses the `DS_PROMETHEUS` datasource variable +- the PromQL expressions exactly match the queries specified above +- the panel titles, units, thresholds, and grid positions match the specification above +- `/actuator` traffic is excluded from the API panels + +Return only the final Grafana dashboard JSON. +``` diff --git a/docs/video-clipping-service-implementation-plan.md b/docs/video-clipping-service-implementation-plan.md index 6541231..9ab4666 100644 --- a/docs/video-clipping-service-implementation-plan.md +++ b/docs/video-clipping-service-implementation-plan.md @@ -719,7 +719,7 @@ Keep the domain independent from Spring framework details. Adapters implement st ### Operational Readiness -1. [ ] Dashboards. +1. [x] Dashboards. 2. [ ] Alerts. 3. [ ] Runbooks. 4. [ ] Backup and restore validation. diff --git a/tmp/ffmpeg-input/uploads/upl_367df9ed2414/source b/tmp/ffmpeg-input/uploads/upl_367df9ed2414/source deleted file mode 100644 index cb3d12a..0000000 --- a/tmp/ffmpeg-input/uploads/upl_367df9ed2414/source +++ /dev/null @@ -1 +0,0 @@ -stub-video-content \ No newline at end of file