Add highlight audio analysis

This commit is contained in:
JSLMPR 2026-07-11 10:23:07 +02:00
parent 5d5f8022f2
commit fc17578549
13 changed files with 356 additions and 6 deletions

View File

@ -303,7 +303,8 @@ The plan should be strict JSON so a cheaper model or deterministic renderer can
- [x] Milestone 3 progress: Added highlight source analysis for the new project contract, reusing FFprobe inspection, frame extraction, contact sheet generation, proxy generation, and adding waveform image generation under `analysis/`; the highlight scheduler now runs analysis immediately after creating a project.
- [x] Milestone 4: Implement shot and scene segmentation using FFmpeg scene detection or PySceneDetect.
- [x] Milestone 4 progress: Added configurable FFmpeg scene detection for highlight sources, persisted `analysis/scene-segments.json`, attached shot segments to `source-analysis.json`, and logged shot segment counts when the highlight scheduler finishes analysis.
- [ ] Milestone 5: Implement audio analysis for loudness, silence, peaks, speech/music/noise sections, and optional ASR transcript.
- [x] Milestone 5: Implement audio analysis for loudness, silence, peaks, speech/music/noise sections, and optional ASR transcript.
- [x] Milestone 5 progress: Added deterministic FFmpeg audio analysis for mean/max volume, silence detection, missing-audio handling, persisted `analysis/audio-analysis.json`, and source-analysis embedding. Non-silent sections are labeled `unclassified_audio` until a speech/music/noise classifier or ASR provider is added.
- [ ] Milestone 6: Implement visual analysis for blur, exposure, motion, faces, object labels, composition quality, and representative thumbnails.
- [x] Milestone 7: Implement content category classification for family vlog, food vlog, car vlog, and generic fallback.
- [x] Milestone 8: Implement category-specific highlight candidate scoring.

View File

@ -507,6 +507,10 @@ public class VideoClippingProperties {
private double minimumSceneDurationSeconds = 1.0;
private double silenceThresholdDb = -35.0;
private double silenceMinimumDurationSeconds = 0.5;
@Min(1)
private int targetDurationSeconds = 60;
@ -638,6 +642,22 @@ public class VideoClippingProperties {
this.minimumSceneDurationSeconds = minimumSceneDurationSeconds;
}
public double getSilenceThresholdDb() {
return silenceThresholdDb;
}
public void setSilenceThresholdDb(double silenceThresholdDb) {
this.silenceThresholdDb = silenceThresholdDb;
}
public double getSilenceMinimumDurationSeconds() {
return silenceMinimumDurationSeconds;
}
public void setSilenceMinimumDurationSeconds(double silenceMinimumDurationSeconds) {
this.silenceMinimumDurationSeconds = silenceMinimumDurationSeconds;
}
public int getTargetDurationSeconds() {
return targetDurationSeconds;
}

View File

@ -0,0 +1,10 @@
package org.example.videoclips.editing;
public record AudioSection(
String sectionId,
String kind,
double startSeconds,
double endSeconds,
double durationSeconds
) {
}

View File

@ -13,6 +13,8 @@ public record HighlightSourceAnalysis(
String waveformPath,
String sceneSegmentsPath,
List<ShotSegment> shotSegments,
String audioAnalysisPath,
SourceAudioAnalysis audioAnalysis,
Instant createdAt
) {
}

View File

@ -22,6 +22,7 @@ public class HighlightSourceAnalyzer {
private final ProxyGenerator proxyGenerator;
private final WaveformGenerator waveformGenerator;
private final ShotSceneSegmenter shotSceneSegmenter;
private final SourceAudioAnalyzer sourceAudioAnalyzer;
private final Clock clock;
@Autowired
@ -32,10 +33,11 @@ public class HighlightSourceAnalyzer {
ContactSheetGenerator contactSheetGenerator,
ProxyGenerator proxyGenerator,
WaveformGenerator waveformGenerator,
ShotSceneSegmenter shotSceneSegmenter
ShotSceneSegmenter shotSceneSegmenter,
SourceAudioAnalyzer sourceAudioAnalyzer
) {
this(store, inspector, thumbnailExtractor, contactSheetGenerator, proxyGenerator, waveformGenerator,
shotSceneSegmenter,
shotSceneSegmenter, sourceAudioAnalyzer,
Clock.systemUTC());
}
@ -47,6 +49,7 @@ public class HighlightSourceAnalyzer {
ProxyGenerator proxyGenerator,
WaveformGenerator waveformGenerator,
ShotSceneSegmenter shotSceneSegmenter,
SourceAudioAnalyzer sourceAudioAnalyzer,
Clock clock
) {
this.store = store;
@ -56,6 +59,7 @@ public class HighlightSourceAnalyzer {
this.proxyGenerator = proxyGenerator;
this.waveformGenerator = waveformGenerator;
this.shotSceneSegmenter = shotSceneSegmenter;
this.sourceAudioAnalyzer = sourceAudioAnalyzer;
this.clock = clock;
}
@ -74,6 +78,7 @@ public class HighlightSourceAnalyzer {
String proxyPath = proxyGenerator.generate(inspected, proxyDirectory).orElse(null);
String waveformPath = waveformGenerator.generate(inspected, audioDirectory);
List<ShotSegment> shotSegments = shotSceneSegmenter.segment(inspected);
SourceAudioAnalysis audioAnalysis = sourceAudioAnalyzer.analyze(inspected);
ClipAnalysis sourceAnalysis = enriched(inspected, thumbnails, contactSheet, proxyPath);
HighlightSourceAnalysis analysis = new HighlightSourceAnalysis(
projectId,
@ -85,10 +90,13 @@ public class HighlightSourceAnalyzer {
waveformPath,
"analysis/scene-segments.json",
List.copyOf(shotSegments),
"analysis/audio-analysis.json",
audioAnalysis,
Instant.now(clock)
);
store.writeJson(projectId, "analysis/ffprobe.json", sourceAnalysis);
store.writeJson(projectId, "analysis/scene-segments.json", shotSegments);
store.writeJson(projectId, "analysis/audio-analysis.json", audioAnalysis);
store.writeJson(projectId, "analysis/source-analysis.json", analysis);
return analysis;
}

View File

@ -153,9 +153,11 @@ public class HighlightSourceScheduler {
scanId, projectId, processedFile.getFileName(), projectDirectory, "analysis/source-analysis.json",
elapsedMillis(startedAt));
log.info("event=highlight_source_analyzed scan_id={} project_id={} duration_seconds={} thumbnails={} "
+ "proxy={} waveform={} shot_segments={}",
+ "proxy={} waveform={} shot_segments={} audio_sections={} mean_volume_db={} max_volume_db={}",
scanId, projectId, analysis.source().durationSeconds(), analysis.thumbnails().size(),
analysis.proxyPath(), analysis.waveformPath(), analysis.shotSegments().size());
analysis.proxyPath(), analysis.waveformPath(), analysis.shotSegments().size(),
analysis.audioAnalysis().sections().size(), analysis.audioAnalysis().meanVolumeDb(),
analysis.audioAnalysis().maxVolumeDb());
}
private void handleFailure(long scanId, Path activeFile, RuntimeException failure) {

View File

@ -0,0 +1,15 @@
package org.example.videoclips.editing;
import java.util.List;
public record SourceAudioAnalysis(
String clipId,
boolean audioPresent,
double meanVolumeDb,
double maxVolumeDb,
double silenceThresholdDb,
double silenceDurationSeconds,
List<AudioSection> sections,
String transcriptPath
) {
}

View File

@ -0,0 +1,173 @@
package org.example.videoclips.editing;
import org.example.videoclips.config.VideoClippingProperties;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Component
@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true)
public class SourceAudioAnalyzer {
private static final Pattern MEAN_VOLUME_PATTERN = Pattern.compile("mean_volume:\\s*(-?[0-9]+(?:\\.[0-9]+)?) dB");
private static final Pattern MAX_VOLUME_PATTERN = Pattern.compile("max_volume:\\s*(-?[0-9]+(?:\\.[0-9]+)?) dB");
private static final Pattern SILENCE_START_PATTERN = Pattern.compile("silence_start:\\s*([0-9]+(?:\\.[0-9]+)?)");
private static final Pattern SILENCE_END_PATTERN = Pattern.compile("silence_end:\\s*([0-9]+(?:\\.[0-9]+)?)");
private final VideoClippingProperties.Editing properties;
private final ProcessExecutor processExecutor;
@Autowired
public SourceAudioAnalyzer(VideoClippingProperties properties) {
this(properties, SourceAudioAnalyzer::execute);
}
SourceAudioAnalyzer(VideoClippingProperties properties, ProcessExecutor processExecutor) {
this.properties = properties.getEditing();
this.processExecutor = processExecutor;
}
public SourceAudioAnalysis analyze(ClipAnalysis source) {
if (source.audioCodec() == null || source.audioCodec().isBlank()) {
return new SourceAudioAnalysis(source.clipId(), false, 0, 0,
properties.getSilenceThresholdDb(), properties.getSilenceMinimumDurationSeconds(),
List.of(new AudioSection("audio_section_0001", "missing_audio", 0,
round(source.durationSeconds()), round(source.durationSeconds()))),
null);
}
List<String> command = List.of(
properties.getFfmpegBinary(),
"-hide_banner",
"-i", source.sourcePath(),
"-af", "silencedetect=n=" + properties.getSilenceThresholdDb() + "dB:d="
+ properties.getSilenceMinimumDurationSeconds() + ",volumedetect",
"-vn",
"-f", "null",
"-"
);
ProcessResult result;
try {
result = processExecutor.execute(command);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new IllegalStateException("Interrupted while analyzing source audio", ex);
} catch (IOException ex) {
throw new IllegalStateException("Unable to run FFmpeg for source audio analysis", ex);
}
if (result.exitCode() != 0) {
throw new IllegalStateException("FFmpeg source audio analysis exited with code " + result.exitCode());
}
ParsedAudio parsed = parse(result.output());
return new SourceAudioAnalysis(
source.clipId(),
true,
parsed.meanVolumeDb(),
parsed.maxVolumeDb(),
properties.getSilenceThresholdDb(),
properties.getSilenceMinimumDurationSeconds(),
sections(source.durationSeconds(), parsed.silences()),
null
);
}
private ParsedAudio parse(String output) {
double meanVolume = 0;
double maxVolume = 0;
List<SilenceRange> silences = new ArrayList<>();
Double pendingSilenceStart = null;
for (String line : output.split("\\R")) {
Matcher meanMatcher = MEAN_VOLUME_PATTERN.matcher(line);
if (meanMatcher.find()) {
meanVolume = Double.parseDouble(meanMatcher.group(1));
}
Matcher maxMatcher = MAX_VOLUME_PATTERN.matcher(line);
if (maxMatcher.find()) {
maxVolume = Double.parseDouble(maxMatcher.group(1));
}
Matcher startMatcher = SILENCE_START_PATTERN.matcher(line);
if (startMatcher.find()) {
pendingSilenceStart = Double.parseDouble(startMatcher.group(1));
}
Matcher endMatcher = SILENCE_END_PATTERN.matcher(line);
if (endMatcher.find() && pendingSilenceStart != null) {
silences.add(new SilenceRange(pendingSilenceStart, Double.parseDouble(endMatcher.group(1))));
pendingSilenceStart = null;
}
}
return new ParsedAudio(round(meanVolume), round(maxVolume), silences);
}
private List<AudioSection> sections(double durationSeconds, List<SilenceRange> silences) {
double duration = round(Math.max(0, durationSeconds));
List<AudioSection> sections = new ArrayList<>();
double cursor = 0;
for (SilenceRange silence : silences.stream()
.sorted(Comparator.comparingDouble(SilenceRange::startSeconds))
.toList()) {
double start = clamp(silence.startSeconds(), 0, duration);
double end = clamp(silence.endSeconds(), start, duration);
if (start > cursor) {
sections.add(section(sections.size() + 1, "unclassified_audio", cursor, start));
}
if (end > start) {
sections.add(section(sections.size() + 1, "silence", start, end));
}
cursor = Math.max(cursor, end);
}
if (cursor < duration) {
sections.add(section(sections.size() + 1, "unclassified_audio", cursor, duration));
}
if (sections.isEmpty()) {
sections.add(section(1, "unclassified_audio", 0, duration));
}
return List.copyOf(sections);
}
private AudioSection section(int index, String kind, double start, double end) {
double roundedStart = round(start);
double roundedEnd = round(end);
return new AudioSection(
"audio_section_%04d".formatted(index),
kind,
roundedStart,
roundedEnd,
round(roundedEnd - roundedStart)
);
}
private double clamp(double value, double min, double max) {
return Math.min(max, Math.max(min, value));
}
private double round(double value) {
return Math.round(value * 1000.0) / 1000.0;
}
private static ProcessResult execute(List<String> command) throws IOException, InterruptedException {
Process process = new ProcessBuilder(command).redirectErrorStream(true).start();
String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8);
return new ProcessResult(process.waitFor(), output);
}
private record ParsedAudio(double meanVolumeDb, double maxVolumeDb, List<SilenceRange> silences) {
}
private record SilenceRange(double startSeconds, double endSeconds) {
}
record ProcessResult(int exitCode, String output) {
}
@FunctionalInterface
interface ProcessExecutor {
ProcessResult execute(List<String> command) throws IOException, InterruptedException;
}
}

View File

@ -25,6 +25,8 @@ video-clipping:
proxy-width: ${VIDEO_EDITING_PROXY_WIDTH:640}
scene-detection-threshold: ${VIDEO_EDITING_SCENE_DETECTION_THRESHOLD:0.35}
minimum-scene-duration-seconds: ${VIDEO_EDITING_MINIMUM_SCENE_DURATION_SECONDS:1.0}
silence-threshold-db: ${VIDEO_EDITING_SILENCE_THRESHOLD_DB:-35.0}
silence-minimum-duration-seconds: ${VIDEO_EDITING_SILENCE_MINIMUM_DURATION_SECONDS:0.5}
target-duration-seconds: ${VIDEO_EDITING_TARGET_DURATION_SECONDS:60}
output-width: ${VIDEO_EDITING_OUTPUT_WIDTH:1920}
output-height: ${VIDEO_EDITING_OUTPUT_HEIGHT:1080}

View File

@ -29,6 +29,8 @@ class VideoClippingPropertiesTest {
assertThat(properties.getEditing().getProxyWidth()).isEqualTo(640);
assertThat(properties.getEditing().getSceneDetectionThreshold()).isEqualTo(0.35);
assertThat(properties.getEditing().getMinimumSceneDurationSeconds()).isEqualTo(1.0);
assertThat(properties.getEditing().getSilenceThresholdDb()).isEqualTo(-35.0);
assertThat(properties.getEditing().getSilenceMinimumDurationSeconds()).isEqualTo(0.5);
assertThat(properties.getEditing().getTargetDurationSeconds()).isEqualTo(60);
assertThat(properties.getEditing().getOutputWidth()).isEqualTo(1920);
assertThat(properties.getEditing().getOutputHeight()).isEqualTo(1080);
@ -75,6 +77,8 @@ class VideoClippingPropertiesTest {
"video-clipping.editing.thumbnail-count-per-clip=7",
"video-clipping.editing.scene-detection-threshold=0.42",
"video-clipping.editing.minimum-scene-duration-seconds=2.5",
"video-clipping.editing.silence-threshold-db=-40",
"video-clipping.editing.silence-minimum-duration-seconds=1.25",
"video-clipping.editing.voiceover-provider=remote",
"video-clipping.editing.loudness-target-i=-14",
"video-clipping.editing.loudness-true-peak=-2",
@ -107,6 +111,8 @@ class VideoClippingPropertiesTest {
assertThat(properties.getEditing().getThumbnailCountPerClip()).isEqualTo(7);
assertThat(properties.getEditing().getSceneDetectionThreshold()).isEqualTo(0.42);
assertThat(properties.getEditing().getMinimumSceneDurationSeconds()).isEqualTo(2.5);
assertThat(properties.getEditing().getSilenceThresholdDb()).isEqualTo(-40.0);
assertThat(properties.getEditing().getSilenceMinimumDurationSeconds()).isEqualTo(1.25);
assertThat(properties.getEditing().getVoiceoverProvider()).isEqualTo("remote");
assertThat(properties.getEditing().getLoudnessTargetI()).isEqualTo(-14.0);
assertThat(properties.getEditing().getLoudnessTruePeak()).isEqualTo(-2.0);

View File

@ -64,15 +64,22 @@ class HighlightSourceAnalyzerTest {
new ShotSegment("shot_0001", 0, 12, 12, 0, 6),
new ShotSegment("shot_0002", 12, 42, 30, 0.61, 27)
);
assertThat(result.audioAnalysisPath()).isEqualTo("analysis/audio-analysis.json");
assertThat(result.audioAnalysis().sections()).containsExactly(
new AudioSection("audio_section_0001", "unclassified_audio", 0, 42, 42)
);
assertThat(result.createdAt()).isEqualTo(Instant.parse("2026-07-11T08:00:00Z"));
ClipAnalysis ffprobe = store.readJson("porsche", "analysis/ffprobe.json", ClipAnalysis.class);
ShotSegment[] shots = store.readJson("porsche", "analysis/scene-segments.json", ShotSegment[].class);
SourceAudioAnalysis audioAnalysis = store.readJson("porsche", "analysis/audio-analysis.json",
SourceAudioAnalysis.class);
HighlightSourceAnalysis persisted = store.readJson("porsche", "analysis/source-analysis.json",
HighlightSourceAnalysis.class);
assertThat(ffprobe.sourcePath()).isEqualTo(source.toString());
assertThat(ffprobe.thumbnails()).hasSize(2);
assertThat(shots).containsExactly(result.shotSegments().toArray(ShotSegment[]::new));
assertThat(audioAnalysis).isEqualTo(result.audioAnalysis());
assertThat(persisted).isEqualTo(result);
}
@ -115,8 +122,13 @@ class HighlightSourceAnalyzerTest {
new ShotSegment("shot_0002", 12, 42, 30, 0.61, 27)
));
SourceAudioAnalyzer sourceAudioAnalyzer = mock(SourceAudioAnalyzer.class);
when(sourceAudioAnalyzer.analyze(any())).thenReturn(new SourceAudioAnalysis("porsche", true, -18.2,
-3.1, -35.0, 0.5,
List.of(new AudioSection("audio_section_0001", "unclassified_audio", 0, 42, 42)), null));
Clock clock = Clock.fixed(Instant.parse("2026-07-11T08:00:00Z"), ZoneOffset.UTC);
return new HighlightSourceAnalyzer(store, inspector, thumbnails, contactSheet, proxy, waveform,
shotSceneSegmenter, clock);
shotSceneSegmenter, sourceAudioAnalyzer, clock);
}
}

View File

@ -116,6 +116,9 @@ class HighlightSourceSchedulerTest {
.toString(), 42.0, "h264", "aac", 1920, 1080, 30.0, List.of(), null, null, 0, 0, 0);
return new HighlightSourceAnalysis(projectId, projectId + ".mp4", source, List.of(), null, null, null,
"analysis/scene-segments.json", List.of(new ShotSegment("shot_0001", 0, 42, 42, 0, 21)),
"analysis/audio-analysis.json", new SourceAudioAnalysis(projectId, true, -18.0, -3.0,
-35.0, 0.5, List.of(new AudioSection("audio_section_0001", "unclassified_audio", 0, 42, 42)),
null),
Instant.parse("2026-07-11T08:00:00Z"));
}
}

View File

@ -0,0 +1,96 @@
package org.example.videoclips.editing;
import org.example.videoclips.config.VideoClippingProperties;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
class SourceAudioAnalyzerTest {
@Test
void analyzesLoudnessSilenceAndAudioSections() {
AtomicReference<List<String>> command = new AtomicReference<>();
VideoClippingProperties properties = new VideoClippingProperties();
properties.getEditing().setSilenceThresholdDb(-40);
properties.getEditing().setSilenceMinimumDurationSeconds(1.25);
SourceAudioAnalyzer analyzer = new SourceAudioAnalyzer(properties, arguments -> {
command.set(arguments);
return new SourceAudioAnalyzer.ProcessResult(0, """
[silencedetect @ 0x1] silence_start: 2.5
[silencedetect @ 0x1] silence_end: 4.0 | silence_duration: 1.5
[Parsed_volumedetect_1 @ 0x2] mean_volume: -19.2 dB
[Parsed_volumedetect_1 @ 0x2] max_volume: -2.7 dB
""");
});
SourceAudioAnalysis analysis = analyzer.analyze(clip("aac"));
assertThat(analysis.audioPresent()).isTrue();
assertThat(analysis.meanVolumeDb()).isEqualTo(-19.2);
assertThat(analysis.maxVolumeDb()).isEqualTo(-2.7);
assertThat(analysis.silenceThresholdDb()).isEqualTo(-40);
assertThat(analysis.silenceDurationSeconds()).isEqualTo(1.25);
assertThat(analysis.sections()).containsExactly(
new AudioSection("audio_section_0001", "unclassified_audio", 0, 2.5, 2.5),
new AudioSection("audio_section_0002", "silence", 2.5, 4, 1.5),
new AudioSection("audio_section_0003", "unclassified_audio", 4, 8, 4)
);
assertCommandPair(command.get(), "-i", "source.mp4");
assertCommandPair(command.get(), "-af", "silencedetect=n=-40.0dB:d=1.25,volumedetect");
}
@Test
void returnsMissingAudioAnalysisWhenSourceHasNoAudioCodec() {
SourceAudioAnalyzer analyzer = new SourceAudioAnalyzer(new VideoClippingProperties(), command -> {
throw new AssertionError("ffmpeg should not run without an audio stream");
});
SourceAudioAnalysis analysis = analyzer.analyze(clip(null));
assertThat(analysis.audioPresent()).isFalse();
assertThat(analysis.sections()).containsExactly(
new AudioSection("audio_section_0001", "missing_audio", 0, 8, 8)
);
}
@Test
void rejectsFfmpegFailures() {
SourceAudioAnalyzer analyzer = new SourceAudioAnalyzer(new VideoClippingProperties(), command ->
new SourceAudioAnalyzer.ProcessResult(1, "failed"));
assertThrows(IllegalStateException.class, () -> analyzer.analyze(clip("aac")));
}
@Test
void reportsProcessStartupAndInterruptionFailures() {
assertThrows(IllegalStateException.class, () -> new SourceAudioAnalyzer(new VideoClippingProperties(), command -> {
throw new IOException("missing executable");
}).analyze(clip("aac")));
try {
assertThrows(IllegalStateException.class, () -> new SourceAudioAnalyzer(new VideoClippingProperties(), command -> {
throw new InterruptedException("stopped");
}).analyze(clip("aac")));
assertTrue(Thread.currentThread().isInterrupted());
} finally {
Thread.interrupted();
}
}
private ClipAnalysis clip(String audioCodec) {
return new ClipAnalysis("source", "source.mp4", 8.0, "h264", audioCodec,
1920, 1080, 30.0, List.of(), null, null, 0, 0, 0);
}
private void assertCommandPair(List<String> command, String option, String value) {
int index = command.indexOf(option);
assertThat(index).isGreaterThanOrEqualTo(0);
assertThat(command.get(index + 1)).isEqualTo(value);
}
}