jsl
/
video_editing_poc
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add cinematic storyboard prompt generation

This commit is contained in:
JSLMPR 2026-07-10 15:35:15 +02:00
parent 0ccfd99147
commit 8a1dd0ac3a
10 changed files with 273 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/cinematic-video-editing-service-plan.md
View File

@ -791,7 +791,7 @@ It writes the script only and requires an imported audio file.
8. [x] Add contact sheet generation.
9. [x] Add optional proxy generation.
10. [x] Add `analysis.json` writing and reading.
11. [ ] Add storyboard prompt generation from `analysis.json`.
11. [x] Add storyboard prompt generation from `analysis.json`.
12. [ ] Add local AI director prompt generation with thumbnail/contact-sheet instructions.
13. [ ] Add local director source-folder scheduler.
14. [ ] Add project `inbox/` plan pickup.

11
src/main/java/org/example/videoclips/api/EditProjectController.java
View File

@ -3,6 +3,7 @@ package org.example.videoclips.api;
import jakarta.validation.Valid;
import org.example.videoclips.api.dto.CreateEditProjectRequest;
import org.example.videoclips.editing.EditProjectService;
import org.example.videoclips.editing.StoryboardPromptGenerator;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.GetMapping;
@ -19,9 +20,12 @@ import org.springframework.web.bind.annotation.RestController;
public class EditProjectController {
private final EditProjectService editProjectService;
private final StoryboardPromptGenerator storyboardPromptGenerator;
public EditProjectController(EditProjectService editProjectService) {
public EditProjectController(EditProjectService editProjectService,
StoryboardPromptGenerator storyboardPromptGenerator) {
this.editProjectService = editProjectService;
this.storyboardPromptGenerator = storyboardPromptGenerator;
}
@PostMapping
@ -34,4 +38,9 @@ public class EditProjectController {
public Object getProject(@PathVariable String projectId) {
return editProjectService.getProject(projectId);
}
@PostMapping("/{projectId}:storyboard-prompt")
public Object generateStoryboardPrompt(@PathVariable String projectId) {
return storyboardPromptGenerator.generate(projectId);
}
}

5
src/main/java/org/example/videoclips/api/dto/EditProjectResponse.java
View File

@ -10,6 +10,11 @@ public record EditProjectResponse(
EditProjectStatus status,
String inputDirectory,
String outputDirectory,
int targetDurationSeconds,
String style,
boolean voiceoverEnabled,
boolean musicEnabled,
boolean soundEffectsEnabled,
Instant createdAt,
Instant updatedAt,
String failureReason

7
src/main/java/org/example/videoclips/api/dto/StoryboardPromptResponse.java Normal file
View File

@ -0,0 +1,7 @@
package org.example.videoclips.api.dto;
public record StoryboardPromptResponse(
String projectId,
String promptPath
) {
}

5
src/main/java/org/example/videoclips/editing/EditProject.java
View File

@ -8,6 +8,11 @@ public record EditProject(
EditProjectStatus status,
String inputDirectory,
String outputDirectory,
int targetDurationSeconds,
String style,
boolean voiceoverEnabled,
boolean musicEnabled,
boolean soundEffectsEnabled,
Instant createdAt,
Instant updatedAt,
String failureReason

10
src/main/java/org/example/videoclips/editing/EditProjectService.java
View File

@ -46,6 +46,11 @@ public class EditProjectService {
EditProjectStatus.CREATED,
inputDirectory.toString(),
outputDirectory.toString(),
request.targetDurationSeconds(),
request.style(),
Boolean.TRUE.equals(request.voiceoverEnabled()),
Boolean.TRUE.equals(request.musicEnabled()),
Boolean.TRUE.equals(request.soundEffectsEnabled()),
now,
now,
null
@ -85,6 +90,11 @@ public class EditProjectService {
project.status(),
project.inputDirectory(),
project.outputDirectory(),
project.targetDurationSeconds(),
project.style(),
project.voiceoverEnabled(),
project.musicEnabled(),
project.soundEffectsEnabled(),
project.createdAt(),
project.updatedAt(),
project.failureReason()

123
src/main/java/org/example/videoclips/editing/StoryboardPromptGenerator.java Normal file
View File

@ -0,0 +1,123 @@
package org.example.videoclips.editing;
import org.example.videoclips.api.dto.StoryboardPromptResponse;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
@Component
@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true)
public class StoryboardPromptGenerator {
private final EditProjectStore store;
public StoryboardPromptGenerator(EditProjectStore store) {
this.store = store;
}
public StoryboardPromptResponse generate(String projectId) {
EditProject project = store.readJson(projectId, "project.json", EditProject.class);
EditProjectAnalysis analysis = store.readJson(projectId, "analysis.json", EditProjectAnalysis.class);
Path promptPath = store.projectDirectory(projectId).resolve("storyboard-prompt.md");
try {
Files.writeString(promptPath, prompt(project, analysis));
} catch (IOException ex) {
throw new IllegalStateException("Unable to write storyboard prompt for project: " + projectId, ex);
}
return new StoryboardPromptResponse(projectId, promptPath.toString());
}
String prompt(EditProject project, EditProjectAnalysis analysis) {
StringBuilder clips = new StringBuilder();
for (ClipAnalysis clip : analysis.clips()) {
clips.append("- clipId: ").append(clip.clipId()).append('\n')
.append(" durationSeconds: ").append(clip.durationSeconds()).append('\n')
.append(" video: ").append(clip.width()).append('x').append(clip.height())
.append(" ").append(clip.frameRate()).append("fps ").append(clip.videoCodec()).append('\n')
.append(" audioCodec: ").append(value(clip.audioCodec())).append('\n')
.append(" thumbnails: ").append(references("thumbnails", clip.thumbnails())).append('\n')
.append(" contactSheet: ").append(reference("contact-sheets", clip.contactSheet())).append('\n')
.append(" proxy: ").append(reference("proxies", clip.proxyPath())).append('\n');
}
return """
# Cinematic Storyboard Director Brief
Create one cinematic edit plan for project `%s`.
Constraints:
- Style: `%s`
- Target duration: %d seconds
- Voiceover enabled: %s
- Music enabled: %s
- Sound effects enabled: %s
- Use only the clip IDs and source ranges listed below.
- Keep every source range within its clip duration.
- Timeline ranges must be chronological and must not overlap.
- Prefer an intentional opening hook, escalating middle, and memorable final hero shot.
- Return strict JSON only. Do not use Markdown fences or add commentary.
Available clips and project-relative visual references:
%s
Required JSON object shape:
{
"projectId": "%s",
"style": "%s",
"targetDurationSeconds": %d,
"decisions": [{
"clipId": "clip_00001",
"sourceStartSeconds": 0.0,
"sourceEndSeconds": 2.5,
"timelineStartSeconds": 0.0,
"timelineEndSeconds": 2.5,
"transitionIn": "cut",
"transitionOut": "crossfade",
"playbackSpeed": 1.0,
"visualTreatment": "cinematic grade",
"reason": "story purpose"
}],
"audioCues": [{
"type": "music|sfx",
"asset": "descriptive asset name",
"timelineStartSeconds": 0.0,
"timelineEndSeconds": 2.5,
"gainDb": -12.0,
"reason": "story purpose"
}],
"voiceover": [{
"text": "narration",
"timelineStartSeconds": 0.0,
"timelineEndSeconds": 2.5,
"delivery": "confident cinematic narrator"
}],
"renderProfile": "mp4-h264-aac-1080p",
"summary": "short creative rationale"
}
""".formatted(
project.id(), project.style(), project.targetDurationSeconds(),
project.voiceoverEnabled(), project.musicEnabled(), project.soundEffectsEnabled(),
clips.toString().stripTrailing(), project.id(), project.style(), project.targetDurationSeconds());
}
private String references(String directory, List<String> paths) {
if (paths == null || paths.isEmpty()) {
return "[]";
}
return paths.stream().map(path -> reference(directory, path)).toList().toString();
}
private String reference(String directory, String path) {
if (path == null || path.isBlank()) {
return "none";
}
return directory + "/" + Path.of(path).getFileName();
}
private String value(String value) {
return value == null || value.isBlank() ? "none" : value;
}
}

38
src/test/java/org/example/videoclips/api/EditProjectControllerTest.java
View File

@ -15,6 +15,7 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.assertj.core.api.Assertions.assertThat;
@SpringBootTest(properties = {
"video-clipping.editing.project-directory=target/edit-project-controller-test/projects"
@ -47,6 +48,8 @@ class EditProjectControllerTest {
.andExpect(jsonPath("$.projectId").exists())
.andExpect(jsonPath("$.status").value("CREATED"))
.andExpect(jsonPath("$.inputDirectory").value(inputDirectory.toString()))
.andExpect(jsonPath("$.targetDurationSeconds").value(60))
.andExpect(jsonPath("$.style").value("cinematic-porsche-promo"))
.andReturn()
.getResponse()
.getContentAsString();
@ -61,6 +64,41 @@ class EditProjectControllerTest {
+ projectId));
}
@Test
void generatesStoryboardPromptFromPersistedProjectAndAnalysis() throws Exception {
Path inputDirectory = Files.createDirectories(Path.of("target/edit-project-controller-test/input-"
+ UUID.randomUUID()));
String response = mockMvc.perform(post("/v1/edit-projects")
.contentType(MediaType.APPLICATION_JSON)
.content("""
{
"name": "Storyboard %s",
"inputDirectory": "%s",
"targetDurationSeconds": 45,
"style": "cinematic-porsche-promo"
}
""".formatted(UUID.randomUUID(), inputDirectory)))
.andReturn().getResponse().getContentAsString();
String projectId = JsonTestHelper.read(response, "$.projectId");
Path projectDirectory = Path.of("target/edit-project-controller-test/projects", projectId);
Files.writeString(projectDirectory.resolve("analysis.json"), """
{
"projectId": "%s",
"clips": [],
"errors": [],
"createdAt": "2026-07-10T10:00:00Z"
}
""".formatted(projectId));
mockMvc.perform(post("/v1/edit-projects/{projectId}:storyboard-prompt", projectId))
.andExpect(status().isOk())
.andExpect(jsonPath("$.projectId").value(projectId))
.andExpect(jsonPath("$.promptPath").value(projectDirectory.resolve("storyboard-prompt.md").toString()));
assertThat(Files.readString(projectDirectory.resolve("storyboard-prompt.md")))
.contains("Target duration: 45 seconds");
}
@Test
void rejectsMissingInputDirectory() throws Exception {
mockMvc.perform(post("/v1/edit-projects")

5
src/test/java/org/example/videoclips/editing/EditDomainJsonTest.java
View File

@ -20,6 +20,11 @@ class EditDomainJsonTest {
EditProjectStatus.WAITING_FOR_DIRECTOR,
"./input/editing/processed/porsche-session-001",
"./output/edit-projects/porsche-session-001",
60,
"cinematic-porsche-promo",
true,
true,
true,
Instant.parse("2026-07-10T10:00:00Z"),
Instant.parse("2026-07-10T10:01:00Z"),
null

69
src/test/java/org/example/videoclips/editing/StoryboardPromptGeneratorTest.java Normal file
View File

@ -0,0 +1,69 @@
package org.example.videoclips.editing;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.example.videoclips.config.VideoClippingProperties;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
class StoryboardPromptGeneratorTest {
@TempDir
Path tempDir;
@Test
void writesPasteReadyStrictJsonPromptWithoutAbsoluteMediaPaths() throws Exception {
FileSystemEditProjectStore store = store();
store.createProject("porsche-edit");
store.writeJson("porsche-edit", "project.json", project());
store.writeJson("porsche-edit", "analysis.json", analysis());
var response = new StoryboardPromptGenerator(store).generate("porsche-edit");
String prompt = Files.readString(Path.of(response.promptPath()));
assertThat(prompt)
.contains("clipId: clip_00001")
.contains("durationSeconds: 8.0")
.contains("Target duration: 60 seconds")
.contains("Style: `cinematic-porsche-promo`")
.contains("\"decisions\"")
.contains("\"audioCues\"")
.contains("\"voiceover\"")
.contains("Return strict JSON only")
.contains("thumbnails/clip_00001_0001.jpg")
.contains("contact-sheets/clip_00001.jpg")
.doesNotContain(tempDir.toAbsolutePath().toString())
.doesNotContain("/private/source/video.mp4");
assertThat(response.projectId()).isEqualTo("porsche-edit");
assertThat(response.promptPath()).endsWith("storyboard-prompt.md");
}
private FileSystemEditProjectStore store() {
VideoClippingProperties properties = new VideoClippingProperties();
properties.getEditing().setProjectDirectory(tempDir.resolve("projects").toString());
return new FileSystemEditProjectStore(properties, new ObjectMapper().findAndRegisterModules());
}
private EditProject project() {
Instant now = Instant.parse("2026-07-10T10:00:00Z");
return new EditProject("porsche-edit", "Porsche Edit", EditProjectStatus.ANALYZED,
"/private/source", tempDir.resolve("projects/porsche-edit").toString(), 60,
"cinematic-porsche-promo", true, true, true, now, now, null);
}
private EditProjectAnalysis analysis() {
ClipAnalysis clip = new ClipAnalysis("clip_00001", "/private/source/video.mp4", 8.0,
"h264", "aac", 1920, 1080, 30.0,
List.of(tempDir.resolve("projects/porsche-edit/thumbnails/clip_00001_0001.jpg").toString()),
tempDir.resolve("projects/porsche-edit/contact-sheets/clip_00001.jpg").toString(),
tempDir.resolve("projects/porsche-edit/proxies/clip_00001.mp4").toString(), 0.8, 0.7, 0.9);
return new EditProjectAnalysis("porsche-edit", List.of(clip), List.of(),
Instant.parse("2026-07-10T10:01:00Z"));
}
}