diff --git a/plugins/infermatrix-copilot/skills/explainer-video/SKILL.md b/plugins/infermatrix-copilot/skills/explainer-video/SKILL.md index 6fc32db5..2b0d2a95 100644 --- a/plugins/infermatrix-copilot/skills/explainer-video/SKILL.md +++ b/plugins/infermatrix-copilot/skills/explainer-video/SKILL.md @@ -10,7 +10,7 @@ The video is a deterministic web page. `renderAt(t)` draws the exact frame for t ```mermaid flowchart LR A[narration.json] -->|tts.py| B[audio/*.mp3 + timeline.json] - B --> C[stage/timeline.js] + B -->|render.py count| C[stage/timeline.js] C --> D[stage: renderAt t] D -->|render.py stills| E[QA frames] D -->|render.py seg K| F[segs/seg_KK.mp4] @@ -77,9 +77,11 @@ Requires `ffmpeg` and `ffprobe` on PATH. ```bash python tts.py . +python render.py . count ``` - Writes `build/audio/_.mp3` and `build/timeline.json`, and prints each scene's start and length. +- After each TTS run, `render.py . count` exports the current timeline and configured FPS to `stage/timeline.js` for browser preview, and prints the segment count. Run both commands again after narration or timing changes. - Existing MP3 files are reused only when their adjacent JSON metadata matches `say`, `voice` and `rate`. Changing any of these, or using an older cache without metadata, regenerates that line. Caption-only edits reuse the audio. - For a strict total, shorten narration first, then use modest `atempo` adjustment only if speech stays clear. Measure the fitted audio with `ffprobe`, retain lead/tail space, and require every line to end within its scene. The supplied `tts.py` uses narration-driven lengths; fixed-budget timing needs a project-local adaptation, not just smaller `min_len` values. Update audio cues, captions and JS animation timings together. - When adding, reordering or lengthening shots, update the full timeline, transitions, captions and narration cues, then remix continuous music and ducking against the new cues. Deliver the complete revised video when requested. @@ -89,7 +91,7 @@ python tts.py . ## 4. Preview and QA before the full render -1. Open `stage/index.html` in a browser for live playback, or `index.html?t=12.5` to freeze one moment. A short animated preview can establish composition and motion for a substantial structural change; apply only the approval gate established in §1. +1. After both commands in §3, open `stage/index.html` in a browser for live playback, or `index.html?t=12.5` to freeze one moment. A short animated preview can establish composition and motion for a substantial structural change; apply only the approval gate established in §1. 2. Capture stills at the busiest moments of each scene: `python render.py . stills 3.2 12.5 24 ...` → `build/still_*.png`. 3. Read every still. Check, in this order: caption overlapping content, text cut off or wrapped badly, elements crossing each other (lines through numbers, labels on nodes), and wrong facts. diff --git a/plugins/infermatrix-copilot/skills/explainer-video/templates/render.py b/plugins/infermatrix-copilot/skills/explainer-video/templates/render.py index 23c9cdb0..43fe1abb 100644 --- a/plugins/infermatrix-copilot/skills/explainer-video/templates/render.py +++ b/plugins/infermatrix-copilot/skills/explainer-video/templates/render.py @@ -151,7 +151,7 @@ def mix(pj: Project, silent: Path) -> None: args += ["-i", str(pj.music)] m = len(lines) + 1 parts += [ - f"{vins}amix=inputs={len(lines)}:normalize=0,asplit=2[voice][sc]", + f"{vins}amix=inputs={len(lines)}:normalize=0,apad=whole_dur={total},asplit=2[voice][sc]", f"[{m}:a]atrim=0:{total},asetpts=PTS-STARTPTS,volume={gain:.2f}dB," f"afade=t=in:d=2,afade=t=out:st={total - 3:.3f}:d=3[bed]", # voice above threshold pushes the bed down ~10 dB; 400 ms release avoids pumping between lines diff --git a/test/test_explainer_video_segments.py b/test/test_explainer_video_segments.py index 8e136dd4..2d979761 100644 --- a/test/test_explainer_video_segments.py +++ b/test/test_explainer_video_segments.py @@ -1,8 +1,11 @@ +import array import asyncio import importlib.util import json +import math import shutil import subprocess +import sys import tempfile import unittest from pathlib import Path @@ -91,6 +94,72 @@ def test_fingerprint_tracks_inputs_but_ignores_music_only_edits(self): finally: setattr(self.renderer, name, previous) + @unittest.skipUnless(importlib.util.find_spec("playwright"), + "optional preview check requires Playwright/Chromium") + def test_count_prepares_fresh_and_updated_browser_preview(self): + from playwright.sync_api import sync_playwright + + exported = self.root / "stage" / "timeline.js" + self.assertFalse(exported.exists()) + command = [sys.executable, str(self.root / "render.py"), str(self.root), "count"] + with sync_playwright() as engine: + if not Path(engine.chromium.executable_path).is_file(): + self.skipTest("Chromium is absent; run python -m playwright install chromium") + browser = engine.chromium.launch() + try: + page = browser.new_page() + errors = [] + page.on("pageerror", lambda error: errors.append(str(error))) + for caption in ["Title", "Updated preview"]: + with self.subTest(caption=caption): + self.timeline["scenes"][0]["lines"][0]["cap"] = caption + self.save_timeline() + result = subprocess.run(command, check=True, capture_output=True, text=True) + self.assertEqual(result.stdout.strip(), "1") + page.goto((self.root / "stage" / "index.html").as_uri() + "?t=0.05") + page.wait_for_function("window.READY === true") + self.assertEqual(page.evaluate("window.TL.fps"), self.renderer.FPS) + self.assertEqual(page.locator("#cap").inner_text(), caption) + self.assertFalse(errors, errors) + finally: + browser.close() + + @unittest.skipUnless(shutil.which("ffmpeg"), "optional audio check requires FFmpeg") + def test_music_survives_narration_end_and_fades_during_scene_tail(self): + silent = self.root / "build" / "silent.mp4" + voice = self.root / "voice.wav" + music = self.root / "music.wav" + subprocess.run(["ffmpeg", "-y", "-v", "error", "-f", "lavfi", "-i", + "color=c=black:s=16x16:r=30:d=6", "-c:v", "libx264", "-threads", "1", + "-pix_fmt", "yuv420p", str(silent)], check=True) + for path, frequency, seconds in [(voice, 880, 1), (music, 220, 6)]: + subprocess.run(["ffmpeg", "-y", "-v", "error", "-f", "lavfi", "-i", + f"sine=frequency={frequency}:sample_rate=48000:duration={seconds}", + str(path)], check=True) + (self.root / "narration.json").write_text(json.dumps({ + "output": "out.mp4", "music": music.name, + }), encoding="utf-8") + self.timeline.update(total=6) + self.timeline["scenes"][0].update(len=3, lines=[{ + "file": str(voice), "start": .5, "dur": 1, "cap": "Short narration", + }]) + self.timeline["scenes"][1].update(start=3, len=3, lines=[]) + self.save_timeline() + project = self.renderer.Project(self.root) + self.renderer.mix(project, silent) + decoded = subprocess.run(["ffmpeg", "-v", "error", "-xerror", "-i", str(project.out), + "-vn", "-ac", "1", "-ar", "48000", "-f", "f32le", "-"], + check=True, capture_output=True) + samples = array.array("f", decoded.stdout) + levels = [] + for start in [2, 3.5, 4.5, 5.5]: + window = samples[round(start * 48000):round((start + .1) * 48000)] + self.assertEqual(len(window), 4800) + rms = math.sqrt(sum(value * value for value in window) / len(window)) + self.assertGreater(rms, .0001, f"music is silent at {start}s") + levels.append(rms) + self.assertLess(levels[-1], levels[-2] * .7) + @unittest.skipUnless(importlib.util.find_spec("playwright") and shutil.which("ffmpeg"), "optional media check requires Playwright/Chromium and FFmpeg") def test_real_render_resume_invalidation_and_final_guard(self):