Fix pipeline correctness: quality mapping, stale VHS warnings, ffmpeg timeout, binary paths

README.md

@@ -55,7 +55,7 @@ docgen validate --pre-push  # validate all outputs before committing
 | `docgen tts [--segment 01] [--dry-run]` | Generate TTS audio |
 | `docgen manim [--scene StackDAGScene]` | Render Manim animations |
 | `docgen vhs [--tape 02-quickstart.tape] [--strict]` | Render VHS terminal recordings |
-| `docgen compose [01 02 03]` | Compose segments (audio + video) |
+| `docgen compose [01 02 03] [--ffmpeg-timeout 900]` | Compose segments (audio + video) |
 | `docgen validate [--max-drift 2.75] [--pre-push]` | Run all validation checks |
 | `docgen concat [--config full-demo]` | Concatenate full demo files |
 | `docgen pages [--force]` | Generate index.html, pages.yml, .gitattributes, .gitignore |
@@ -66,6 +66,23 @@ docgen validate --pre-push  # validate all outputs before committing
 
 Create a `docgen.yaml` in your demos directory. See [examples/minimal-bundle/docgen.yaml](examples/minimal-bundle/docgen.yaml) for a starting point.
 
+Useful pipeline options:
+
+```yaml
+manim:
+  quality: 1080p30          # supports 480p15, 720p30, 1080p30, 1080p60, 1440p30, 1440p60, 2160p60
+  manim_path: ""            # optional explicit binary path (relative to docgen.yaml or absolute)
+
+vhs:
+  vhs_path: ""              # optional explicit binary path (relative to docgen.yaml or absolute)
+
+compose:
+  ffmpeg_timeout_sec: 300   # can also be overridden with: docgen compose --ffmpeg-timeout N
+  warn_stale_vhs: true      # warns if terminal/*.tape is newer than terminal/rendered/*.mp4
+```
+
+If you edit a `.tape` file, run `docgen vhs` before `docgen compose` so compose does not use stale rendered terminal video.
+
 ## System dependencies
 
 - **ffmpeg** — composition and probing

docs/demos/docgen.yaml

@@ -65,10 +65,18 @@ visual_map:
 
 manim:
   quality: 720p30
+  manim_path: ""
   scenes:
     - DocgenOverviewScene
     - WizardGUIScene
 
+vhs:
+  vhs_path: ""
+
+compose:
+  ffmpeg_timeout_sec: 300
+  warn_stale_vhs: true
+
 tts:
   model: gpt-4o-mini-tts
   voice: coral

src/docgen/binaries.py

@@ -0,0 +1,57 @@
+"""Helpers for resolving external binary paths."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class BinaryResolution:
+    name: str
+    path: str | None
+    tried: list[str] = field(default_factory=list)
+
+
+def resolve_binary(
+    name: str,
+    *,
+    configured_path: str | None = None,
+    extra_candidates: list[str | Path] | None = None,
+) -> BinaryResolution:
+    """Resolve an executable path using config overrides + sensible defaults."""
+    candidates: list[str] = []
+
+    def _add(candidate: str | Path | None) -> None:
+        if not candidate:
+            return
+        p = str(candidate).strip()
+        if not p:
+            return
+        p = os.path.expanduser(os.path.expandvars(p))
+        if p not in candidates:
+            candidates.append(p)
+
+    _add(configured_path)
+
+    # If docgen is running from a virtualenv, prefer binaries next to this python.
+    _add(Path(sys.executable).resolve().parent / name)
+
+    for candidate in extra_candidates or []:
+        _add(candidate)
+
+    which_hit = shutil.which(name)
+    if which_hit:
+        _add(which_hit)
+
+    tried: list[str] = []
+    for candidate in candidates:
+        tried.append(candidate)
+        cpath = Path(candidate)
+        if cpath.exists() and os.access(cpath, os.X_OK):
+            return BinaryResolution(name=name, path=str(cpath), tried=tried)
+
+    return BinaryResolution(name=name, path=None, tried=tried)

src/docgen/cli.py

@@ -124,16 +124,22 @@ def vhs(ctx: click.Context, tape: str | None, strict: bool) -> None:
 
 @main.command()
 @click.argument("segments", nargs=-1)
+@click.option(
+    "--ffmpeg-timeout",
+    default=None,
+    type=int,
+    help="Override ffmpeg timeout in seconds (default from docgen.yaml compose.ffmpeg_timeout_sec).",
+)
 @click.pass_context
-def compose(ctx: click.Context, segments: tuple[str, ...]) -> None:
+def compose(ctx: click.Context, segments: tuple[str, ...], ffmpeg_timeout: int | None) -> None:
     """Compose segments (audio + video via ffmpeg).
 
     Pass segment IDs to compose specific ones, or omit for the default set.
     """
     from docgen.compose import Composer
 
     cfg = ctx.obj["config"]
-    comp = Composer(cfg)
+    comp = Composer(cfg, ffmpeg_timeout_sec=ffmpeg_timeout)
     target = list(segments) if segments else cfg.segments_default
     click.echo(f"=== Composing {len(target)} segments ===")
     comp.compose_segments(target)

src/docgen/compose.py

@@ -15,8 +15,13 @@ class ComposeError(RuntimeError):
 
 
 class Composer:
-    def __init__(self, config: Config) -> None:
+    def __init__(self, config: Config, ffmpeg_timeout_sec: int | None = None) -> None:
         self.config = config
+        self.ffmpeg_timeout_sec = (
+            int(ffmpeg_timeout_sec)
+            if ffmpeg_timeout_sec is not None
+            else int(self.config.ffmpeg_timeout_sec)
+        )
 
     def compose_segments(self, segment_ids: list[str], *, strict: bool = True) -> int:
         composed = 0
@@ -30,7 +35,9 @@ def compose_segments(self, segment_ids: list[str], *, strict: bool = True) -> in
             if vtype == "manim":
                 ok = self._compose_simple(seg_id, self._manim_path(vmap), strict=strict)
             elif vtype == "vhs":
-                ok = self._compose_simple(seg_id, self._vhs_path(vmap), strict=strict)
+                video_path = self._vhs_path(vmap)
+                self._warn_if_stale_vhs(vmap, video_path)
+                ok = self._compose_simple(seg_id, video_path, strict=strict)
             elif vtype == "mixed":
                 sources = [self._resolve_source(s) for s in vmap.get("sources", [])]
                 ok = self._compose_mixed(seg_id, sources)
@@ -223,16 +230,24 @@ def _find_audio(self, seg_id: str) -> Path | None:
 
     def _manim_path(self, vmap: dict[str, Any]) -> Path:
         src = vmap.get("source", "")
-        return self.config.animations_dir / "media" / "videos" / "scenes" / "720p30" / src
+        if not src:
+            return self.config.animations_dir / "media" / "videos" / "scenes" / "720p30"
+
+        for base in self._manim_video_dirs():
+            candidate = base / src
+            if candidate.exists():
+                return candidate
+        return self._manim_video_dirs()[0] / src
 
     def _vhs_path(self, vmap: dict[str, Any]) -> Path:
         src = vmap.get("source", "")
         return self.config.terminal_dir / "rendered" / src
 
     def _resolve_source(self, source: str) -> Path:
-        manim_path = self.config.animations_dir / "media" / "videos" / "scenes" / "720p30" / source
-        if manim_path.exists():
-            return manim_path
+        for base in self._manim_video_dirs():
+            manim_path = base / source
+            if manim_path.exists():
+                return manim_path
         vhs_path = self.config.terminal_dir / "rendered" / source
         if vhs_path.exists():
             return vhs_path
@@ -254,13 +269,69 @@ def _probe_duration(path: Path) -> float | None:
         except (ValueError, subprocess.TimeoutExpired, FileNotFoundError):
             return None
 
-    @staticmethod
-    def _run_ffmpeg(cmd: list[str]) -> None:
+    def _run_ffmpeg(self, cmd: list[str]) -> None:
+        timeout_sec = max(1, int(self.ffmpeg_timeout_sec))
+        out_path = Path(cmd[-1]) if cmd else None
         try:
-            subprocess.run(cmd, check=True, capture_output=True, text=True, timeout=300)
+            subprocess.run(cmd, check=True, capture_output=True, text=True, timeout=timeout_sec)
         except FileNotFoundError:
-            print("    ERROR: ffmpeg not found in PATH")
+            raise ComposeError("ffmpeg not found in PATH")
         except subprocess.CalledProcessError as exc:
-            print(f"    ERROR: ffmpeg failed: {exc.stderr[:300]}")
+            detail = (exc.stderr or exc.stdout or "")[:400]
+            raise ComposeError(f"ffmpeg failed: {detail}")
         except subprocess.TimeoutExpired:
-            print("    ERROR: ffmpeg timed out")
+            if out_path and out_path.exists() and out_path.stat().st_size > 0:
+                print(
+                    f"    WARNING: ffmpeg timed out after {timeout_sec}s, "
+                    f"but output exists at {out_path}."
+                )
+                return
+            raise ComposeError(f"ffmpeg timed out after {timeout_sec}s")
+
+    def _manim_video_dirs(self) -> list[Path]:
+        root = self.config.animations_dir / "media" / "videos"
+        quality = str(self.config.manim_quality).strip().lower()
+        fallback_qualities = [
+            quality,
+            "1080p30",
+            "1080p60",
+            "1440p30",
+            "1440p60",
+            "720p30",
+            "480p15",
+            "2160p60",
+        ]
+
+        ordered_qualities: list[str] = []
+        for q in fallback_qualities:
+            if q and q not in ordered_qualities:
+                ordered_qualities.append(q)
+
+        dirs: list[Path] = []
+        for q in ordered_qualities:
+            dirs.append(root / "scenes" / q)
+            dirs.append(root / q)
+        return dirs
+
+    def _warn_if_stale_vhs(self, vmap: dict[str, Any], video_path: Path) -> None:
+        if not self.config.warn_stale_vhs:
+            return
+
+        tape_name = str(vmap.get("tape", "")).strip()
+        if not tape_name:
+            source_name = str(vmap.get("source", "")).strip()
+            if source_name:
+                tape_name = f"{Path(source_name).stem}.tape"
+        if not tape_name:
+            return
+
+        tape_path = self.config.terminal_dir / tape_name
+        if not tape_path.exists() or not video_path.exists():
+            return
+
+        if tape_path.stat().st_mtime > (video_path.stat().st_mtime + 1):
+            print(
+                "    WARNING: tape is newer than rendered video "
+                f"({tape_path.name} > {video_path.name}). "
+                "Run `docgen vhs` before `docgen compose` to avoid stale output."
+            )

src/docgen/config.py

@@ -88,6 +88,38 @@ def manim_scenes(self) -> list[str]:
     def manim_quality(self) -> str:
         return self.raw.get("manim", {}).get("quality", "720p30")
 
+    @property
+    def manim_path(self) -> str | None:
+        """Optional absolute/relative path to the Manim executable."""
+        value = self.raw.get("manim", {}).get("manim_path")
+        return str(value) if value else None
+
+    @property
+    def vhs_path(self) -> str | None:
+        """Optional absolute/relative path to the VHS executable."""
+        value = self.raw.get("vhs", {}).get("vhs_path")
+        return str(value) if value else None
+
+    # -- Compose ----------------------------------------------------------------
+
+    @property
+    def compose_config(self) -> dict[str, Any]:
+        defaults: dict[str, Any] = {
+            "ffmpeg_timeout_sec": 300,
+            "warn_stale_vhs": True,
+        }
+        defaults.update(self.raw.get("compose", {}))
+        return defaults
+
+    @property
+    def ffmpeg_timeout_sec(self) -> int:
+        value = self.compose_config.get("ffmpeg_timeout_sec", 300)
+        return int(value)
+
+    @property
+    def warn_stale_vhs(self) -> bool:
+        return bool(self.compose_config.get("warn_stale_vhs", True))
+
     # -- Validation ------------------------------------------------------------
 
     @property

src/docgen/init.py

@@ -253,6 +253,14 @@ def _write_config(plan: InitPlan) -> str:
         "manim": {
             "quality": "720p30",
             "scenes": [f"Scene{s['id']}" for s in plan.segments],
+            "manim_path": "",
+        },
+        "vhs": {
+            "vhs_path": "",
+        },
+        "compose": {
+            "ffmpeg_timeout_sec": 300,
+            "warn_stale_vhs": True,
         },
         "tts": {
             "model": plan.tts_model,