Initial MaterialSub release

Derived from pengchujin/jzsub at 222a90265d2a8797ca258eb1a980cee0863a8311; preserve the upstream MIT license and attribution.
This commit is contained in:
2026-08-25 13:07:03 +08:00
commit e27cc22cfb
24 changed files with 7620 additions and 0 deletions
+972
View File
@@ -0,0 +1,972 @@
#!/usr/bin/env python3
"""Burn one ASS subtitle track into a high-quality H.264 MP4."""
from __future__ import annotations
import argparse
from collections import deque
import hashlib
import json
import math
import os
import re
import shutil
import subprocess
import sys
import tempfile
import textwrap
from fractions import Fraction
from pathlib import Path
from typing import Any, Sequence
DEFAULT_ENCODER = "libx264"
PROGRESS_BAR_WIDTH = 20
PROGRESS_STEP_PERCENT = 5
MP4_COPY_AUDIO_CODECS = frozenset({"aac", "ac3", "alac", "eac3", "mp3"})
HDR_TRANSFERS = frozenset({"arib-std-b67", "smpte2084"})
HDR_SIDE_DATA = (
"content light level",
"dolby vision",
"dovi",
"dynamic hdr",
"hdr10+",
"mastering display",
)
FFMPEG_FULL_CANDIDATES = (
Path("/opt/homebrew/opt/ffmpeg-full/bin/ffmpeg"),
Path("/usr/local/opt/ffmpeg-full/bin/ffmpeg"),
)
ASS_WORD_JOINER = "\u2060"
class BurnError(RuntimeError):
"""A user-actionable burn or verification failure."""
def _positive_crf(value: str) -> int:
try:
crf = int(value)
except ValueError as exc:
raise argparse.ArgumentTypeError("CRF must be an integer from 0 to 51") from exc
if not 0 <= crf <= 51:
raise argparse.ArgumentTypeError("CRF must be an integer from 0 to 51")
return crf
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description=(
"Burn an ASS subtitle file exactly once into an H.264/yuv420p MP4 "
"while preserving the source dimensions and frame timing."
)
)
parser.add_argument("video", type=Path, help="input video")
parser.add_argument("subtitle", type=Path, help="input ASS subtitle file")
parser.add_argument("output", type=Path, help="output MP4")
parser.add_argument(
"--force",
action="store_true",
help="replace OUTPUT if it already exists",
)
parser.add_argument(
"--crf",
type=_positive_crf,
default=18,
help="H.264 constant-rate-factor quality (default: 18)",
)
parser.add_argument(
"--preset",
default="slow",
help="encoder preset (default: slow)",
)
parser.add_argument(
"--encoder",
default=DEFAULT_ENCODER,
help=f"FFmpeg H.264 encoder (default: {DEFAULT_ENCODER})",
)
parser.add_argument(
"--validation-report",
type=Path,
help="subtitle validation JSON (default: validation.json next to the ASS file)",
)
parser.add_argument(
"--allow-missing-font",
action="store_true",
help="continue with libass font substitution when the validated font is not installed",
)
parser.add_argument(
"--citation-file",
type=Path,
help="approved UTF-8 citation text to burn at the top left",
)
return parser
def _required_executables() -> tuple[str, str]:
ffmpeg = shutil.which("ffmpeg")
ffprobe = shutil.which("ffprobe")
missing = [name for name, path in (("ffmpeg", ffmpeg), ("ffprobe", ffprobe)) if not path]
if missing:
raise BurnError(f"required executable not found in PATH: {', '.join(missing)}")
assert ffmpeg is not None and ffprobe is not None
ffmpeg = _select_libass_ffmpeg(ffmpeg)
sibling_ffprobe = Path(ffmpeg).with_name("ffprobe")
if sibling_ffprobe.is_file():
ffprobe = str(sibling_ffprobe)
return ffmpeg, ffprobe
def _ffmpeg_has_subtitles_filter(ffmpeg: str | Path) -> bool:
result = subprocess.run(
[str(ffmpeg), "-hide_banner", "-filters"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
check=False,
)
return result.returncode == 0 and any(
len(fields := line.split()) >= 2 and fields[1] == "subtitles"
for line in result.stdout.splitlines()
)
def _select_libass_ffmpeg(
default: str,
*,
candidates: Sequence[Path] = FFMPEG_FULL_CANDIDATES,
) -> str:
for candidate in (Path(default), *candidates):
if candidate.is_file() and _ffmpeg_has_subtitles_filter(candidate):
return str(candidate)
return default
def _require_libass_subtitles_filter(ffmpeg: str) -> None:
if not _ffmpeg_has_subtitles_filter(ffmpeg):
raise BurnError(
"FFmpeg has no usable 'subtitles' filter; install an FFmpeg build "
"with libass support"
)
def _last_error_line(stderr: str) -> str:
lines = [line.strip() for line in stderr.splitlines() if line.strip()]
return f": {lines[-1]}" if lines else ""
def _clock(seconds: float) -> str:
total = max(0, int(seconds))
hours, remainder = divmod(total, 3600)
minutes, seconds = divmod(remainder, 60)
if hours:
return f"{hours:02d}:{minutes:02d}:{seconds:02d}"
return f"{minutes:02d}:{seconds:02d}"
def _format_progress(
percent: int,
encoded_seconds: float,
duration: float,
speed: str,
) -> str:
percent = max(0, min(100, int(percent)))
filled = round(percent * PROGRESS_BAR_WIDTH / 100)
bar = "" * filled + "" * (PROGRESS_BAR_WIDTH - filled)
speed = speed.strip() or "--"
return (
f"烧录 [{bar}] {percent:3d}% "
f"{_clock(encoded_seconds)} / {_clock(duration)} {speed}"
)
def _progress_seconds(values: dict[str, str]) -> float:
raw = values.get("out_time_us") or values.get("out_time_ms")
if raw:
try:
return max(0.0, int(raw) / 1_000_000)
except ValueError:
pass
clock = values.get("out_time", "")
try:
hours, minutes, seconds = clock.split(":", 2)
return max(0.0, int(hours) * 3600 + int(minutes) * 60 + float(seconds))
except (TypeError, ValueError):
return 0.0
def _run_ffmpeg_with_progress(command: Sequence[str], duration: float) -> tuple[int, str]:
process = subprocess.Popen(
list(command),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
encoding="utf-8",
errors="replace",
bufsize=1,
)
if process.stdout is None:
process.kill()
raise BurnError("FFmpeg progress pipe was not available")
values: dict[str, str] = {}
diagnostics: deque[str] = deque(maxlen=12)
last_bucket = 0
print(_format_progress(0, 0, duration, "--"), file=sys.stderr, flush=True)
for raw_line in process.stdout:
line = raw_line.strip()
if not line:
continue
if "=" not in line:
diagnostics.append(line)
continue
key, value = line.split("=", 1)
values[key] = value
if key != "progress":
continue
encoded_seconds = _progress_seconds(values)
raw_percent = 100 * encoded_seconds / duration if duration > 0 else 0
bucket = min(
100,
int(raw_percent // PROGRESS_STEP_PERCENT) * PROGRESS_STEP_PERCENT,
)
if value == "end":
bucket = 100
encoded_seconds = duration
if bucket > last_bucket:
print(
_format_progress(
bucket,
encoded_seconds,
duration,
values.get("speed", "--"),
),
file=sys.stderr,
flush=True,
)
last_bucket = bucket
returncode = process.wait()
process.stdout.close()
return returncode, "\n".join(diagnostics)
def _probe(ffprobe: str, path: Path) -> dict[str, Any]:
result = subprocess.run(
[
ffprobe,
"-v",
"error",
"-show_format",
"-show_streams",
"-of",
"json",
str(path),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
check=False,
)
if result.returncode != 0:
raise BurnError(f"ffprobe could not read {path}{_last_error_line(result.stderr)}")
try:
data = json.loads(result.stdout)
except json.JSONDecodeError as exc:
raise BurnError(f"ffprobe returned invalid JSON for {path}: {exc}") from exc
if not isinstance(data, dict):
raise BurnError(f"ffprobe returned an unexpected result for {path}")
return data
def _streams(probe: dict[str, Any], kind: str) -> list[dict[str, Any]]:
streams = probe.get("streams", [])
if not isinstance(streams, list):
return []
return [
stream
for stream in streams
if isinstance(stream, dict) and stream.get("codec_type") == kind
]
def _main_video_stream(probe: dict[str, Any]) -> dict[str, Any]:
videos = _streams(probe, "video")
if not videos:
raise BurnError("input contains no video stream")
return next(
(
stream
for stream in videos
if not bool((stream.get("disposition") or {}).get("attached_pic"))
),
videos[0],
)
def _stream_dimensions(stream: dict[str, Any]) -> tuple[int, int]:
try:
width = int(stream["width"])
height = int(stream["height"])
except (KeyError, TypeError, ValueError) as exc:
raise BurnError("video stream has no valid dimensions") from exc
if width <= 0 or height <= 0:
raise BurnError("video stream has no valid dimensions")
return width, height
def _duration(probe: dict[str, Any]) -> float:
candidates: list[Any] = []
file_format = probe.get("format")
if isinstance(file_format, dict):
candidates.append(file_format.get("duration"))
for stream in probe.get("streams", []):
if isinstance(stream, dict):
candidates.append(stream.get("duration"))
durations: list[float] = []
for candidate in candidates:
try:
duration = float(candidate)
except (TypeError, ValueError):
continue
if math.isfinite(duration) and duration > 0:
durations.append(duration)
return max(durations, default=0.0)
def _frame_rate(stream: dict[str, Any]) -> Fraction | None:
for key in ("avg_frame_rate", "r_frame_rate"):
value = stream.get(key)
if not isinstance(value, str):
continue
try:
rate = Fraction(value)
except (ValueError, ZeroDivisionError):
continue
if rate > 0:
return rate
return None
def _is_hdr(stream: dict[str, Any]) -> bool:
if str(stream.get("color_transfer", "")).lower() in HDR_TRANSFERS:
return True
if str(stream.get("color_primaries", "")).lower() == "bt2020":
try:
bit_depth = int(stream.get("bits_per_raw_sample", 0))
except (TypeError, ValueError):
bit_depth = 0
pixel_format = str(stream.get("pix_fmt", "")).lower()
if bit_depth >= 10 or re.search(r"(?:10|12|14|16)(?:le|be)?$", pixel_format):
return True
side_data = stream.get("side_data_list")
if isinstance(side_data, list):
for item in side_data:
if not isinstance(item, dict):
continue
description = " ".join(str(value).lower() for value in item.values())
if any(marker in description for marker in HDR_SIDE_DATA):
return True
return False
def _escape_filter_value(value: str) -> str:
"""Escape a value through FFmpeg's option and filtergraph parser layers."""
def escape(text: str, special: str) -> str:
return "".join(f"\\{char}" if char in special else char for char in text)
option_escaped = escape(value, "\\':")
return escape(option_escaped, "\\'[],;")
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _citation_text(path: Path) -> str:
try:
value = path.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError) as exc:
raise BurnError(f"citation file is not readable UTF-8: {path}: {exc}") from exc
lines = [" ".join(line.split()) for line in value.splitlines()]
while lines and not lines[0]:
lines.pop(0)
while lines and not lines[-1]:
lines.pop()
normalized: list[str] = []
for line in lines:
if line or not normalized or normalized[-1]:
normalized.append(line)
value = "\n".join(normalized)
if not value:
raise BurnError("citation file cannot be empty")
if len(value) > 2000 or any(ord(character) < 32 and character != "\n" for character in value):
raise BurnError("citation file contains unsupported or excessive text")
return value
def _ass_time(seconds: float) -> str:
centiseconds = max(1, math.ceil(seconds * 100))
hours, remainder = divmod(centiseconds, 360000)
minutes, remainder = divmod(remainder, 6000)
whole_seconds, fraction = divmod(remainder, 100)
return f"{hours}:{minutes:02d}:{whole_seconds:02d}.{fraction:02d}"
def _ass_escape(value: str) -> str:
"""Losslessly encode untrusted visible text for an ASS Dialogue field."""
output: list[str] = []
for character in value:
if character == "\\":
output.append("\\" + ASS_WORD_JOINER)
elif character == "{":
output.append(r"\{{}")
elif character == "\n":
output.append(r"\N")
else:
output.append(character)
return "".join(output)
def _render_citation_ass(text: str, width: int, height: int, duration: float) -> str:
font_size = max(16, min(28, round(height * 0.022)))
notice_size = max(14, font_size - 4)
margin_x = max(18, round(width * 0.0208))
margin_y = max(18, round(height * 0.0278))
padding_x = max(12, round(width * 0.0083))
padding_y = max(10, round(height * 0.0111))
portrait = height > width
panel_width = min(
width - 2 * margin_x,
max(
round(width * (0.90 if portrait else 0.375)),
round(24 * font_size * 0.41) + 2 * padding_x,
),
)
max_columns = max(
24,
round((panel_width - 2 * padding_x) / (font_size * 0.41)),
)
raw_lines = text.splitlines()
notice: str | None = None
if len(raw_lines) >= 3 and raw_lines[1] == "":
notice = raw_lines[0]
raw_lines = raw_lines[2:]
wrapped_lines: list[str] = []
for paragraph in raw_lines:
wrapped_lines.extend(
textwrap.wrap(
paragraph,
width=max_columns,
break_long_words=False,
break_on_hyphens=False,
) or [""]
)
citation_display = _ass_escape("\n".join(wrapped_lines))
if notice is not None:
display = (
rf"{{\fs{notice_size}\1c&HCCCCCC&}}{_ass_escape(notice)}"
rf"\N{{\fs8}} \N{{\fs{font_size}\1c&HFFFFFF&}}{citation_display}"
)
else:
display = rf"{{\fs{font_size}\1c&HFFFFFF&}}{citation_display}"
line_height = font_size * 1.05
panel_height = math.ceil(
2 * padding_y
+ len(wrapped_lines) * line_height
+ (notice_size * 1.05 + 8 if notice is not None else 0)
+ 3
)
text_x = margin_x + padding_x
text_y = margin_y + padding_y
panel = (
rf"{{\an7\pos({margin_x},{margin_y})\p1\1c&H000000&\1a&H78&}}"
f"m 0 0 l {panel_width} 0 {panel_width} {panel_height} 0 {panel_height}"
)
return (
"[Script Info]\n"
"ScriptType: v4.00+\n"
"WrapStyle: 2\n"
"ScaledBorderAndShadow: yes\n"
f"PlayResX: {width}\nPlayResY: {height}\n\n"
"[V4+ Styles]\n"
"Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, "
"OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, "
"ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, "
"Alignment, MarginL, MarginR, MarginV, Encoding\n"
f"Style: Citation,Arial,{font_size},&H10FFFFFF,&H10FFFFFF,&H00000000,"
"&H00000000,0,0,0,0,100,100,0,0,1,0,0,7,0,0,0,1\n\n"
"[Events]\n"
"Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n"
f"Dialogue: 9,0:00:00.00,{_ass_time(duration)},Citation,,0,0,0,,{panel}\n"
f"Dialogue: 10,0:00:00.00,{_ass_time(duration)},Citation,,0,0,0,,"
rf"{{\an7\pos({text_x},{text_y})}}{display}\n"
)
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
try:
with os.fdopen(fd, "w", encoding="utf-8") as handle:
json.dump(value, handle, ensure_ascii=False, indent=2, sort_keys=True)
handle.write("\n")
handle.flush()
os.fsync(handle.fileno())
os.replace(temporary, path)
except BaseException:
try:
os.close(fd)
except OSError:
pass
Path(temporary).unlink(missing_ok=True)
raise
def _write_citation_receipt(output: Path, citation_file: Path) -> Path:
receipt = output.with_suffix(output.suffix + ".citation.json")
_atomic_json(
receipt,
{
"schema_version": 1,
"output_file": output.name,
"output_sha256": _sha256_file(output),
"citation_sha256": _sha256_file(citation_file),
"position": "top-left",
},
)
return receipt
def _validate_validation_report(subtitle: Path, report_path: Path) -> dict[str, Any]:
subtitle = subtitle.expanduser().resolve()
report_path = report_path.expanduser().resolve()
if not report_path.is_file():
raise BurnError(f"validation report does not exist or is not a file: {report_path}")
try:
report = json.loads(report_path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise BurnError(f"validation report is not valid UTF-8 JSON: {report_path}: {exc}") from exc
if not isinstance(report, dict):
raise BurnError("validation report root must be a JSON object")
if report.get("structurally_valid") is not True:
raise BurnError("validation report must declare structurally_valid=true")
if report.get("validation_scope") != "structural_source_integrity":
raise BurnError(
"validation report scope must be structural_source_integrity"
)
segment_count = report.get("segment_count")
translation_count = report.get("translation_count")
counts = (segment_count, translation_count)
if any(isinstance(value, bool) or not isinstance(value, int) or value <= 0 for value in counts):
raise BurnError(
"validation report segment_count and translation_count must be positive integers"
)
if segment_count != translation_count:
raise BurnError(
"validation report segment_count and translation_count must be equal"
)
outputs = report.get("outputs")
recorded_hash = outputs.get("bilingual.ass") if isinstance(outputs, dict) else None
if not isinstance(recorded_hash, str) or not re.fullmatch(
r"[0-9a-fA-F]{64}", recorded_hash
):
raise BurnError(
"validation report outputs['bilingual.ass'] must be a SHA-256 checksum"
)
if _sha256_file(subtitle) != recorded_hash.lower():
raise BurnError("bilingual.ass SHA-256 does not match the validation report")
return report
_FONT_FILE_SUFFIXES = frozenset({".ttf", ".otf", ".ttc"})
_FONT_DIRECTORIES = (
"~/Library/Fonts",
"/Library/Fonts",
"/System/Library/Fonts",
"~/.fonts",
"~/.local/share/fonts",
"/usr/share/fonts",
"/usr/local/share/fonts",
)
def _font_installed(family: str) -> bool | None:
"""Return True/False when detection is trustworthy, None when unavailable.
libass silently substitutes another font when the requested family is
missing, which would pass every later gate with the wrong deliverable.
"""
fc_list = shutil.which("fc-list")
if fc_list:
result = subprocess.run(
[fc_list, ":", "family"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
check=False,
)
if result.returncode == 0 and result.stdout.strip():
needle = family.casefold()
return any(
needle in entry.strip().casefold()
for line in result.stdout.splitlines()
for entry in line.split(",")
)
token = re.sub(r"[\s_-]+", "", family).casefold()
if not token:
return None
searched = False
for directory in _FONT_DIRECTORIES:
base = Path(directory).expanduser()
if not base.is_dir():
continue
searched = True
for path in base.rglob("*"):
if (
path.suffix.lower() in _FONT_FILE_SUFFIXES
and token in re.sub(r"[\s_-]+", "", path.stem).casefold()
):
return True
return False if searched else None
def _require_subtitle_font(report: dict[str, Any], *, allow_missing_font: bool) -> None:
font = str(report.get("font") or "").strip()
if not font:
return
installed = _font_installed(font)
if installed is True:
return
if installed is None:
print(
f"warning: could not verify that font {font!r} is installed; "
"libass substitutes missing fonts silently",
file=sys.stderr,
)
return
message = (
f"font {font!r} required by the validated subtitles was not found; install it "
"(MiSans: https://hyperos.mi.com/font/zh/download/)"
)
if allow_missing_font:
print(f"warning: {message}; continuing with libass substitution", file=sys.stderr)
return
raise BurnError(f"{message} or pass --allow-missing-font to accept substitution")
def _audio_options(audio_streams: Sequence[dict[str, Any]]) -> tuple[list[str], list[str]]:
if not audio_streams:
return [], []
options = ["-c:a", "copy"]
modes: list[str] = []
for output_index, stream in enumerate(audio_streams):
codec = str(stream.get("codec_name", "")).lower()
if codec in MP4_COPY_AUDIO_CODECS:
modes.append(f"audio {output_index}: copied {codec}")
continue
options.extend(
[
f"-c:a:{output_index}",
"aac",
f"-b:a:{output_index}",
"256k",
]
)
modes.append(f"audio {output_index}: {codec or 'unknown'} -> AAC")
return options, modes
def _encode_command(
ffmpeg: str,
video: Path,
subtitle: Path,
output: Path,
video_stream: dict[str, Any],
audio_streams: Sequence[dict[str, Any]],
*,
force: bool,
crf: int,
preset: str,
encoder: str,
citation_ass: Path | None = None,
) -> tuple[list[str], list[str]]:
try:
stream_index = int(video_stream["index"])
except (KeyError, TypeError, ValueError) as exc:
raise BurnError("input video stream has no valid index") from exc
audio_options, audio_modes = _audio_options(audio_streams)
subtitle_filters = [f"subtitles=filename={_escape_filter_value(str(subtitle))}"]
if citation_ass is not None:
subtitle_filters.append(
f"subtitles=filename={_escape_filter_value(str(citation_ass))}"
)
command = [
ffmpeg,
"-hide_banner",
"-loglevel",
"error",
"-nostats",
"-stats_period",
"1",
"-progress",
"pipe:1",
"-y" if force else "-n",
"-i",
str(video),
"-map",
f"0:{stream_index}",
"-map",
"0:a?",
"-map_metadata",
"0",
"-map_chapters",
"0",
"-sn",
"-dn",
"-vf",
",".join(subtitle_filters),
"-fps_mode:v:0",
"passthrough",
"-c:v",
encoder,
"-crf",
str(crf),
"-preset",
preset,
"-pix_fmt",
"yuv420p",
*audio_options,
"-movflags",
"+faststart",
"-f",
"mp4",
str(output),
]
return command, audio_modes
def _verify_output(
ffprobe: str,
output: Path,
input_video: dict[str, Any],
input_had_audio: bool,
*,
input_duration: float,
) -> None:
result = _probe(ffprobe, output)
file_format = result.get("format")
format_name = file_format.get("format_name", "") if isinstance(file_format, dict) else ""
if "mp4" not in str(format_name).split(","):
raise BurnError(f"output verification failed: container is not MP4 ({format_name or 'unknown'})")
output_videos = _streams(result, "video")
if not output_videos:
raise BurnError("output verification failed: no video stream")
output_video = output_videos[0]
if output_video.get("codec_name") != "h264":
raise BurnError(
"output verification failed: video codec is "
f"{output_video.get('codec_name', 'unknown')}, not H.264"
)
output_duration = _duration(result)
if output_duration <= 0:
raise BurnError("output verification failed: duration is zero or unavailable")
if input_duration <= 0:
raise BurnError("output verification failed: input duration is zero or unavailable")
duration_tolerance = max(0.5, input_duration * 0.01)
if abs(output_duration - input_duration) > duration_tolerance:
raise BurnError(
"output verification failed: duration changed from "
f"{input_duration:.3f}s to {output_duration:.3f}s "
f"(allowed difference {duration_tolerance:.3f}s)"
)
input_dimensions = _stream_dimensions(input_video)
output_dimensions = _stream_dimensions(output_video)
if output_dimensions != input_dimensions:
raise BurnError(
"output verification failed: dimensions changed from "
f"{input_dimensions[0]}x{input_dimensions[1]} to "
f"{output_dimensions[0]}x{output_dimensions[1]}"
)
input_rate = _frame_rate(input_video)
output_rate = _frame_rate(output_video)
if input_rate is not None and output_rate is not None:
relative_drift = abs(float(output_rate - input_rate)) / float(input_rate)
if relative_drift > 0.005:
raise BurnError(
"output verification failed: frame rate changed from "
f"{float(input_rate):.6g} to {float(output_rate):.6g} fps"
)
if input_had_audio and not _streams(result, "audio"):
raise BurnError("output verification failed: input audio is missing from output")
def burn_subtitles(
video: Path,
subtitle: Path,
output: Path,
*,
force: bool = False,
crf: int = 18,
preset: str = "slow",
encoder: str = DEFAULT_ENCODER,
validation_report: Path | None = None,
allow_missing_font: bool = False,
citation_file: Path | None = None,
) -> list[str]:
video = video.expanduser().resolve()
subtitle = subtitle.expanduser().resolve()
output = output.expanduser().resolve()
report_path = (
validation_report.expanduser().resolve()
if validation_report is not None
else subtitle.with_name("validation.json")
)
citation_path = citation_file.expanduser().resolve() if citation_file is not None else None
if not video.is_file():
raise BurnError(f"input video does not exist or is not a file: {video}")
if not subtitle.is_file():
raise BurnError(f"ASS subtitle does not exist or is not a file: {subtitle}")
if subtitle.suffix.lower() != ".ass":
raise BurnError(f"subtitle must be an .ass file: {subtitle}")
if output in (video, subtitle, report_path):
raise BurnError("output must be different from all input files")
if citation_path is not None and not citation_path.is_file():
raise BurnError(f"citation file does not exist or is not a file: {citation_path}")
if citation_path is not None and output == citation_path:
raise BurnError("output must be different from the citation file")
if not output.parent.is_dir():
raise BurnError(f"output directory does not exist: {output.parent}")
if output.exists() and not force:
raise BurnError(f"output already exists (use --force to replace it): {output}")
if output.exists() and not output.is_file():
raise BurnError(f"output exists and is not a regular file: {output}")
if not preset.strip():
raise BurnError("encoder preset cannot be empty")
if not encoder.strip():
raise BurnError("encoder cannot be empty")
report = _validate_validation_report(subtitle, report_path)
_require_subtitle_font(report, allow_missing_font=allow_missing_font)
ffmpeg, ffprobe = _required_executables()
_require_libass_subtitles_filter(ffmpeg)
input_probe = _probe(ffprobe, video)
input_video = _main_video_stream(input_probe)
width, height = _stream_dimensions(input_video)
input_duration = _duration(input_probe)
if input_duration <= 0:
raise BurnError("input duration is zero or unavailable")
audio_streams = _streams(input_probe, "audio")
if _is_hdr(input_video):
print(
"warning: HDR input detected. The compatibility H.264/yuv420p output is "
"intended for SDR playback; HDR metadata and appearance may not be preserved.",
file=sys.stderr,
)
citation_ass: Path | None = None
temporary_dir: tempfile.TemporaryDirectory[str] | None = None
if citation_path is not None:
temporary_dir = tempfile.TemporaryDirectory(prefix="materialsub-citation-")
citation_ass = Path(temporary_dir.name) / "citation.ass"
citation_ass.write_text(
_render_citation_ass(_citation_text(citation_path), width, height, input_duration),
encoding="utf-8",
)
try:
command, audio_modes = _encode_command(
ffmpeg,
video,
subtitle,
output,
input_video,
audio_streams,
force=force,
crf=crf,
preset=preset,
encoder=encoder,
citation_ass=citation_ass,
)
returncode, diagnostic = _run_ffmpeg_with_progress(command, input_duration)
finally:
if temporary_dir is not None:
temporary_dir.cleanup()
if returncode != 0:
if output.is_file():
output.unlink()
detail = _last_error_line(diagnostic)
raise BurnError(f"FFmpeg subtitle burn failed with exit code {returncode}{detail}")
try:
_verify_output(
ffprobe,
output,
input_video,
bool(audio_streams),
input_duration=input_duration,
)
except BurnError:
if output.is_file():
output.unlink()
raise
receipt = output.with_suffix(output.suffix + ".citation.json")
if citation_path is not None:
_write_citation_receipt(output, citation_path)
elif receipt.is_file():
receipt.unlink()
return audio_modes
def main(argv: Sequence[str] | None = None) -> int:
args = _parser().parse_args(argv)
try:
audio_modes = burn_subtitles(
args.video,
args.subtitle,
args.output,
force=args.force,
crf=args.crf,
preset=args.preset,
encoder=args.encoder,
validation_report=args.validation_report,
allow_missing_font=args.allow_missing_font,
citation_file=args.citation_file,
)
except (BurnError, OSError) as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
print(
"validated ASS was burned into verified MP4: "
f"{args.output.expanduser().resolve()}"
)
for mode in audio_modes:
print(mode)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+196
View File
@@ -0,0 +1,196 @@
#!/usr/bin/env python3
"""Attach a user-approved citation watermark specification to a MaterialSub job."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
from pathlib import Path
import re
import sys
import tempfile
from typing import Any, Sequence
from urllib.parse import parse_qsl, urlsplit
MAX_CITATION_CHARACTERS = 1200
DEFAULT_INTERNAL_NOTICE = "内容引自网络,仅供内部交流"
SENSITIVE_QUERY_KEYS = re.compile(
r"(?:^|_)(?:auth|credential|expires?|key|policy|signature|signed|token)(?:_|$)",
re.IGNORECASE,
)
class CitationError(RuntimeError):
"""A malformed citation request or job manifest."""
def _read_json(path: Path) -> dict[str, Any]:
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise CitationError(f"could not read JSON manifest {path}: {exc}") from exc
if not isinstance(value, dict):
raise CitationError("download manifest root must be an object")
return value
def _normalize_citation(value: str) -> str:
lines = [" ".join(line.split()) for line in value.splitlines()]
value = "\n".join(line for line in lines if line)
if not value:
raise CitationError("citation cannot be empty")
if len(value) > MAX_CITATION_CHARACTERS:
raise CitationError(
f"citation is too long ({len(value)} characters; maximum {MAX_CITATION_CHARACTERS})"
)
if any(ord(character) < 32 and character != "\n" for character in value):
raise CitationError("citation contains unsupported control characters")
return value
def _normalize_notice(value: str) -> str:
value = " ".join(value.split())
if not value:
raise CitationError("notice cannot be empty")
if len(value) > 120 or any(ord(character) < 32 for character in value):
raise CitationError("notice contains unsupported or excessive text")
return value
def _public_source_url(manifest: dict[str, Any]) -> str:
source = manifest.get("source")
value = source.get("url") if isinstance(source, dict) else None
if not isinstance(value, str) or not value.strip():
raise CitationError("download manifest has no canonical source URL")
parts = urlsplit(value.strip())
if parts.scheme not in ("http", "https") or not parts.netloc or parts.username or parts.password:
raise CitationError("manifest source URL is not a public HTTP(S) URL")
for key, _ in parse_qsl(parts.query, keep_blank_values=True):
if SENSITIVE_QUERY_KEYS.search(key):
raise CitationError("refusing to put a signed or credential-like URL in a watermark")
return value.strip()
def _atomic_write_text(path: Path, value: str) -> None:
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
try:
with os.fdopen(fd, "w", encoding="utf-8") as handle:
handle.write(value)
handle.flush()
os.fsync(handle.fileno())
os.replace(temporary, path)
except BaseException:
try:
os.close(fd)
except OSError:
pass
Path(temporary).unlink(missing_ok=True)
raise
def _atomic_write_json(path: Path, value: dict[str, Any]) -> None:
_atomic_write_text(
path,
json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
)
def attach_citation(
manifest_path: Path,
citation: str,
*,
include_source_url: bool = False,
notice: str | None = None,
) -> Path:
manifest_path = manifest_path.expanduser().resolve()
manifest = _read_json(manifest_path)
citation = _normalize_citation(citation)
notice = _normalize_notice(notice) if notice is not None else None
source_url = _public_source_url(manifest) if include_source_url else None
citation_body = citation + (f"\nSource: {source_url}" if source_url else "")
rendered_text = (f"{notice}\n\n" if notice else "") + citation_body + "\n"
citation_path = manifest_path.parent / "citation-watermark.txt"
_atomic_write_text(citation_path, rendered_text)
digest = hashlib.sha256(citation_path.read_bytes()).hexdigest()
manifest["citation_watermark"] = {
"enabled": True,
"citation_file": citation_path.name,
"citation_sha256": digest,
"include_source_url": include_source_url,
"layout": (
"notice-plus-three-line-citation"
if notice and len(citation.splitlines()) == 3 and not include_source_url
else "notice-plus-citation" if notice else "citation-only"
),
"notice": notice,
"position": "top-left",
}
_atomic_write_json(manifest_path, manifest)
return citation_path
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Bind a user-approved top-left citation watermark to a MaterialSub job."
)
parser.add_argument("manifest", type=Path, help="download-manifest.json")
citation = parser.add_mutually_exclusive_group()
citation.add_argument("--citation", help="complete formal citation")
citation.add_argument("--citation-file", type=Path, help="UTF-8 file containing the citation")
parser.add_argument("--authors", help="citation authors line")
parser.add_argument("--title", help="citation title line")
parser.add_argument("--publication", help="journal, identifiers, and year line")
parser.add_argument(
"--notice",
nargs="?",
const=DEFAULT_INTERNAL_NOTICE,
help=(
"add an approved notice above the citation; without TEXT uses "
f"{DEFAULT_INTERNAL_NOTICE!r}"
),
)
parser.add_argument(
"--include-source-url",
action="store_true",
help="also show the manifest's canonical public source URL",
)
return parser
def main(argv: Sequence[str] | None = None) -> int:
args = _parser().parse_args(argv)
try:
structured = (args.authors, args.title, args.publication)
has_structured = any(value is not None for value in structured)
if has_structured:
if not all(value is not None for value in structured):
raise CitationError("--authors, --title, and --publication must be used together")
if args.citation is not None or args.citation_file is not None:
raise CitationError("structured citation fields cannot be combined with --citation")
citation = "\n".join(str(value) for value in structured)
elif args.citation_file is not None:
citation = args.citation_file.read_text(encoding="utf-8")
elif args.citation is not None:
citation = args.citation
else:
raise CitationError(
"provide --citation, --citation-file, or all three structured citation fields"
)
path = attach_citation(
args.manifest,
citation,
include_source_url=args.include_source_url,
notice=args.notice,
)
except (CitationError, OSError, UnicodeDecodeError) as exc:
print(f"citation watermark error: {exc}", file=sys.stderr)
return 2
print(path)
return 0
if __name__ == "__main__":
raise SystemExit(main())
File diff suppressed because it is too large Load Diff
+527
View File
@@ -0,0 +1,527 @@
#!/usr/bin/env python3
"""Ingest one browser-confirmed, authorized embedded HLS video.
The browser-facing agent writes a private resource map after it has verified
that playback is authorized. This script never receives signed URLs on the
command line, never prints them, and propagates a playlist's authorization
query only to same-origin HLS resources. It rejects master playlists and DRM
methods so the ordinary yt-dlp route remains the preferred path.
"""
from __future__ import annotations
import argparse
import json
import os
import re
import tempfile
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Any, Sequence
import fetch_video as fetch
RESOURCE_MAP_SCHEMA = 1
ALLOWED_SUBTITLE_KINDS = {"manual", "automatic"}
_URI_ATTRIBUTE = re.compile(r'URI="([^"]+)"')
_EXTENSION = re.compile(r"^[A-Za-z0-9]{1,8}$")
class IngestError(fetch.FetchError):
"""An embedded-resource ingest failed safely."""
def _read_private_resource_map(path: Path) -> dict[str, Any]:
path = path.expanduser().resolve()
if not path.is_file():
raise IngestError(f"Resource map does not exist or is not a file: {path}")
if path.stat().st_mode & 0o077:
raise IngestError("Resource map must be private (chmod 600) because it may contain signed URLs")
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
raise IngestError("Could not read the private resource map") from exc
if not isinstance(value, dict) or value.get("schema_version") != RESOURCE_MAP_SCHEMA:
raise IngestError(f"Resource map must be a schema_version={RESOURCE_MAP_SCHEMA} object")
return value
def _prepare_output_dir(
output_dir: Path,
resource_map_path: Path,
*,
resume: bool,
cleanup_resource_map: bool,
) -> Path:
if output_dir.exists() and not output_dir.is_dir():
raise IngestError(f"Output path is not a directory: {output_dir}")
output_dir.mkdir(parents=True, exist_ok=True)
if resume:
return output_dir
entries = list(output_dir.iterdir())
allowed_map = (
cleanup_resource_map
and resource_map_path.parent == output_dir
and resource_map_path in entries
)
unexpected = [path for path in entries if not (allowed_map and path == resource_map_path)]
if unexpected or (entries and not allowed_map):
raise IngestError(
f"Output directory is not empty: {output_dir}. Choose a new directory or pass --resume explicitly."
)
return output_dir
def _required_text(value: Any, field: str) -> str:
if not isinstance(value, str) or not value.strip():
raise IngestError(f"Resource map field {field!r} must be a non-empty string")
return value.strip()
def _optional_number(value: Any, field: str) -> float | int | None:
if value is None:
return None
if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
raise IngestError(f"Resource map field {field!r} must be a positive number")
return value
def _validated_url(value: Any, field: str) -> str:
try:
return fetch.validate_url(_required_text(value, field))
except fetch.FetchError as exc:
raise IngestError(f"Invalid resource map URL field {field!r}") from exc
def _same_origin(left: str, right: str) -> bool:
a = urllib.parse.urlsplit(left)
b = urllib.parse.urlsplit(right)
return (a.scheme.lower(), a.hostname, a.port) == (b.scheme.lower(), b.hostname, b.port)
def _authorized_resource_url(resource_url: str, playlist_url: str) -> str:
"""Add missing root-playlist query fields to same-origin resources only."""
absolute = urllib.parse.urljoin(playlist_url, resource_url)
if not _same_origin(absolute, playlist_url):
return absolute
root = urllib.parse.urlsplit(playlist_url)
target = urllib.parse.urlsplit(absolute)
root_query = urllib.parse.parse_qsl(root.query, keep_blank_values=True)
target_query = urllib.parse.parse_qsl(target.query, keep_blank_values=True)
existing = {key for key, _ in target_query}
merged = target_query + [(key, value) for key, value in root_query if key not in existing]
return urllib.parse.urlunsplit(
(target.scheme, target.netloc, target.path, urllib.parse.urlencode(merged), "")
)
def _validate_hls_protection(playlist: str) -> None:
for line in playlist.splitlines():
if not line.startswith("#EXT-X-KEY:"):
continue
attributes = line.split(":", 1)[1]
method = re.search(r"(?:^|,)METHOD=([^,]+)", attributes)
method_value = method.group(1).strip().upper() if method else ""
keyformat = re.search(r'(?:^|,)KEYFORMAT="?([^,"]+)', attributes)
keyformat_value = keyformat.group(1).strip().lower() if keyformat else "identity"
if method_value not in {"NONE", "AES-128"} or keyformat_value != "identity":
raise IngestError(
"Embedded HLS uses an unsupported protection method; MaterialSub will not bypass DRM"
)
def rewrite_media_playlist(playlist: str, playlist_url: str) -> str:
"""Return a signed, absolute media playlist without logging its secrets."""
if not playlist.lstrip().startswith("#EXTM3U"):
raise IngestError("The selected resource is not an HLS playlist")
if "#EXT-X-STREAM-INF" in playlist or "#EXT-X-MEDIA:" in playlist:
raise IngestError(
"The resource is an HLS master playlist; use the browser-observed media playlist selected during playback"
)
_validate_hls_protection(playlist)
rewritten: list[str] = []
for line in playlist.splitlines():
if line.startswith("#"):
line = _URI_ATTRIBUTE.sub(
lambda match: f'URI="{_authorized_resource_url(match.group(1), playlist_url)}"',
line,
)
elif line.strip():
line = _authorized_resource_url(line.strip(), playlist_url)
rewritten.append(line)
return "\n".join(rewritten) + "\n"
def _request_bytes(url: str, *, playlist_url: str | None = None) -> bytes:
resolved = _authorized_resource_url(url, playlist_url) if playlist_url else url
request = urllib.request.Request(resolved, headers={"User-Agent": "Mozilla/5.0 MaterialSub/1"})
try:
with urllib.request.urlopen(request, timeout=45) as response:
return response.read()
except Exception as exc:
raise IngestError(f"Failed to download an authorized resource from {fetch.display_url(url)}") from exc
def _atomic_download(url: str, destination: Path, *, playlist_url: str | None = None) -> Path:
if destination.exists():
raise IngestError(f"Refusing to replace existing artifact without --resume: {destination}")
payload = _request_bytes(url, playlist_url=playlist_url)
if not payload:
raise IngestError(f"Downloaded resource is empty: {fetch.display_url(url)}")
fd, temporary_name = tempfile.mkstemp(
prefix=f".{destination.stem}.", suffix=destination.suffix, dir=destination.parent
)
try:
with os.fdopen(fd, "wb") as handle:
handle.write(payload)
handle.flush()
os.fsync(handle.fileno())
os.replace(temporary_name, destination)
finally:
Path(temporary_name).unlink(missing_ok=True)
return destination
def _download_cover(
url: str | None,
destination: Path,
*,
playlist_url: str,
ffmpeg: str,
resume: bool,
) -> Path | None:
if not url:
return None
if destination.exists() and resume:
return destination
with tempfile.TemporaryDirectory(prefix="materialsub-cover-") as directory:
raw = Path(directory) / "cover.input"
raw.write_bytes(_request_bytes(url, playlist_url=playlist_url))
if not raw.stat().st_size:
raise IngestError("Downloaded cover is empty")
temporary = Path(directory) / "cover.jpg"
result = fetch._run(
[
ffmpeg,
"-hide_banner",
"-loglevel",
"error",
"-nostdin",
"-i",
str(raw),
"-frames:v",
"1",
str(temporary),
],
"converting the embedded cover to JPEG",
secrets=(url, str(raw), str(temporary)),
check=False,
)
if result.returncode or not temporary.is_file() or not temporary.stat().st_size:
raise IngestError("Failed to convert the embedded cover to JPEG")
os.replace(temporary, destination)
return destination
def _subtitle_choice(resource_map: dict[str, Any], target_language: str) -> tuple[fetch.SubtitleChoice | None, dict[str, Any] | None]:
subtitle = resource_map.get("subtitle")
if subtitle is None:
return None, None
if not isinstance(subtitle, dict):
raise IngestError("Resource map subtitle must be an object or null")
url = _validated_url(subtitle.get("url"), "subtitle.url")
language = _required_text(subtitle.get("language"), "subtitle.language")
kind = _required_text(subtitle.get("kind"), "subtitle.kind").lower()
if kind not in ALLOWED_SUBTITLE_KINDS:
raise IngestError("subtitle.kind must be 'manual' or 'automatic'")
original_format = str(subtitle.get("format") or Path(urllib.parse.urlsplit(url).path).suffix.lstrip(".") or "vtt").lower()
if not _EXTENSION.fullmatch(original_format):
raise IngestError("subtitle.format must be a short filename extension")
if fetch._excluded_language(language, fetch._target_language_bases(target_language)):
raise IngestError("Selected subtitle is not a foreign-language dialogue track for the requested target language")
return fetch.SubtitleChoice(language, kind, original_format, (original_format,)), subtitle
def _download_hls(
playlist_url: str,
output_dir: Path,
base: str,
*,
yt_dlp: str,
concurrent_fragments: int,
) -> tuple[Path, Any]:
source = _request_bytes(playlist_url).decode("utf-8-sig")
rewritten = rewrite_media_playlist(source, playlist_url)
with tempfile.TemporaryDirectory(prefix="materialsub-authorized-hls-") as directory:
playlist_path = Path(directory) / "media.m3u8"
playlist_path.write_text(rewritten, encoding="utf-8")
os.chmod(playlist_path, 0o600)
result = fetch._run(
[
yt_dlp,
"--ignore-config",
"--no-playlist",
"--no-write-playlist-metafiles",
"--no-progress",
"--enable-file-urls",
"--concurrent-fragments",
str(concurrent_fragments),
"-P",
str(output_dir),
"-f",
fetch.FORMAT_SELECTOR,
"--merge-output-format",
"mkv",
"--remux-video",
"mkv",
"--no-overwrites",
"--no-post-overwrites",
"-o",
f"{base}.intermediate.%(ext)s",
playlist_path.as_uri(),
],
"downloading the authorized embedded HLS media playlist",
secrets=(playlist_url, urllib.parse.urlsplit(playlist_url).query, rewritten),
)
intermediate = fetch._artifact(output_dir, f"{base}.intermediate.")
if intermediate is None:
raise IngestError("yt-dlp completed but no embedded HLS intermediate was written")
return intermediate, result
def _build_manifest(
resource_map: dict[str, Any],
output_dir: Path,
*,
target_language: str,
deliverable: str,
choice: fetch.SubtitleChoice | None,
) -> dict[str, Any]:
title = _required_text(resource_map.get("title"), "title")
video_id = _required_text(resource_map.get("id"), "id")
page_url = _validated_url(resource_map.get("page_url"), "page_url")
info = {
"title": title,
"id": video_id,
"webpage_url": page_url,
"extractor_key": "AuthorizedEmbeddedHLS",
"duration": _optional_number(resource_map.get("duration_seconds"), "duration_seconds"),
"width": _optional_number(resource_map.get("width"), "width"),
"height": _optional_number(resource_map.get("height"), "height"),
"language": choice.language if choice else resource_map.get("declared_language"),
}
manifest = fetch._manifest_base(
info=info,
url=page_url,
output_dir=output_dir,
browser_cookies="browser-session",
allow_remote_ejs=False,
choice=choice,
deliverable=deliverable,
target_language=target_language,
)
manifest["authentication"]["mode"] = "browser-confirmed-resource-map"
manifest["execution"] = {
"resume": False,
"embedded_hls": True,
"authorization_query_logged": False,
"temporary_playlist_cleaned": True,
}
return manifest
def execute(args: argparse.Namespace) -> int:
resource_map_path = args.resource_map.expanduser().resolve()
resource_map = _read_private_resource_map(resource_map_path)
target_language = fetch._validated_target_language(args.target_lang)
playlist_url = _validated_url(resource_map.get("playlist_url"), "playlist_url")
choice, subtitle_config = _subtitle_choice(resource_map, target_language)
if args.deliver in fetch.SUBTITLE_ONLY_DELIVERABLES and choice is None:
raise IngestError("A subtitle-only delivery requires one selected source subtitle")
yt_dlp = fetch._require_executable("yt-dlp", "Install the current official yt-dlp release.")
ffmpeg = fetch._require_executable("ffmpeg", "Install FFmpeg.")
ffprobe = fetch._require_executable("ffprobe", "Install FFmpeg with ffprobe.")
output_dir = _prepare_output_dir(
args.output_dir.expanduser().resolve(),
resource_map_path,
resume=args.resume,
cleanup_resource_map=args.cleanup_resource_map,
)
manifest = _build_manifest(
resource_map,
output_dir,
target_language=target_language,
deliverable=args.deliver,
choice=choice,
)
manifest["execution"]["resume"] = bool(args.resume)
title = manifest["source"]["title"]
video_id = manifest["source"]["id"]
base = fetch.safe_stem(title, video_id)
names = manifest["delivery_names"]
subtitles_only = args.deliver in fetch.SUBTITLE_ONLY_DELIVERABLES
warnings: list[str] = []
completed: list[Any] = []
intermediate: Path | None = None
media_probe: dict[str, Any] = {}
master: Path | None = None
fallback: Path | None = None
fallback_error: str | None = None
if not subtitles_only:
print("Downloading authorized embedded HLS at maximum observed quality…", file=os.sys.stderr)
intermediate, completed_result = _download_hls(
playlist_url,
output_dir,
base,
yt_dlp=yt_dlp,
concurrent_fragments=args.concurrent_fragments,
)
completed.append(completed_result)
media_probe = fetch._ffprobe(intermediate, ffprobe)
if not any(
isinstance(stream, dict) and stream.get("codec_type") == "video"
for stream in media_probe.get("streams", [])
):
raise IngestError("Embedded HLS intermediate has no video stream")
master, remux_error = fetch._try_lossless_mp4(
intermediate,
output_dir / f"{base}.master.mp4",
ffmpeg,
replace_existing=args.resume,
)
if remux_error:
warnings.append(f"Lossless MP4 remux unavailable: {remux_error}")
if master is None and args.mp4_fallback:
fallback, fallback_error = fetch._create_fallback_mp4(
intermediate,
output_dir / f"{base}.fallback.mp4",
ffmpeg,
replace_existing=args.resume,
)
if fallback_error:
raise IngestError(f"Requested MP4 fallback failed: {fallback_error}")
cover = _download_cover(
resource_map.get("cover_url"),
output_dir / names["cover"],
playlist_url=playlist_url,
ffmpeg=ffmpeg,
resume=args.resume,
)
if not subtitles_only and cover is None:
warnings.append("The embedded player resource map did not declare a cover")
original_subtitle: Path | None = None
source_srt: Path | None = None
conversion_method: str | None = None
parent_hash: str | None = None
if choice and subtitle_config:
language_label = fetch._subtitle_language_label(choice.language)
original_subtitle = output_dir / (
f"{base}.source-original.{language_label}.{choice.original_format}"
)
if not (args.resume and original_subtitle.is_file()):
_atomic_download(
_validated_url(subtitle_config.get("url"), "subtitle.url"),
original_subtitle,
playlist_url=playlist_url,
)
source_srt, conversion_method, parent_hash = fetch._derive_source_srt(
original_subtitle,
output_dir / f"{base}.source-srt.{language_label}.srt",
ffmpeg=ffmpeg,
replace_existing=args.resume,
)
original_record = fetch._file_record(original_subtitle, output_dir, checksum=True)
source_srt_record = fetch._file_record(source_srt, output_dir, checksum=True)
if original_record and source_srt_record:
if original_record.get("sha256") != parent_hash:
raise IngestError("Original subtitle changed during SRT derivation")
original_record["content_role"] = "immutable-parent"
source_srt_record["conversion_method"] = conversion_method
source_srt_record["derived_from"] = {
"path": original_record["path"],
"sha256": parent_hash,
}
manifest["status"] = "downloaded"
manifest["artifacts"] = {
"intermediate": fetch._file_record(intermediate, output_dir),
"media_streams": media_probe.get("streams", []),
"lossless_mp4_master": fetch._file_record(master, output_dir),
"lossy_mp4_fallback": {
"requested": bool(args.mp4_fallback),
"created": fetch._file_record(fallback, output_dir),
"reason_not_created": (
"lossless_master_available"
if args.mp4_fallback and master is not None
else (fallback_error if args.mp4_fallback and fallback is None else None)
),
"video_encoding": "libx264 preset=slow crf=18" if fallback else None,
"audio_encoding": "aac 256k" if fallback else None,
},
"cover": fetch._file_record(cover, output_dir, checksum=True),
"subtitle": {
"language": choice.language,
"kind": choice.kind,
"label": subtitle_config.get("label"),
"original": original_record,
"source_srt": source_srt_record,
"original_is_never_modified_by_this_script": True,
}
if choice and subtitle_config
else None,
}
manifest["warnings"].extend(warnings)
manifest["warnings"].extend(
warning
for warning in fetch._warning_lines(
completed, (playlist_url, urllib.parse.urlsplit(playlist_url).query)
)
if warning not in manifest["warnings"]
)
destination = fetch._write_manifest(output_dir, manifest)
exit_code = fetch._advance_bilingual_stage(destination)
if args.cleanup_resource_map:
resource_map_path.unlink()
return exit_code
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Ingest one browser-confirmed authorized embedded HLS media playlist."
)
parser.add_argument("--resource-map", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--target-lang", default=fetch.DEFAULT_TARGET_LANGUAGE)
parser.add_argument("--deliver", choices=fetch.DELIVERABLES, default="full")
parser.add_argument("--concurrent-fragments", type=int, default=8)
parser.add_argument("--mp4-fallback", action="store_true")
parser.add_argument("--resume", action="store_true")
parser.add_argument(
"--cleanup-resource-map",
action="store_true",
help="delete the exact private resource-map file after a successful manifest write",
)
return parser
def main(argv: Sequence[str] | None = None) -> int:
parser = _parser()
args = parser.parse_args(argv)
if not 1 <= args.concurrent_fragments <= 32:
parser.error("--concurrent-fragments must be between 1 and 32")
try:
return execute(args)
except (IngestError, fetch.FetchError) as exc:
print(f"error: {fetch.sanitize_diagnostic(str(exc))}", file=os.sys.stderr)
return 2
if __name__ == "__main__":
raise SystemExit(main())
+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env python3
"""Run MaterialSub dependency and font checks before a long download."""
from __future__ import annotations
import argparse
import json
import shutil
import sys
from typing import Any, Sequence
import burn_subtitles as burn
def assess(*, font: str, allow_missing_font: bool, youtube: bool) -> tuple[int, dict[str, Any]]:
if sys.version_info < (3, 10):
return 2, {
"complete": False,
"stage": "dependency_required",
"error": "Python 3.10 or newer is required",
}
yt_dlp = shutil.which("yt-dlp")
if not yt_dlp:
return 2, {
"complete": False,
"stage": "dependency_required",
"error": "yt-dlp was not found in PATH",
}
try:
ffmpeg, ffprobe = burn._required_executables()
burn._require_libass_subtitles_filter(ffmpeg)
except burn.BurnError as exc:
return 2, {
"complete": False,
"stage": "dependency_required",
"error": str(exc),
}
warnings: list[str] = []
if youtube and not shutil.which("deno"):
warnings.append(
"Deno was not found; current yt-dlp may expose fewer YouTube formats"
)
installed = burn._font_installed(font)
if installed is False and not allow_missing_font:
return 3, {
"complete": False,
"stage": "font_decision_required",
"font": font,
"font_installed": False,
"substitution_allowed": False,
"instruction": (
"Install the requested font, or rerun preflight and the final burn "
"with --allow-missing-font to accept libass substitution"
),
"warnings": warnings,
}
if installed is False:
warnings.append(f"Font {font!r} is missing; libass substitution was accepted")
elif installed is None:
warnings.append(f"Could not verify whether font {font!r} is installed")
return 0, {
"complete": True,
"stage": "preflight_complete",
"font": font,
"font_installed": installed,
"substitution_allowed": bool(installed is False and allow_missing_font),
"executables": {
"yt_dlp": yt_dlp,
"ffmpeg": ffmpeg,
"ffprobe": ffprobe,
},
"warnings": warnings,
}
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Check MaterialSub dependencies before downloading.")
parser.add_argument("--font", default="MiSans")
parser.add_argument("--allow-missing-font", action="store_true")
parser.add_argument("--youtube", action="store_true")
return parser
def main(argv: Sequence[str] | None = None) -> int:
args = _parser().parse_args(argv)
font = args.font.strip()
if not font:
print(json.dumps({"complete": False, "error": "font cannot be empty"}))
return 2
exit_code, result = assess(
font=font,
allow_missing_font=args.allow_missing_font,
youtube=args.youtube,
)
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
return exit_code
if __name__ == "__main__":
raise SystemExit(main())
File diff suppressed because it is too large Load Diff
+247
View File
@@ -0,0 +1,247 @@
#!/usr/bin/env python3
"""Fail closed until a downloaded video job reaches its required deliverable."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
import re
import sys
from typing import Any, Sequence
class DeliveryError(RuntimeError):
"""A malformed or unreadable delivery job."""
def _read_json(path: Path) -> dict[str, Any]:
try:
value = json.loads(path.read_text(encoding="utf-8"))
except FileNotFoundError as exc:
raise DeliveryError(f"manifest not found: {path}") from exc
except json.JSONDecodeError as exc:
raise DeliveryError(f"invalid JSON in {path}: {exc}") from exc
if not isinstance(value, dict):
raise DeliveryError(f"manifest root must be an object: {path}")
return value
def _artifact_path(job_dir: Path, value: Any) -> Path | None:
if not isinstance(value, dict) or not isinstance(value.get("path"), str):
return None
path = Path(value["path"])
return path if path.is_absolute() else job_dir / path
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _verify_citation_watermark(job_dir: Path, download: dict[str, Any], burned: Path) -> None:
citation = download.get("citation_watermark")
if not isinstance(citation, dict) or citation.get("enabled") is not True:
return
citation_name = citation.get("citation_file")
citation_hash = citation.get("citation_sha256")
if not isinstance(citation_name, str) or Path(citation_name).name != citation_name:
raise DeliveryError("citation_watermark.citation_file must be a plain filename")
if not isinstance(citation_hash, str) or not re.fullmatch(r"[0-9a-fA-F]{64}", citation_hash):
raise DeliveryError("citation_watermark.citation_sha256 is invalid")
citation_path = job_dir / citation_name
if not citation_path.is_file() or _sha256_file(citation_path) != citation_hash.lower():
raise DeliveryError("citation watermark text is missing or its checksum changed")
receipt_path = burned.with_suffix(burned.suffix + ".citation.json")
if not receipt_path.is_file():
raise DeliveryError(f"citation watermark burn receipt is missing: {receipt_path}")
receipt = _read_json(receipt_path)
expected = {
"output_file": burned.name,
"output_sha256": _sha256_file(burned),
"citation_sha256": citation_hash.lower(),
"position": "top-left",
}
for key, value in expected.items():
if receipt.get(key) != value:
raise DeliveryError(f"citation watermark receipt has a stale or invalid {key}")
def _existing_video_artifact(job_dir: Path, artifacts: dict[str, Any]) -> Path | None:
records = [artifacts.get("lossless_mp4_master"), artifacts.get("intermediate")]
fallback = artifacts.get("lossy_mp4_fallback")
if isinstance(fallback, dict):
records.append(fallback.get("created"))
for record in records:
path = _artifact_path(job_dir, record)
if path is not None and path.is_file() and path.stat().st_size:
return path
return None
DELIVERABLES = ("full", "video", "subs", "bilingual-subs")
def assess_delivery(download_manifest: Path) -> dict[str, Any]:
download_manifest = download_manifest.expanduser().resolve()
download = _read_json(download_manifest)
configured_dir = download.get("output_directory")
job_dir = (
Path(configured_dir).expanduser().resolve()
if isinstance(configured_dir, str)
else download_manifest.parent
)
deliverable = download.get("deliverable")
if deliverable not in DELIVERABLES:
deliverable = "full"
artifacts = download.get("artifacts")
if not isinstance(artifacts, dict):
raise DeliveryError("download manifest has no artifacts object")
if deliverable in ("full", "video") and _existing_video_artifact(job_dir, artifacts) is None:
raise DeliveryError("no declared video artifact exists on disk")
def complete(stage: str, **extra: Any) -> dict[str, Any]:
return {
"complete": True,
"stage": stage,
"deliverable": deliverable,
"job_dir": str(job_dir),
"missing": [],
**extra,
}
subtitle_record = artifacts.get("subtitle")
subtitle = (
_artifact_path(job_dir, subtitle_record.get("source_srt"))
if isinstance(subtitle_record, dict)
else None
)
if subtitle is not None and not subtitle.is_file():
raise DeliveryError(f"declared source subtitle is missing: {subtitle}")
if deliverable in ("subs", "bilingual-subs") and subtitle is None:
raise DeliveryError(
"a subtitle delivery was requested, but the manifest declares no source subtitle"
)
has_dialogue = (
subtitle is not None
and isinstance(subtitle_record, dict)
and subtitle_record.get("dialogue") is not False
)
if deliverable == "video":
return complete("video_complete")
if deliverable == "subs":
return complete("subs_complete")
if not has_dialogue:
# full falls back to plain video; bilingual-subs still delivered the
# source subtitle files even though nothing was translatable.
stage = "video_only_complete" if deliverable == "full" else "subs_complete"
return complete(stage)
subtitle_dir = job_dir / "subtitles"
subtitle_manifest_path = subtitle_dir / "subtitle-manifest.json"
if not subtitle_manifest_path.is_file():
return {
"complete": False,
"stage": "subtitle_prepare_required",
"job_dir": str(job_dir),
"missing": [str(subtitle_manifest_path)],
}
subtitle_manifest = _read_json(subtitle_manifest_path)
batches = subtitle_manifest.get("translation_batches")
if not isinstance(batches, list) or not batches:
raise DeliveryError("subtitle manifest has no translation batches")
output_dir_value = subtitle_manifest.get("translation_output_dir")
translation_output_dir = (
Path(output_dir_value)
if isinstance(output_dir_value, str)
else subtitle_dir / "translation-output"
)
missing_batches: list[str] = []
for batch in batches:
if not isinstance(batch, dict) or not isinstance(batch.get("path"), str):
raise DeliveryError("subtitle manifest has an invalid translation batch")
name = Path(batch["path"]).name
if not (translation_output_dir / name).is_file():
missing_batches.append(name)
if missing_batches:
return {
"complete": False,
"stage": "translation_required",
"job_dir": str(job_dir),
"missing": missing_batches,
}
rendered_dir = subtitle_dir / "rendered"
required_rendered = [rendered_dir / "bilingual.ass", rendered_dir / "validation.json"]
missing_rendered = [str(path) for path in required_rendered if not path.is_file()]
if missing_rendered:
return {
"complete": False,
"stage": "render_required",
"job_dir": str(job_dir),
"missing": missing_rendered,
}
if deliverable == "bilingual-subs":
return complete("bilingual_subs_complete", rendered_dir=str(rendered_dir))
delivery_names = download.get("delivery_names")
burned_name = (
delivery_names.get("bilingual_video")
if isinstance(delivery_names, dict)
else None
)
if isinstance(burned_name, str) and burned_name:
if Path(burned_name).name != burned_name:
raise DeliveryError("delivery_names.bilingual_video must be a plain filename")
burned = job_dir / burned_name
if burned.is_file() and burned.stat().st_size:
_verify_citation_watermark(job_dir, download, burned)
return complete("bilingual_complete", burned_video=str(burned))
missing = [str(burned)]
else:
legacy = sorted(
path
for path in job_dir.glob("*.bilingual.mp4")
if path.is_file() and path.stat().st_size
)
if legacy:
_verify_citation_watermark(job_dir, download, legacy[-1])
return complete("bilingual_complete", burned_video=str(legacy[-1]))
missing = ["*.bilingual.mp4"]
if missing:
return {
"complete": False,
"stage": "burn_required",
"job_dir": str(job_dir),
"missing": missing,
}
raise AssertionError("unreachable")
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Check whether a video job is video-only complete or bilingual complete."
)
parser.add_argument("download_manifest", type=Path)
return parser
def main(argv: Sequence[str] | None = None) -> int:
args = _parser().parse_args(argv)
try:
result = assess_delivery(args.download_manifest)
except (DeliveryError, OSError) as exc:
print(f"delivery verification error: {exc}", file=sys.stderr)
return 2
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
return 0 if result["complete"] else 3
if __name__ == "__main__":
raise SystemExit(main())