Initial MaterialSub release
Derived from pengchujin/jzsub at 222a90265d2a8797ca258eb1a980cee0863a8311; preserve the upstream MIT license and attribution.
This commit is contained in:
Executable
+972
@@ -0,0 +1,972 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Burn one ASS subtitle track into a high-quality H.264 MP4."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import deque
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
from fractions import Fraction
|
||||
from pathlib import Path
|
||||
from typing import Any, Sequence
|
||||
|
||||
|
||||
DEFAULT_ENCODER = "libx264"
|
||||
PROGRESS_BAR_WIDTH = 20
|
||||
PROGRESS_STEP_PERCENT = 5
|
||||
MP4_COPY_AUDIO_CODECS = frozenset({"aac", "ac3", "alac", "eac3", "mp3"})
|
||||
HDR_TRANSFERS = frozenset({"arib-std-b67", "smpte2084"})
|
||||
HDR_SIDE_DATA = (
|
||||
"content light level",
|
||||
"dolby vision",
|
||||
"dovi",
|
||||
"dynamic hdr",
|
||||
"hdr10+",
|
||||
"mastering display",
|
||||
)
|
||||
FFMPEG_FULL_CANDIDATES = (
|
||||
Path("/opt/homebrew/opt/ffmpeg-full/bin/ffmpeg"),
|
||||
Path("/usr/local/opt/ffmpeg-full/bin/ffmpeg"),
|
||||
)
|
||||
ASS_WORD_JOINER = "\u2060"
|
||||
|
||||
|
||||
class BurnError(RuntimeError):
|
||||
"""A user-actionable burn or verification failure."""
|
||||
|
||||
|
||||
def _positive_crf(value: str) -> int:
|
||||
try:
|
||||
crf = int(value)
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError("CRF must be an integer from 0 to 51") from exc
|
||||
if not 0 <= crf <= 51:
|
||||
raise argparse.ArgumentTypeError("CRF must be an integer from 0 to 51")
|
||||
return crf
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Burn an ASS subtitle file exactly once into an H.264/yuv420p MP4 "
|
||||
"while preserving the source dimensions and frame timing."
|
||||
)
|
||||
)
|
||||
parser.add_argument("video", type=Path, help="input video")
|
||||
parser.add_argument("subtitle", type=Path, help="input ASS subtitle file")
|
||||
parser.add_argument("output", type=Path, help="output MP4")
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="replace OUTPUT if it already exists",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--crf",
|
||||
type=_positive_crf,
|
||||
default=18,
|
||||
help="H.264 constant-rate-factor quality (default: 18)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--preset",
|
||||
default="slow",
|
||||
help="encoder preset (default: slow)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--encoder",
|
||||
default=DEFAULT_ENCODER,
|
||||
help=f"FFmpeg H.264 encoder (default: {DEFAULT_ENCODER})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--validation-report",
|
||||
type=Path,
|
||||
help="subtitle validation JSON (default: validation.json next to the ASS file)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--allow-missing-font",
|
||||
action="store_true",
|
||||
help="continue with libass font substitution when the validated font is not installed",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--citation-file",
|
||||
type=Path,
|
||||
help="approved UTF-8 citation text to burn at the top left",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def _required_executables() -> tuple[str, str]:
|
||||
ffmpeg = shutil.which("ffmpeg")
|
||||
ffprobe = shutil.which("ffprobe")
|
||||
missing = [name for name, path in (("ffmpeg", ffmpeg), ("ffprobe", ffprobe)) if not path]
|
||||
if missing:
|
||||
raise BurnError(f"required executable not found in PATH: {', '.join(missing)}")
|
||||
assert ffmpeg is not None and ffprobe is not None
|
||||
ffmpeg = _select_libass_ffmpeg(ffmpeg)
|
||||
sibling_ffprobe = Path(ffmpeg).with_name("ffprobe")
|
||||
if sibling_ffprobe.is_file():
|
||||
ffprobe = str(sibling_ffprobe)
|
||||
return ffmpeg, ffprobe
|
||||
|
||||
|
||||
def _ffmpeg_has_subtitles_filter(ffmpeg: str | Path) -> bool:
|
||||
result = subprocess.run(
|
||||
[str(ffmpeg), "-hide_banner", "-filters"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return result.returncode == 0 and any(
|
||||
len(fields := line.split()) >= 2 and fields[1] == "subtitles"
|
||||
for line in result.stdout.splitlines()
|
||||
)
|
||||
|
||||
|
||||
def _select_libass_ffmpeg(
|
||||
default: str,
|
||||
*,
|
||||
candidates: Sequence[Path] = FFMPEG_FULL_CANDIDATES,
|
||||
) -> str:
|
||||
for candidate in (Path(default), *candidates):
|
||||
if candidate.is_file() and _ffmpeg_has_subtitles_filter(candidate):
|
||||
return str(candidate)
|
||||
return default
|
||||
|
||||
|
||||
def _require_libass_subtitles_filter(ffmpeg: str) -> None:
|
||||
if not _ffmpeg_has_subtitles_filter(ffmpeg):
|
||||
raise BurnError(
|
||||
"FFmpeg has no usable 'subtitles' filter; install an FFmpeg build "
|
||||
"with libass support"
|
||||
)
|
||||
|
||||
|
||||
def _last_error_line(stderr: str) -> str:
|
||||
lines = [line.strip() for line in stderr.splitlines() if line.strip()]
|
||||
return f": {lines[-1]}" if lines else ""
|
||||
|
||||
|
||||
def _clock(seconds: float) -> str:
|
||||
total = max(0, int(seconds))
|
||||
hours, remainder = divmod(total, 3600)
|
||||
minutes, seconds = divmod(remainder, 60)
|
||||
if hours:
|
||||
return f"{hours:02d}:{minutes:02d}:{seconds:02d}"
|
||||
return f"{minutes:02d}:{seconds:02d}"
|
||||
|
||||
|
||||
def _format_progress(
|
||||
percent: int,
|
||||
encoded_seconds: float,
|
||||
duration: float,
|
||||
speed: str,
|
||||
) -> str:
|
||||
percent = max(0, min(100, int(percent)))
|
||||
filled = round(percent * PROGRESS_BAR_WIDTH / 100)
|
||||
bar = "█" * filled + "░" * (PROGRESS_BAR_WIDTH - filled)
|
||||
speed = speed.strip() or "--"
|
||||
return (
|
||||
f"烧录 [{bar}] {percent:3d}% "
|
||||
f"{_clock(encoded_seconds)} / {_clock(duration)} {speed}"
|
||||
)
|
||||
|
||||
|
||||
def _progress_seconds(values: dict[str, str]) -> float:
|
||||
raw = values.get("out_time_us") or values.get("out_time_ms")
|
||||
if raw:
|
||||
try:
|
||||
return max(0.0, int(raw) / 1_000_000)
|
||||
except ValueError:
|
||||
pass
|
||||
clock = values.get("out_time", "")
|
||||
try:
|
||||
hours, minutes, seconds = clock.split(":", 2)
|
||||
return max(0.0, int(hours) * 3600 + int(minutes) * 60 + float(seconds))
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def _run_ffmpeg_with_progress(command: Sequence[str], duration: float) -> tuple[int, str]:
|
||||
process = subprocess.Popen(
|
||||
list(command),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
bufsize=1,
|
||||
)
|
||||
if process.stdout is None:
|
||||
process.kill()
|
||||
raise BurnError("FFmpeg progress pipe was not available")
|
||||
|
||||
values: dict[str, str] = {}
|
||||
diagnostics: deque[str] = deque(maxlen=12)
|
||||
last_bucket = 0
|
||||
print(_format_progress(0, 0, duration, "--"), file=sys.stderr, flush=True)
|
||||
for raw_line in process.stdout:
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if "=" not in line:
|
||||
diagnostics.append(line)
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
values[key] = value
|
||||
if key != "progress":
|
||||
continue
|
||||
|
||||
encoded_seconds = _progress_seconds(values)
|
||||
raw_percent = 100 * encoded_seconds / duration if duration > 0 else 0
|
||||
bucket = min(
|
||||
100,
|
||||
int(raw_percent // PROGRESS_STEP_PERCENT) * PROGRESS_STEP_PERCENT,
|
||||
)
|
||||
if value == "end":
|
||||
bucket = 100
|
||||
encoded_seconds = duration
|
||||
if bucket > last_bucket:
|
||||
print(
|
||||
_format_progress(
|
||||
bucket,
|
||||
encoded_seconds,
|
||||
duration,
|
||||
values.get("speed", "--"),
|
||||
),
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
last_bucket = bucket
|
||||
|
||||
returncode = process.wait()
|
||||
process.stdout.close()
|
||||
return returncode, "\n".join(diagnostics)
|
||||
|
||||
|
||||
def _probe(ffprobe: str, path: Path) -> dict[str, Any]:
|
||||
result = subprocess.run(
|
||||
[
|
||||
ffprobe,
|
||||
"-v",
|
||||
"error",
|
||||
"-show_format",
|
||||
"-show_streams",
|
||||
"-of",
|
||||
"json",
|
||||
str(path),
|
||||
],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise BurnError(f"ffprobe could not read {path}{_last_error_line(result.stderr)}")
|
||||
try:
|
||||
data = json.loads(result.stdout)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise BurnError(f"ffprobe returned invalid JSON for {path}: {exc}") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise BurnError(f"ffprobe returned an unexpected result for {path}")
|
||||
return data
|
||||
|
||||
|
||||
def _streams(probe: dict[str, Any], kind: str) -> list[dict[str, Any]]:
|
||||
streams = probe.get("streams", [])
|
||||
if not isinstance(streams, list):
|
||||
return []
|
||||
return [
|
||||
stream
|
||||
for stream in streams
|
||||
if isinstance(stream, dict) and stream.get("codec_type") == kind
|
||||
]
|
||||
|
||||
|
||||
def _main_video_stream(probe: dict[str, Any]) -> dict[str, Any]:
|
||||
videos = _streams(probe, "video")
|
||||
if not videos:
|
||||
raise BurnError("input contains no video stream")
|
||||
return next(
|
||||
(
|
||||
stream
|
||||
for stream in videos
|
||||
if not bool((stream.get("disposition") or {}).get("attached_pic"))
|
||||
),
|
||||
videos[0],
|
||||
)
|
||||
|
||||
|
||||
def _stream_dimensions(stream: dict[str, Any]) -> tuple[int, int]:
|
||||
try:
|
||||
width = int(stream["width"])
|
||||
height = int(stream["height"])
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise BurnError("video stream has no valid dimensions") from exc
|
||||
if width <= 0 or height <= 0:
|
||||
raise BurnError("video stream has no valid dimensions")
|
||||
return width, height
|
||||
|
||||
|
||||
def _duration(probe: dict[str, Any]) -> float:
|
||||
candidates: list[Any] = []
|
||||
file_format = probe.get("format")
|
||||
if isinstance(file_format, dict):
|
||||
candidates.append(file_format.get("duration"))
|
||||
for stream in probe.get("streams", []):
|
||||
if isinstance(stream, dict):
|
||||
candidates.append(stream.get("duration"))
|
||||
|
||||
durations: list[float] = []
|
||||
for candidate in candidates:
|
||||
try:
|
||||
duration = float(candidate)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if math.isfinite(duration) and duration > 0:
|
||||
durations.append(duration)
|
||||
return max(durations, default=0.0)
|
||||
|
||||
|
||||
def _frame_rate(stream: dict[str, Any]) -> Fraction | None:
|
||||
for key in ("avg_frame_rate", "r_frame_rate"):
|
||||
value = stream.get(key)
|
||||
if not isinstance(value, str):
|
||||
continue
|
||||
try:
|
||||
rate = Fraction(value)
|
||||
except (ValueError, ZeroDivisionError):
|
||||
continue
|
||||
if rate > 0:
|
||||
return rate
|
||||
return None
|
||||
|
||||
|
||||
def _is_hdr(stream: dict[str, Any]) -> bool:
|
||||
if str(stream.get("color_transfer", "")).lower() in HDR_TRANSFERS:
|
||||
return True
|
||||
|
||||
if str(stream.get("color_primaries", "")).lower() == "bt2020":
|
||||
try:
|
||||
bit_depth = int(stream.get("bits_per_raw_sample", 0))
|
||||
except (TypeError, ValueError):
|
||||
bit_depth = 0
|
||||
pixel_format = str(stream.get("pix_fmt", "")).lower()
|
||||
if bit_depth >= 10 or re.search(r"(?:10|12|14|16)(?:le|be)?$", pixel_format):
|
||||
return True
|
||||
|
||||
side_data = stream.get("side_data_list")
|
||||
if isinstance(side_data, list):
|
||||
for item in side_data:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
description = " ".join(str(value).lower() for value in item.values())
|
||||
if any(marker in description for marker in HDR_SIDE_DATA):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _escape_filter_value(value: str) -> str:
|
||||
"""Escape a value through FFmpeg's option and filtergraph parser layers."""
|
||||
|
||||
def escape(text: str, special: str) -> str:
|
||||
return "".join(f"\\{char}" if char in special else char for char in text)
|
||||
|
||||
option_escaped = escape(value, "\\':")
|
||||
return escape(option_escaped, "\\'[],;")
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _citation_text(path: Path) -> str:
|
||||
try:
|
||||
value = path.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
raise BurnError(f"citation file is not readable UTF-8: {path}: {exc}") from exc
|
||||
lines = [" ".join(line.split()) for line in value.splitlines()]
|
||||
while lines and not lines[0]:
|
||||
lines.pop(0)
|
||||
while lines and not lines[-1]:
|
||||
lines.pop()
|
||||
normalized: list[str] = []
|
||||
for line in lines:
|
||||
if line or not normalized or normalized[-1]:
|
||||
normalized.append(line)
|
||||
value = "\n".join(normalized)
|
||||
if not value:
|
||||
raise BurnError("citation file cannot be empty")
|
||||
if len(value) > 2000 or any(ord(character) < 32 and character != "\n" for character in value):
|
||||
raise BurnError("citation file contains unsupported or excessive text")
|
||||
return value
|
||||
|
||||
|
||||
def _ass_time(seconds: float) -> str:
|
||||
centiseconds = max(1, math.ceil(seconds * 100))
|
||||
hours, remainder = divmod(centiseconds, 360000)
|
||||
minutes, remainder = divmod(remainder, 6000)
|
||||
whole_seconds, fraction = divmod(remainder, 100)
|
||||
return f"{hours}:{minutes:02d}:{whole_seconds:02d}.{fraction:02d}"
|
||||
|
||||
|
||||
def _ass_escape(value: str) -> str:
|
||||
"""Losslessly encode untrusted visible text for an ASS Dialogue field."""
|
||||
|
||||
output: list[str] = []
|
||||
for character in value:
|
||||
if character == "\\":
|
||||
output.append("\\" + ASS_WORD_JOINER)
|
||||
elif character == "{":
|
||||
output.append(r"\{{}")
|
||||
elif character == "\n":
|
||||
output.append(r"\N")
|
||||
else:
|
||||
output.append(character)
|
||||
return "".join(output)
|
||||
|
||||
|
||||
def _render_citation_ass(text: str, width: int, height: int, duration: float) -> str:
|
||||
font_size = max(16, min(28, round(height * 0.022)))
|
||||
notice_size = max(14, font_size - 4)
|
||||
margin_x = max(18, round(width * 0.0208))
|
||||
margin_y = max(18, round(height * 0.0278))
|
||||
padding_x = max(12, round(width * 0.0083))
|
||||
padding_y = max(10, round(height * 0.0111))
|
||||
portrait = height > width
|
||||
panel_width = min(
|
||||
width - 2 * margin_x,
|
||||
max(
|
||||
round(width * (0.90 if portrait else 0.375)),
|
||||
round(24 * font_size * 0.41) + 2 * padding_x,
|
||||
),
|
||||
)
|
||||
max_columns = max(
|
||||
24,
|
||||
round((panel_width - 2 * padding_x) / (font_size * 0.41)),
|
||||
)
|
||||
raw_lines = text.splitlines()
|
||||
notice: str | None = None
|
||||
if len(raw_lines) >= 3 and raw_lines[1] == "":
|
||||
notice = raw_lines[0]
|
||||
raw_lines = raw_lines[2:]
|
||||
wrapped_lines: list[str] = []
|
||||
for paragraph in raw_lines:
|
||||
wrapped_lines.extend(
|
||||
textwrap.wrap(
|
||||
paragraph,
|
||||
width=max_columns,
|
||||
break_long_words=False,
|
||||
break_on_hyphens=False,
|
||||
) or [""]
|
||||
)
|
||||
citation_display = _ass_escape("\n".join(wrapped_lines))
|
||||
if notice is not None:
|
||||
display = (
|
||||
rf"{{\fs{notice_size}\1c&HCCCCCC&}}{_ass_escape(notice)}"
|
||||
rf"\N{{\fs8}} \N{{\fs{font_size}\1c&HFFFFFF&}}{citation_display}"
|
||||
)
|
||||
else:
|
||||
display = rf"{{\fs{font_size}\1c&HFFFFFF&}}{citation_display}"
|
||||
line_height = font_size * 1.05
|
||||
panel_height = math.ceil(
|
||||
2 * padding_y
|
||||
+ len(wrapped_lines) * line_height
|
||||
+ (notice_size * 1.05 + 8 if notice is not None else 0)
|
||||
+ 3
|
||||
)
|
||||
text_x = margin_x + padding_x
|
||||
text_y = margin_y + padding_y
|
||||
panel = (
|
||||
rf"{{\an7\pos({margin_x},{margin_y})\p1\1c&H000000&\1a&H78&}}"
|
||||
f"m 0 0 l {panel_width} 0 {panel_width} {panel_height} 0 {panel_height}"
|
||||
)
|
||||
return (
|
||||
"[Script Info]\n"
|
||||
"ScriptType: v4.00+\n"
|
||||
"WrapStyle: 2\n"
|
||||
"ScaledBorderAndShadow: yes\n"
|
||||
f"PlayResX: {width}\nPlayResY: {height}\n\n"
|
||||
"[V4+ Styles]\n"
|
||||
"Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, "
|
||||
"OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, "
|
||||
"ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, "
|
||||
"Alignment, MarginL, MarginR, MarginV, Encoding\n"
|
||||
f"Style: Citation,Arial,{font_size},&H10FFFFFF,&H10FFFFFF,&H00000000,"
|
||||
"&H00000000,0,0,0,0,100,100,0,0,1,0,0,7,0,0,0,1\n\n"
|
||||
"[Events]\n"
|
||||
"Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n"
|
||||
f"Dialogue: 9,0:00:00.00,{_ass_time(duration)},Citation,,0,0,0,,{panel}\n"
|
||||
f"Dialogue: 10,0:00:00.00,{_ass_time(duration)},Citation,,0,0,0,,"
|
||||
rf"{{\an7\pos({text_x},{text_y})}}{display}\n"
|
||||
)
|
||||
|
||||
|
||||
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
|
||||
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
json.dump(value, handle, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
Path(temporary).unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def _write_citation_receipt(output: Path, citation_file: Path) -> Path:
|
||||
receipt = output.with_suffix(output.suffix + ".citation.json")
|
||||
_atomic_json(
|
||||
receipt,
|
||||
{
|
||||
"schema_version": 1,
|
||||
"output_file": output.name,
|
||||
"output_sha256": _sha256_file(output),
|
||||
"citation_sha256": _sha256_file(citation_file),
|
||||
"position": "top-left",
|
||||
},
|
||||
)
|
||||
return receipt
|
||||
|
||||
|
||||
def _validate_validation_report(subtitle: Path, report_path: Path) -> dict[str, Any]:
|
||||
subtitle = subtitle.expanduser().resolve()
|
||||
report_path = report_path.expanduser().resolve()
|
||||
if not report_path.is_file():
|
||||
raise BurnError(f"validation report does not exist or is not a file: {report_path}")
|
||||
|
||||
try:
|
||||
report = json.loads(report_path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise BurnError(f"validation report is not valid UTF-8 JSON: {report_path}: {exc}") from exc
|
||||
if not isinstance(report, dict):
|
||||
raise BurnError("validation report root must be a JSON object")
|
||||
if report.get("structurally_valid") is not True:
|
||||
raise BurnError("validation report must declare structurally_valid=true")
|
||||
if report.get("validation_scope") != "structural_source_integrity":
|
||||
raise BurnError(
|
||||
"validation report scope must be structural_source_integrity"
|
||||
)
|
||||
|
||||
segment_count = report.get("segment_count")
|
||||
translation_count = report.get("translation_count")
|
||||
counts = (segment_count, translation_count)
|
||||
if any(isinstance(value, bool) or not isinstance(value, int) or value <= 0 for value in counts):
|
||||
raise BurnError(
|
||||
"validation report segment_count and translation_count must be positive integers"
|
||||
)
|
||||
if segment_count != translation_count:
|
||||
raise BurnError(
|
||||
"validation report segment_count and translation_count must be equal"
|
||||
)
|
||||
|
||||
outputs = report.get("outputs")
|
||||
recorded_hash = outputs.get("bilingual.ass") if isinstance(outputs, dict) else None
|
||||
if not isinstance(recorded_hash, str) or not re.fullmatch(
|
||||
r"[0-9a-fA-F]{64}", recorded_hash
|
||||
):
|
||||
raise BurnError(
|
||||
"validation report outputs['bilingual.ass'] must be a SHA-256 checksum"
|
||||
)
|
||||
if _sha256_file(subtitle) != recorded_hash.lower():
|
||||
raise BurnError("bilingual.ass SHA-256 does not match the validation report")
|
||||
return report
|
||||
|
||||
|
||||
_FONT_FILE_SUFFIXES = frozenset({".ttf", ".otf", ".ttc"})
|
||||
_FONT_DIRECTORIES = (
|
||||
"~/Library/Fonts",
|
||||
"/Library/Fonts",
|
||||
"/System/Library/Fonts",
|
||||
"~/.fonts",
|
||||
"~/.local/share/fonts",
|
||||
"/usr/share/fonts",
|
||||
"/usr/local/share/fonts",
|
||||
)
|
||||
|
||||
|
||||
def _font_installed(family: str) -> bool | None:
|
||||
"""Return True/False when detection is trustworthy, None when unavailable.
|
||||
|
||||
libass silently substitutes another font when the requested family is
|
||||
missing, which would pass every later gate with the wrong deliverable.
|
||||
"""
|
||||
|
||||
fc_list = shutil.which("fc-list")
|
||||
if fc_list:
|
||||
result = subprocess.run(
|
||||
[fc_list, ":", "family"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
needle = family.casefold()
|
||||
return any(
|
||||
needle in entry.strip().casefold()
|
||||
for line in result.stdout.splitlines()
|
||||
for entry in line.split(",")
|
||||
)
|
||||
token = re.sub(r"[\s_-]+", "", family).casefold()
|
||||
if not token:
|
||||
return None
|
||||
searched = False
|
||||
for directory in _FONT_DIRECTORIES:
|
||||
base = Path(directory).expanduser()
|
||||
if not base.is_dir():
|
||||
continue
|
||||
searched = True
|
||||
for path in base.rglob("*"):
|
||||
if (
|
||||
path.suffix.lower() in _FONT_FILE_SUFFIXES
|
||||
and token in re.sub(r"[\s_-]+", "", path.stem).casefold()
|
||||
):
|
||||
return True
|
||||
return False if searched else None
|
||||
|
||||
|
||||
def _require_subtitle_font(report: dict[str, Any], *, allow_missing_font: bool) -> None:
|
||||
font = str(report.get("font") or "").strip()
|
||||
if not font:
|
||||
return
|
||||
installed = _font_installed(font)
|
||||
if installed is True:
|
||||
return
|
||||
if installed is None:
|
||||
print(
|
||||
f"warning: could not verify that font {font!r} is installed; "
|
||||
"libass substitutes missing fonts silently",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return
|
||||
message = (
|
||||
f"font {font!r} required by the validated subtitles was not found; install it "
|
||||
"(MiSans: https://hyperos.mi.com/font/zh/download/)"
|
||||
)
|
||||
if allow_missing_font:
|
||||
print(f"warning: {message}; continuing with libass substitution", file=sys.stderr)
|
||||
return
|
||||
raise BurnError(f"{message} or pass --allow-missing-font to accept substitution")
|
||||
|
||||
|
||||
def _audio_options(audio_streams: Sequence[dict[str, Any]]) -> tuple[list[str], list[str]]:
|
||||
if not audio_streams:
|
||||
return [], []
|
||||
|
||||
options = ["-c:a", "copy"]
|
||||
modes: list[str] = []
|
||||
for output_index, stream in enumerate(audio_streams):
|
||||
codec = str(stream.get("codec_name", "")).lower()
|
||||
if codec in MP4_COPY_AUDIO_CODECS:
|
||||
modes.append(f"audio {output_index}: copied {codec}")
|
||||
continue
|
||||
options.extend(
|
||||
[
|
||||
f"-c:a:{output_index}",
|
||||
"aac",
|
||||
f"-b:a:{output_index}",
|
||||
"256k",
|
||||
]
|
||||
)
|
||||
modes.append(f"audio {output_index}: {codec or 'unknown'} -> AAC")
|
||||
return options, modes
|
||||
|
||||
|
||||
def _encode_command(
|
||||
ffmpeg: str,
|
||||
video: Path,
|
||||
subtitle: Path,
|
||||
output: Path,
|
||||
video_stream: dict[str, Any],
|
||||
audio_streams: Sequence[dict[str, Any]],
|
||||
*,
|
||||
force: bool,
|
||||
crf: int,
|
||||
preset: str,
|
||||
encoder: str,
|
||||
citation_ass: Path | None = None,
|
||||
) -> tuple[list[str], list[str]]:
|
||||
try:
|
||||
stream_index = int(video_stream["index"])
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise BurnError("input video stream has no valid index") from exc
|
||||
|
||||
audio_options, audio_modes = _audio_options(audio_streams)
|
||||
subtitle_filters = [f"subtitles=filename={_escape_filter_value(str(subtitle))}"]
|
||||
if citation_ass is not None:
|
||||
subtitle_filters.append(
|
||||
f"subtitles=filename={_escape_filter_value(str(citation_ass))}"
|
||||
)
|
||||
command = [
|
||||
ffmpeg,
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostats",
|
||||
"-stats_period",
|
||||
"1",
|
||||
"-progress",
|
||||
"pipe:1",
|
||||
"-y" if force else "-n",
|
||||
"-i",
|
||||
str(video),
|
||||
"-map",
|
||||
f"0:{stream_index}",
|
||||
"-map",
|
||||
"0:a?",
|
||||
"-map_metadata",
|
||||
"0",
|
||||
"-map_chapters",
|
||||
"0",
|
||||
"-sn",
|
||||
"-dn",
|
||||
"-vf",
|
||||
",".join(subtitle_filters),
|
||||
"-fps_mode:v:0",
|
||||
"passthrough",
|
||||
"-c:v",
|
||||
encoder,
|
||||
"-crf",
|
||||
str(crf),
|
||||
"-preset",
|
||||
preset,
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
*audio_options,
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
"-f",
|
||||
"mp4",
|
||||
str(output),
|
||||
]
|
||||
return command, audio_modes
|
||||
|
||||
|
||||
def _verify_output(
|
||||
ffprobe: str,
|
||||
output: Path,
|
||||
input_video: dict[str, Any],
|
||||
input_had_audio: bool,
|
||||
*,
|
||||
input_duration: float,
|
||||
) -> None:
|
||||
result = _probe(ffprobe, output)
|
||||
file_format = result.get("format")
|
||||
format_name = file_format.get("format_name", "") if isinstance(file_format, dict) else ""
|
||||
if "mp4" not in str(format_name).split(","):
|
||||
raise BurnError(f"output verification failed: container is not MP4 ({format_name or 'unknown'})")
|
||||
|
||||
output_videos = _streams(result, "video")
|
||||
if not output_videos:
|
||||
raise BurnError("output verification failed: no video stream")
|
||||
output_video = output_videos[0]
|
||||
if output_video.get("codec_name") != "h264":
|
||||
raise BurnError(
|
||||
"output verification failed: video codec is "
|
||||
f"{output_video.get('codec_name', 'unknown')}, not H.264"
|
||||
)
|
||||
output_duration = _duration(result)
|
||||
if output_duration <= 0:
|
||||
raise BurnError("output verification failed: duration is zero or unavailable")
|
||||
if input_duration <= 0:
|
||||
raise BurnError("output verification failed: input duration is zero or unavailable")
|
||||
duration_tolerance = max(0.5, input_duration * 0.01)
|
||||
if abs(output_duration - input_duration) > duration_tolerance:
|
||||
raise BurnError(
|
||||
"output verification failed: duration changed from "
|
||||
f"{input_duration:.3f}s to {output_duration:.3f}s "
|
||||
f"(allowed difference {duration_tolerance:.3f}s)"
|
||||
)
|
||||
|
||||
input_dimensions = _stream_dimensions(input_video)
|
||||
output_dimensions = _stream_dimensions(output_video)
|
||||
if output_dimensions != input_dimensions:
|
||||
raise BurnError(
|
||||
"output verification failed: dimensions changed from "
|
||||
f"{input_dimensions[0]}x{input_dimensions[1]} to "
|
||||
f"{output_dimensions[0]}x{output_dimensions[1]}"
|
||||
)
|
||||
|
||||
input_rate = _frame_rate(input_video)
|
||||
output_rate = _frame_rate(output_video)
|
||||
if input_rate is not None and output_rate is not None:
|
||||
relative_drift = abs(float(output_rate - input_rate)) / float(input_rate)
|
||||
if relative_drift > 0.005:
|
||||
raise BurnError(
|
||||
"output verification failed: frame rate changed from "
|
||||
f"{float(input_rate):.6g} to {float(output_rate):.6g} fps"
|
||||
)
|
||||
|
||||
if input_had_audio and not _streams(result, "audio"):
|
||||
raise BurnError("output verification failed: input audio is missing from output")
|
||||
|
||||
|
||||
def burn_subtitles(
|
||||
video: Path,
|
||||
subtitle: Path,
|
||||
output: Path,
|
||||
*,
|
||||
force: bool = False,
|
||||
crf: int = 18,
|
||||
preset: str = "slow",
|
||||
encoder: str = DEFAULT_ENCODER,
|
||||
validation_report: Path | None = None,
|
||||
allow_missing_font: bool = False,
|
||||
citation_file: Path | None = None,
|
||||
) -> list[str]:
|
||||
video = video.expanduser().resolve()
|
||||
subtitle = subtitle.expanduser().resolve()
|
||||
output = output.expanduser().resolve()
|
||||
report_path = (
|
||||
validation_report.expanduser().resolve()
|
||||
if validation_report is not None
|
||||
else subtitle.with_name("validation.json")
|
||||
)
|
||||
citation_path = citation_file.expanduser().resolve() if citation_file is not None else None
|
||||
|
||||
if not video.is_file():
|
||||
raise BurnError(f"input video does not exist or is not a file: {video}")
|
||||
if not subtitle.is_file():
|
||||
raise BurnError(f"ASS subtitle does not exist or is not a file: {subtitle}")
|
||||
if subtitle.suffix.lower() != ".ass":
|
||||
raise BurnError(f"subtitle must be an .ass file: {subtitle}")
|
||||
if output in (video, subtitle, report_path):
|
||||
raise BurnError("output must be different from all input files")
|
||||
if citation_path is not None and not citation_path.is_file():
|
||||
raise BurnError(f"citation file does not exist or is not a file: {citation_path}")
|
||||
if citation_path is not None and output == citation_path:
|
||||
raise BurnError("output must be different from the citation file")
|
||||
if not output.parent.is_dir():
|
||||
raise BurnError(f"output directory does not exist: {output.parent}")
|
||||
if output.exists() and not force:
|
||||
raise BurnError(f"output already exists (use --force to replace it): {output}")
|
||||
if output.exists() and not output.is_file():
|
||||
raise BurnError(f"output exists and is not a regular file: {output}")
|
||||
if not preset.strip():
|
||||
raise BurnError("encoder preset cannot be empty")
|
||||
if not encoder.strip():
|
||||
raise BurnError("encoder cannot be empty")
|
||||
|
||||
report = _validate_validation_report(subtitle, report_path)
|
||||
_require_subtitle_font(report, allow_missing_font=allow_missing_font)
|
||||
|
||||
ffmpeg, ffprobe = _required_executables()
|
||||
_require_libass_subtitles_filter(ffmpeg)
|
||||
|
||||
input_probe = _probe(ffprobe, video)
|
||||
input_video = _main_video_stream(input_probe)
|
||||
width, height = _stream_dimensions(input_video)
|
||||
input_duration = _duration(input_probe)
|
||||
if input_duration <= 0:
|
||||
raise BurnError("input duration is zero or unavailable")
|
||||
audio_streams = _streams(input_probe, "audio")
|
||||
|
||||
if _is_hdr(input_video):
|
||||
print(
|
||||
"warning: HDR input detected. The compatibility H.264/yuv420p output is "
|
||||
"intended for SDR playback; HDR metadata and appearance may not be preserved.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
citation_ass: Path | None = None
|
||||
temporary_dir: tempfile.TemporaryDirectory[str] | None = None
|
||||
if citation_path is not None:
|
||||
temporary_dir = tempfile.TemporaryDirectory(prefix="materialsub-citation-")
|
||||
citation_ass = Path(temporary_dir.name) / "citation.ass"
|
||||
citation_ass.write_text(
|
||||
_render_citation_ass(_citation_text(citation_path), width, height, input_duration),
|
||||
encoding="utf-8",
|
||||
)
|
||||
try:
|
||||
command, audio_modes = _encode_command(
|
||||
ffmpeg,
|
||||
video,
|
||||
subtitle,
|
||||
output,
|
||||
input_video,
|
||||
audio_streams,
|
||||
force=force,
|
||||
crf=crf,
|
||||
preset=preset,
|
||||
encoder=encoder,
|
||||
citation_ass=citation_ass,
|
||||
)
|
||||
returncode, diagnostic = _run_ffmpeg_with_progress(command, input_duration)
|
||||
finally:
|
||||
if temporary_dir is not None:
|
||||
temporary_dir.cleanup()
|
||||
if returncode != 0:
|
||||
if output.is_file():
|
||||
output.unlink()
|
||||
detail = _last_error_line(diagnostic)
|
||||
raise BurnError(f"FFmpeg subtitle burn failed with exit code {returncode}{detail}")
|
||||
|
||||
try:
|
||||
_verify_output(
|
||||
ffprobe,
|
||||
output,
|
||||
input_video,
|
||||
bool(audio_streams),
|
||||
input_duration=input_duration,
|
||||
)
|
||||
except BurnError:
|
||||
if output.is_file():
|
||||
output.unlink()
|
||||
raise
|
||||
receipt = output.with_suffix(output.suffix + ".citation.json")
|
||||
if citation_path is not None:
|
||||
_write_citation_receipt(output, citation_path)
|
||||
elif receipt.is_file():
|
||||
receipt.unlink()
|
||||
return audio_modes
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
try:
|
||||
audio_modes = burn_subtitles(
|
||||
args.video,
|
||||
args.subtitle,
|
||||
args.output,
|
||||
force=args.force,
|
||||
crf=args.crf,
|
||||
preset=args.preset,
|
||||
encoder=args.encoder,
|
||||
validation_report=args.validation_report,
|
||||
allow_missing_font=args.allow_missing_font,
|
||||
citation_file=args.citation_file,
|
||||
)
|
||||
except (BurnError, OSError) as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(
|
||||
"validated ASS was burned into verified MP4: "
|
||||
f"{args.output.expanduser().resolve()}"
|
||||
)
|
||||
for mode in audio_modes:
|
||||
print(mode)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+196
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Attach a user-approved citation watermark specification to a MaterialSub job."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
from typing import Any, Sequence
|
||||
from urllib.parse import parse_qsl, urlsplit
|
||||
|
||||
|
||||
MAX_CITATION_CHARACTERS = 1200
|
||||
DEFAULT_INTERNAL_NOTICE = "内容引自网络,仅供内部交流"
|
||||
SENSITIVE_QUERY_KEYS = re.compile(
|
||||
r"(?:^|_)(?:auth|credential|expires?|key|policy|signature|signed|token)(?:_|$)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
class CitationError(RuntimeError):
|
||||
"""A malformed citation request or job manifest."""
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise CitationError(f"could not read JSON manifest {path}: {exc}") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise CitationError("download manifest root must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _normalize_citation(value: str) -> str:
|
||||
lines = [" ".join(line.split()) for line in value.splitlines()]
|
||||
value = "\n".join(line for line in lines if line)
|
||||
if not value:
|
||||
raise CitationError("citation cannot be empty")
|
||||
if len(value) > MAX_CITATION_CHARACTERS:
|
||||
raise CitationError(
|
||||
f"citation is too long ({len(value)} characters; maximum {MAX_CITATION_CHARACTERS})"
|
||||
)
|
||||
if any(ord(character) < 32 and character != "\n" for character in value):
|
||||
raise CitationError("citation contains unsupported control characters")
|
||||
return value
|
||||
|
||||
|
||||
def _normalize_notice(value: str) -> str:
|
||||
value = " ".join(value.split())
|
||||
if not value:
|
||||
raise CitationError("notice cannot be empty")
|
||||
if len(value) > 120 or any(ord(character) < 32 for character in value):
|
||||
raise CitationError("notice contains unsupported or excessive text")
|
||||
return value
|
||||
|
||||
|
||||
def _public_source_url(manifest: dict[str, Any]) -> str:
|
||||
source = manifest.get("source")
|
||||
value = source.get("url") if isinstance(source, dict) else None
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise CitationError("download manifest has no canonical source URL")
|
||||
parts = urlsplit(value.strip())
|
||||
if parts.scheme not in ("http", "https") or not parts.netloc or parts.username or parts.password:
|
||||
raise CitationError("manifest source URL is not a public HTTP(S) URL")
|
||||
for key, _ in parse_qsl(parts.query, keep_blank_values=True):
|
||||
if SENSITIVE_QUERY_KEYS.search(key):
|
||||
raise CitationError("refusing to put a signed or credential-like URL in a watermark")
|
||||
return value.strip()
|
||||
|
||||
|
||||
def _atomic_write_text(path: Path, value: str) -> None:
|
||||
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
handle.write(value)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
Path(temporary).unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def _atomic_write_json(path: Path, value: dict[str, Any]) -> None:
|
||||
_atomic_write_text(
|
||||
path,
|
||||
json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
)
|
||||
|
||||
|
||||
def attach_citation(
|
||||
manifest_path: Path,
|
||||
citation: str,
|
||||
*,
|
||||
include_source_url: bool = False,
|
||||
notice: str | None = None,
|
||||
) -> Path:
|
||||
manifest_path = manifest_path.expanduser().resolve()
|
||||
manifest = _read_json(manifest_path)
|
||||
citation = _normalize_citation(citation)
|
||||
notice = _normalize_notice(notice) if notice is not None else None
|
||||
source_url = _public_source_url(manifest) if include_source_url else None
|
||||
citation_body = citation + (f"\nSource: {source_url}" if source_url else "")
|
||||
rendered_text = (f"{notice}\n\n" if notice else "") + citation_body + "\n"
|
||||
citation_path = manifest_path.parent / "citation-watermark.txt"
|
||||
_atomic_write_text(citation_path, rendered_text)
|
||||
digest = hashlib.sha256(citation_path.read_bytes()).hexdigest()
|
||||
manifest["citation_watermark"] = {
|
||||
"enabled": True,
|
||||
"citation_file": citation_path.name,
|
||||
"citation_sha256": digest,
|
||||
"include_source_url": include_source_url,
|
||||
"layout": (
|
||||
"notice-plus-three-line-citation"
|
||||
if notice and len(citation.splitlines()) == 3 and not include_source_url
|
||||
else "notice-plus-citation" if notice else "citation-only"
|
||||
),
|
||||
"notice": notice,
|
||||
"position": "top-left",
|
||||
}
|
||||
_atomic_write_json(manifest_path, manifest)
|
||||
return citation_path
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Bind a user-approved top-left citation watermark to a MaterialSub job."
|
||||
)
|
||||
parser.add_argument("manifest", type=Path, help="download-manifest.json")
|
||||
citation = parser.add_mutually_exclusive_group()
|
||||
citation.add_argument("--citation", help="complete formal citation")
|
||||
citation.add_argument("--citation-file", type=Path, help="UTF-8 file containing the citation")
|
||||
parser.add_argument("--authors", help="citation authors line")
|
||||
parser.add_argument("--title", help="citation title line")
|
||||
parser.add_argument("--publication", help="journal, identifiers, and year line")
|
||||
parser.add_argument(
|
||||
"--notice",
|
||||
nargs="?",
|
||||
const=DEFAULT_INTERNAL_NOTICE,
|
||||
help=(
|
||||
"add an approved notice above the citation; without TEXT uses "
|
||||
f"{DEFAULT_INTERNAL_NOTICE!r}"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--include-source-url",
|
||||
action="store_true",
|
||||
help="also show the manifest's canonical public source URL",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
try:
|
||||
structured = (args.authors, args.title, args.publication)
|
||||
has_structured = any(value is not None for value in structured)
|
||||
if has_structured:
|
||||
if not all(value is not None for value in structured):
|
||||
raise CitationError("--authors, --title, and --publication must be used together")
|
||||
if args.citation is not None or args.citation_file is not None:
|
||||
raise CitationError("structured citation fields cannot be combined with --citation")
|
||||
citation = "\n".join(str(value) for value in structured)
|
||||
elif args.citation_file is not None:
|
||||
citation = args.citation_file.read_text(encoding="utf-8")
|
||||
elif args.citation is not None:
|
||||
citation = args.citation
|
||||
else:
|
||||
raise CitationError(
|
||||
"provide --citation, --citation-file, or all three structured citation fields"
|
||||
)
|
||||
path = attach_citation(
|
||||
args.manifest,
|
||||
citation,
|
||||
include_source_url=args.include_source_url,
|
||||
notice=args.notice,
|
||||
)
|
||||
except (CitationError, OSError, UnicodeDecodeError) as exc:
|
||||
print(f"citation watermark error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
print(path)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+2041
File diff suppressed because it is too large
Load Diff
+527
@@ -0,0 +1,527 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Ingest one browser-confirmed, authorized embedded HLS video.
|
||||
|
||||
The browser-facing agent writes a private resource map after it has verified
|
||||
that playback is authorized. This script never receives signed URLs on the
|
||||
command line, never prints them, and propagates a playlist's authorization
|
||||
query only to same-origin HLS resources. It rejects master playlists and DRM
|
||||
methods so the ordinary yt-dlp route remains the preferred path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any, Sequence
|
||||
|
||||
import fetch_video as fetch
|
||||
|
||||
|
||||
RESOURCE_MAP_SCHEMA = 1
|
||||
ALLOWED_SUBTITLE_KINDS = {"manual", "automatic"}
|
||||
_URI_ATTRIBUTE = re.compile(r'URI="([^"]+)"')
|
||||
_EXTENSION = re.compile(r"^[A-Za-z0-9]{1,8}$")
|
||||
|
||||
|
||||
class IngestError(fetch.FetchError):
|
||||
"""An embedded-resource ingest failed safely."""
|
||||
|
||||
|
||||
def _read_private_resource_map(path: Path) -> dict[str, Any]:
|
||||
path = path.expanduser().resolve()
|
||||
if not path.is_file():
|
||||
raise IngestError(f"Resource map does not exist or is not a file: {path}")
|
||||
if path.stat().st_mode & 0o077:
|
||||
raise IngestError("Resource map must be private (chmod 600) because it may contain signed URLs")
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
raise IngestError("Could not read the private resource map") from exc
|
||||
if not isinstance(value, dict) or value.get("schema_version") != RESOURCE_MAP_SCHEMA:
|
||||
raise IngestError(f"Resource map must be a schema_version={RESOURCE_MAP_SCHEMA} object")
|
||||
return value
|
||||
|
||||
|
||||
def _prepare_output_dir(
|
||||
output_dir: Path,
|
||||
resource_map_path: Path,
|
||||
*,
|
||||
resume: bool,
|
||||
cleanup_resource_map: bool,
|
||||
) -> Path:
|
||||
if output_dir.exists() and not output_dir.is_dir():
|
||||
raise IngestError(f"Output path is not a directory: {output_dir}")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
if resume:
|
||||
return output_dir
|
||||
entries = list(output_dir.iterdir())
|
||||
allowed_map = (
|
||||
cleanup_resource_map
|
||||
and resource_map_path.parent == output_dir
|
||||
and resource_map_path in entries
|
||||
)
|
||||
unexpected = [path for path in entries if not (allowed_map and path == resource_map_path)]
|
||||
if unexpected or (entries and not allowed_map):
|
||||
raise IngestError(
|
||||
f"Output directory is not empty: {output_dir}. Choose a new directory or pass --resume explicitly."
|
||||
)
|
||||
return output_dir
|
||||
|
||||
|
||||
def _required_text(value: Any, field: str) -> str:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise IngestError(f"Resource map field {field!r} must be a non-empty string")
|
||||
return value.strip()
|
||||
|
||||
|
||||
def _optional_number(value: Any, field: str) -> float | int | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
|
||||
raise IngestError(f"Resource map field {field!r} must be a positive number")
|
||||
return value
|
||||
|
||||
|
||||
def _validated_url(value: Any, field: str) -> str:
|
||||
try:
|
||||
return fetch.validate_url(_required_text(value, field))
|
||||
except fetch.FetchError as exc:
|
||||
raise IngestError(f"Invalid resource map URL field {field!r}") from exc
|
||||
|
||||
|
||||
def _same_origin(left: str, right: str) -> bool:
|
||||
a = urllib.parse.urlsplit(left)
|
||||
b = urllib.parse.urlsplit(right)
|
||||
return (a.scheme.lower(), a.hostname, a.port) == (b.scheme.lower(), b.hostname, b.port)
|
||||
|
||||
|
||||
def _authorized_resource_url(resource_url: str, playlist_url: str) -> str:
|
||||
"""Add missing root-playlist query fields to same-origin resources only."""
|
||||
absolute = urllib.parse.urljoin(playlist_url, resource_url)
|
||||
if not _same_origin(absolute, playlist_url):
|
||||
return absolute
|
||||
root = urllib.parse.urlsplit(playlist_url)
|
||||
target = urllib.parse.urlsplit(absolute)
|
||||
root_query = urllib.parse.parse_qsl(root.query, keep_blank_values=True)
|
||||
target_query = urllib.parse.parse_qsl(target.query, keep_blank_values=True)
|
||||
existing = {key for key, _ in target_query}
|
||||
merged = target_query + [(key, value) for key, value in root_query if key not in existing]
|
||||
return urllib.parse.urlunsplit(
|
||||
(target.scheme, target.netloc, target.path, urllib.parse.urlencode(merged), "")
|
||||
)
|
||||
|
||||
|
||||
def _validate_hls_protection(playlist: str) -> None:
|
||||
for line in playlist.splitlines():
|
||||
if not line.startswith("#EXT-X-KEY:"):
|
||||
continue
|
||||
attributes = line.split(":", 1)[1]
|
||||
method = re.search(r"(?:^|,)METHOD=([^,]+)", attributes)
|
||||
method_value = method.group(1).strip().upper() if method else ""
|
||||
keyformat = re.search(r'(?:^|,)KEYFORMAT="?([^,"]+)', attributes)
|
||||
keyformat_value = keyformat.group(1).strip().lower() if keyformat else "identity"
|
||||
if method_value not in {"NONE", "AES-128"} or keyformat_value != "identity":
|
||||
raise IngestError(
|
||||
"Embedded HLS uses an unsupported protection method; MaterialSub will not bypass DRM"
|
||||
)
|
||||
|
||||
|
||||
def rewrite_media_playlist(playlist: str, playlist_url: str) -> str:
|
||||
"""Return a signed, absolute media playlist without logging its secrets."""
|
||||
if not playlist.lstrip().startswith("#EXTM3U"):
|
||||
raise IngestError("The selected resource is not an HLS playlist")
|
||||
if "#EXT-X-STREAM-INF" in playlist or "#EXT-X-MEDIA:" in playlist:
|
||||
raise IngestError(
|
||||
"The resource is an HLS master playlist; use the browser-observed media playlist selected during playback"
|
||||
)
|
||||
_validate_hls_protection(playlist)
|
||||
rewritten: list[str] = []
|
||||
for line in playlist.splitlines():
|
||||
if line.startswith("#"):
|
||||
line = _URI_ATTRIBUTE.sub(
|
||||
lambda match: f'URI="{_authorized_resource_url(match.group(1), playlist_url)}"',
|
||||
line,
|
||||
)
|
||||
elif line.strip():
|
||||
line = _authorized_resource_url(line.strip(), playlist_url)
|
||||
rewritten.append(line)
|
||||
return "\n".join(rewritten) + "\n"
|
||||
|
||||
|
||||
def _request_bytes(url: str, *, playlist_url: str | None = None) -> bytes:
|
||||
resolved = _authorized_resource_url(url, playlist_url) if playlist_url else url
|
||||
request = urllib.request.Request(resolved, headers={"User-Agent": "Mozilla/5.0 MaterialSub/1"})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=45) as response:
|
||||
return response.read()
|
||||
except Exception as exc:
|
||||
raise IngestError(f"Failed to download an authorized resource from {fetch.display_url(url)}") from exc
|
||||
|
||||
|
||||
def _atomic_download(url: str, destination: Path, *, playlist_url: str | None = None) -> Path:
|
||||
if destination.exists():
|
||||
raise IngestError(f"Refusing to replace existing artifact without --resume: {destination}")
|
||||
payload = _request_bytes(url, playlist_url=playlist_url)
|
||||
if not payload:
|
||||
raise IngestError(f"Downloaded resource is empty: {fetch.display_url(url)}")
|
||||
fd, temporary_name = tempfile.mkstemp(
|
||||
prefix=f".{destination.stem}.", suffix=destination.suffix, dir=destination.parent
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as handle:
|
||||
handle.write(payload)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary_name, destination)
|
||||
finally:
|
||||
Path(temporary_name).unlink(missing_ok=True)
|
||||
return destination
|
||||
|
||||
|
||||
def _download_cover(
|
||||
url: str | None,
|
||||
destination: Path,
|
||||
*,
|
||||
playlist_url: str,
|
||||
ffmpeg: str,
|
||||
resume: bool,
|
||||
) -> Path | None:
|
||||
if not url:
|
||||
return None
|
||||
if destination.exists() and resume:
|
||||
return destination
|
||||
with tempfile.TemporaryDirectory(prefix="materialsub-cover-") as directory:
|
||||
raw = Path(directory) / "cover.input"
|
||||
raw.write_bytes(_request_bytes(url, playlist_url=playlist_url))
|
||||
if not raw.stat().st_size:
|
||||
raise IngestError("Downloaded cover is empty")
|
||||
temporary = Path(directory) / "cover.jpg"
|
||||
result = fetch._run(
|
||||
[
|
||||
ffmpeg,
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-i",
|
||||
str(raw),
|
||||
"-frames:v",
|
||||
"1",
|
||||
str(temporary),
|
||||
],
|
||||
"converting the embedded cover to JPEG",
|
||||
secrets=(url, str(raw), str(temporary)),
|
||||
check=False,
|
||||
)
|
||||
if result.returncode or not temporary.is_file() or not temporary.stat().st_size:
|
||||
raise IngestError("Failed to convert the embedded cover to JPEG")
|
||||
os.replace(temporary, destination)
|
||||
return destination
|
||||
|
||||
|
||||
def _subtitle_choice(resource_map: dict[str, Any], target_language: str) -> tuple[fetch.SubtitleChoice | None, dict[str, Any] | None]:
|
||||
subtitle = resource_map.get("subtitle")
|
||||
if subtitle is None:
|
||||
return None, None
|
||||
if not isinstance(subtitle, dict):
|
||||
raise IngestError("Resource map subtitle must be an object or null")
|
||||
url = _validated_url(subtitle.get("url"), "subtitle.url")
|
||||
language = _required_text(subtitle.get("language"), "subtitle.language")
|
||||
kind = _required_text(subtitle.get("kind"), "subtitle.kind").lower()
|
||||
if kind not in ALLOWED_SUBTITLE_KINDS:
|
||||
raise IngestError("subtitle.kind must be 'manual' or 'automatic'")
|
||||
original_format = str(subtitle.get("format") or Path(urllib.parse.urlsplit(url).path).suffix.lstrip(".") or "vtt").lower()
|
||||
if not _EXTENSION.fullmatch(original_format):
|
||||
raise IngestError("subtitle.format must be a short filename extension")
|
||||
if fetch._excluded_language(language, fetch._target_language_bases(target_language)):
|
||||
raise IngestError("Selected subtitle is not a foreign-language dialogue track for the requested target language")
|
||||
return fetch.SubtitleChoice(language, kind, original_format, (original_format,)), subtitle
|
||||
|
||||
|
||||
def _download_hls(
|
||||
playlist_url: str,
|
||||
output_dir: Path,
|
||||
base: str,
|
||||
*,
|
||||
yt_dlp: str,
|
||||
concurrent_fragments: int,
|
||||
) -> tuple[Path, Any]:
|
||||
source = _request_bytes(playlist_url).decode("utf-8-sig")
|
||||
rewritten = rewrite_media_playlist(source, playlist_url)
|
||||
with tempfile.TemporaryDirectory(prefix="materialsub-authorized-hls-") as directory:
|
||||
playlist_path = Path(directory) / "media.m3u8"
|
||||
playlist_path.write_text(rewritten, encoding="utf-8")
|
||||
os.chmod(playlist_path, 0o600)
|
||||
result = fetch._run(
|
||||
[
|
||||
yt_dlp,
|
||||
"--ignore-config",
|
||||
"--no-playlist",
|
||||
"--no-write-playlist-metafiles",
|
||||
"--no-progress",
|
||||
"--enable-file-urls",
|
||||
"--concurrent-fragments",
|
||||
str(concurrent_fragments),
|
||||
"-P",
|
||||
str(output_dir),
|
||||
"-f",
|
||||
fetch.FORMAT_SELECTOR,
|
||||
"--merge-output-format",
|
||||
"mkv",
|
||||
"--remux-video",
|
||||
"mkv",
|
||||
"--no-overwrites",
|
||||
"--no-post-overwrites",
|
||||
"-o",
|
||||
f"{base}.intermediate.%(ext)s",
|
||||
playlist_path.as_uri(),
|
||||
],
|
||||
"downloading the authorized embedded HLS media playlist",
|
||||
secrets=(playlist_url, urllib.parse.urlsplit(playlist_url).query, rewritten),
|
||||
)
|
||||
intermediate = fetch._artifact(output_dir, f"{base}.intermediate.")
|
||||
if intermediate is None:
|
||||
raise IngestError("yt-dlp completed but no embedded HLS intermediate was written")
|
||||
return intermediate, result
|
||||
|
||||
|
||||
def _build_manifest(
|
||||
resource_map: dict[str, Any],
|
||||
output_dir: Path,
|
||||
*,
|
||||
target_language: str,
|
||||
deliverable: str,
|
||||
choice: fetch.SubtitleChoice | None,
|
||||
) -> dict[str, Any]:
|
||||
title = _required_text(resource_map.get("title"), "title")
|
||||
video_id = _required_text(resource_map.get("id"), "id")
|
||||
page_url = _validated_url(resource_map.get("page_url"), "page_url")
|
||||
info = {
|
||||
"title": title,
|
||||
"id": video_id,
|
||||
"webpage_url": page_url,
|
||||
"extractor_key": "AuthorizedEmbeddedHLS",
|
||||
"duration": _optional_number(resource_map.get("duration_seconds"), "duration_seconds"),
|
||||
"width": _optional_number(resource_map.get("width"), "width"),
|
||||
"height": _optional_number(resource_map.get("height"), "height"),
|
||||
"language": choice.language if choice else resource_map.get("declared_language"),
|
||||
}
|
||||
manifest = fetch._manifest_base(
|
||||
info=info,
|
||||
url=page_url,
|
||||
output_dir=output_dir,
|
||||
browser_cookies="browser-session",
|
||||
allow_remote_ejs=False,
|
||||
choice=choice,
|
||||
deliverable=deliverable,
|
||||
target_language=target_language,
|
||||
)
|
||||
manifest["authentication"]["mode"] = "browser-confirmed-resource-map"
|
||||
manifest["execution"] = {
|
||||
"resume": False,
|
||||
"embedded_hls": True,
|
||||
"authorization_query_logged": False,
|
||||
"temporary_playlist_cleaned": True,
|
||||
}
|
||||
return manifest
|
||||
|
||||
|
||||
def execute(args: argparse.Namespace) -> int:
|
||||
resource_map_path = args.resource_map.expanduser().resolve()
|
||||
resource_map = _read_private_resource_map(resource_map_path)
|
||||
target_language = fetch._validated_target_language(args.target_lang)
|
||||
playlist_url = _validated_url(resource_map.get("playlist_url"), "playlist_url")
|
||||
choice, subtitle_config = _subtitle_choice(resource_map, target_language)
|
||||
if args.deliver in fetch.SUBTITLE_ONLY_DELIVERABLES and choice is None:
|
||||
raise IngestError("A subtitle-only delivery requires one selected source subtitle")
|
||||
|
||||
yt_dlp = fetch._require_executable("yt-dlp", "Install the current official yt-dlp release.")
|
||||
ffmpeg = fetch._require_executable("ffmpeg", "Install FFmpeg.")
|
||||
ffprobe = fetch._require_executable("ffprobe", "Install FFmpeg with ffprobe.")
|
||||
output_dir = _prepare_output_dir(
|
||||
args.output_dir.expanduser().resolve(),
|
||||
resource_map_path,
|
||||
resume=args.resume,
|
||||
cleanup_resource_map=args.cleanup_resource_map,
|
||||
)
|
||||
manifest = _build_manifest(
|
||||
resource_map,
|
||||
output_dir,
|
||||
target_language=target_language,
|
||||
deliverable=args.deliver,
|
||||
choice=choice,
|
||||
)
|
||||
manifest["execution"]["resume"] = bool(args.resume)
|
||||
title = manifest["source"]["title"]
|
||||
video_id = manifest["source"]["id"]
|
||||
base = fetch.safe_stem(title, video_id)
|
||||
names = manifest["delivery_names"]
|
||||
subtitles_only = args.deliver in fetch.SUBTITLE_ONLY_DELIVERABLES
|
||||
warnings: list[str] = []
|
||||
completed: list[Any] = []
|
||||
|
||||
intermediate: Path | None = None
|
||||
media_probe: dict[str, Any] = {}
|
||||
master: Path | None = None
|
||||
fallback: Path | None = None
|
||||
fallback_error: str | None = None
|
||||
if not subtitles_only:
|
||||
print("Downloading authorized embedded HLS at maximum observed quality…", file=os.sys.stderr)
|
||||
intermediate, completed_result = _download_hls(
|
||||
playlist_url,
|
||||
output_dir,
|
||||
base,
|
||||
yt_dlp=yt_dlp,
|
||||
concurrent_fragments=args.concurrent_fragments,
|
||||
)
|
||||
completed.append(completed_result)
|
||||
media_probe = fetch._ffprobe(intermediate, ffprobe)
|
||||
if not any(
|
||||
isinstance(stream, dict) and stream.get("codec_type") == "video"
|
||||
for stream in media_probe.get("streams", [])
|
||||
):
|
||||
raise IngestError("Embedded HLS intermediate has no video stream")
|
||||
master, remux_error = fetch._try_lossless_mp4(
|
||||
intermediate,
|
||||
output_dir / f"{base}.master.mp4",
|
||||
ffmpeg,
|
||||
replace_existing=args.resume,
|
||||
)
|
||||
if remux_error:
|
||||
warnings.append(f"Lossless MP4 remux unavailable: {remux_error}")
|
||||
if master is None and args.mp4_fallback:
|
||||
fallback, fallback_error = fetch._create_fallback_mp4(
|
||||
intermediate,
|
||||
output_dir / f"{base}.fallback.mp4",
|
||||
ffmpeg,
|
||||
replace_existing=args.resume,
|
||||
)
|
||||
if fallback_error:
|
||||
raise IngestError(f"Requested MP4 fallback failed: {fallback_error}")
|
||||
|
||||
cover = _download_cover(
|
||||
resource_map.get("cover_url"),
|
||||
output_dir / names["cover"],
|
||||
playlist_url=playlist_url,
|
||||
ffmpeg=ffmpeg,
|
||||
resume=args.resume,
|
||||
)
|
||||
if not subtitles_only and cover is None:
|
||||
warnings.append("The embedded player resource map did not declare a cover")
|
||||
|
||||
original_subtitle: Path | None = None
|
||||
source_srt: Path | None = None
|
||||
conversion_method: str | None = None
|
||||
parent_hash: str | None = None
|
||||
if choice and subtitle_config:
|
||||
language_label = fetch._subtitle_language_label(choice.language)
|
||||
original_subtitle = output_dir / (
|
||||
f"{base}.source-original.{language_label}.{choice.original_format}"
|
||||
)
|
||||
if not (args.resume and original_subtitle.is_file()):
|
||||
_atomic_download(
|
||||
_validated_url(subtitle_config.get("url"), "subtitle.url"),
|
||||
original_subtitle,
|
||||
playlist_url=playlist_url,
|
||||
)
|
||||
source_srt, conversion_method, parent_hash = fetch._derive_source_srt(
|
||||
original_subtitle,
|
||||
output_dir / f"{base}.source-srt.{language_label}.srt",
|
||||
ffmpeg=ffmpeg,
|
||||
replace_existing=args.resume,
|
||||
)
|
||||
|
||||
original_record = fetch._file_record(original_subtitle, output_dir, checksum=True)
|
||||
source_srt_record = fetch._file_record(source_srt, output_dir, checksum=True)
|
||||
if original_record and source_srt_record:
|
||||
if original_record.get("sha256") != parent_hash:
|
||||
raise IngestError("Original subtitle changed during SRT derivation")
|
||||
original_record["content_role"] = "immutable-parent"
|
||||
source_srt_record["conversion_method"] = conversion_method
|
||||
source_srt_record["derived_from"] = {
|
||||
"path": original_record["path"],
|
||||
"sha256": parent_hash,
|
||||
}
|
||||
|
||||
manifest["status"] = "downloaded"
|
||||
manifest["artifacts"] = {
|
||||
"intermediate": fetch._file_record(intermediate, output_dir),
|
||||
"media_streams": media_probe.get("streams", []),
|
||||
"lossless_mp4_master": fetch._file_record(master, output_dir),
|
||||
"lossy_mp4_fallback": {
|
||||
"requested": bool(args.mp4_fallback),
|
||||
"created": fetch._file_record(fallback, output_dir),
|
||||
"reason_not_created": (
|
||||
"lossless_master_available"
|
||||
if args.mp4_fallback and master is not None
|
||||
else (fallback_error if args.mp4_fallback and fallback is None else None)
|
||||
),
|
||||
"video_encoding": "libx264 preset=slow crf=18" if fallback else None,
|
||||
"audio_encoding": "aac 256k" if fallback else None,
|
||||
},
|
||||
"cover": fetch._file_record(cover, output_dir, checksum=True),
|
||||
"subtitle": {
|
||||
"language": choice.language,
|
||||
"kind": choice.kind,
|
||||
"label": subtitle_config.get("label"),
|
||||
"original": original_record,
|
||||
"source_srt": source_srt_record,
|
||||
"original_is_never_modified_by_this_script": True,
|
||||
}
|
||||
if choice and subtitle_config
|
||||
else None,
|
||||
}
|
||||
manifest["warnings"].extend(warnings)
|
||||
manifest["warnings"].extend(
|
||||
warning
|
||||
for warning in fetch._warning_lines(
|
||||
completed, (playlist_url, urllib.parse.urlsplit(playlist_url).query)
|
||||
)
|
||||
if warning not in manifest["warnings"]
|
||||
)
|
||||
destination = fetch._write_manifest(output_dir, manifest)
|
||||
exit_code = fetch._advance_bilingual_stage(destination)
|
||||
if args.cleanup_resource_map:
|
||||
resource_map_path.unlink()
|
||||
return exit_code
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Ingest one browser-confirmed authorized embedded HLS media playlist."
|
||||
)
|
||||
parser.add_argument("--resource-map", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--target-lang", default=fetch.DEFAULT_TARGET_LANGUAGE)
|
||||
parser.add_argument("--deliver", choices=fetch.DELIVERABLES, default="full")
|
||||
parser.add_argument("--concurrent-fragments", type=int, default=8)
|
||||
parser.add_argument("--mp4-fallback", action="store_true")
|
||||
parser.add_argument("--resume", action="store_true")
|
||||
parser.add_argument(
|
||||
"--cleanup-resource-map",
|
||||
action="store_true",
|
||||
help="delete the exact private resource-map file after a successful manifest write",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = _parser()
|
||||
args = parser.parse_args(argv)
|
||||
if not 1 <= args.concurrent_fragments <= 32:
|
||||
parser.error("--concurrent-fragments must be between 1 and 32")
|
||||
try:
|
||||
return execute(args)
|
||||
except (IngestError, fetch.FetchError) as exc:
|
||||
print(f"error: {fetch.sanitize_diagnostic(str(exc))}", file=os.sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+101
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run MaterialSub dependency and font checks before a long download."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from typing import Any, Sequence
|
||||
|
||||
import burn_subtitles as burn
|
||||
|
||||
|
||||
def assess(*, font: str, allow_missing_font: bool, youtube: bool) -> tuple[int, dict[str, Any]]:
|
||||
if sys.version_info < (3, 10):
|
||||
return 2, {
|
||||
"complete": False,
|
||||
"stage": "dependency_required",
|
||||
"error": "Python 3.10 or newer is required",
|
||||
}
|
||||
yt_dlp = shutil.which("yt-dlp")
|
||||
if not yt_dlp:
|
||||
return 2, {
|
||||
"complete": False,
|
||||
"stage": "dependency_required",
|
||||
"error": "yt-dlp was not found in PATH",
|
||||
}
|
||||
try:
|
||||
ffmpeg, ffprobe = burn._required_executables()
|
||||
burn._require_libass_subtitles_filter(ffmpeg)
|
||||
except burn.BurnError as exc:
|
||||
return 2, {
|
||||
"complete": False,
|
||||
"stage": "dependency_required",
|
||||
"error": str(exc),
|
||||
}
|
||||
|
||||
warnings: list[str] = []
|
||||
if youtube and not shutil.which("deno"):
|
||||
warnings.append(
|
||||
"Deno was not found; current yt-dlp may expose fewer YouTube formats"
|
||||
)
|
||||
installed = burn._font_installed(font)
|
||||
if installed is False and not allow_missing_font:
|
||||
return 3, {
|
||||
"complete": False,
|
||||
"stage": "font_decision_required",
|
||||
"font": font,
|
||||
"font_installed": False,
|
||||
"substitution_allowed": False,
|
||||
"instruction": (
|
||||
"Install the requested font, or rerun preflight and the final burn "
|
||||
"with --allow-missing-font to accept libass substitution"
|
||||
),
|
||||
"warnings": warnings,
|
||||
}
|
||||
if installed is False:
|
||||
warnings.append(f"Font {font!r} is missing; libass substitution was accepted")
|
||||
elif installed is None:
|
||||
warnings.append(f"Could not verify whether font {font!r} is installed")
|
||||
return 0, {
|
||||
"complete": True,
|
||||
"stage": "preflight_complete",
|
||||
"font": font,
|
||||
"font_installed": installed,
|
||||
"substitution_allowed": bool(installed is False and allow_missing_font),
|
||||
"executables": {
|
||||
"yt_dlp": yt_dlp,
|
||||
"ffmpeg": ffmpeg,
|
||||
"ffprobe": ffprobe,
|
||||
},
|
||||
"warnings": warnings,
|
||||
}
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Check MaterialSub dependencies before downloading.")
|
||||
parser.add_argument("--font", default="MiSans")
|
||||
parser.add_argument("--allow-missing-font", action="store_true")
|
||||
parser.add_argument("--youtube", action="store_true")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
font = args.font.strip()
|
||||
if not font:
|
||||
print(json.dumps({"complete": False, "error": "font cannot be empty"}))
|
||||
return 2
|
||||
exit_code, result = assess(
|
||||
font=font,
|
||||
allow_missing_font=args.allow_missing_font,
|
||||
youtube=args.youtube,
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
||||
return exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+1291
File diff suppressed because it is too large
Load Diff
+247
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail closed until a downloaded video job reaches its required deliverable."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Sequence
|
||||
|
||||
|
||||
class DeliveryError(RuntimeError):
|
||||
"""A malformed or unreadable delivery job."""
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError as exc:
|
||||
raise DeliveryError(f"manifest not found: {path}") from exc
|
||||
except json.JSONDecodeError as exc:
|
||||
raise DeliveryError(f"invalid JSON in {path}: {exc}") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise DeliveryError(f"manifest root must be an object: {path}")
|
||||
return value
|
||||
|
||||
|
||||
def _artifact_path(job_dir: Path, value: Any) -> Path | None:
|
||||
if not isinstance(value, dict) or not isinstance(value.get("path"), str):
|
||||
return None
|
||||
path = Path(value["path"])
|
||||
return path if path.is_absolute() else job_dir / path
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _verify_citation_watermark(job_dir: Path, download: dict[str, Any], burned: Path) -> None:
|
||||
citation = download.get("citation_watermark")
|
||||
if not isinstance(citation, dict) or citation.get("enabled") is not True:
|
||||
return
|
||||
citation_name = citation.get("citation_file")
|
||||
citation_hash = citation.get("citation_sha256")
|
||||
if not isinstance(citation_name, str) or Path(citation_name).name != citation_name:
|
||||
raise DeliveryError("citation_watermark.citation_file must be a plain filename")
|
||||
if not isinstance(citation_hash, str) or not re.fullmatch(r"[0-9a-fA-F]{64}", citation_hash):
|
||||
raise DeliveryError("citation_watermark.citation_sha256 is invalid")
|
||||
citation_path = job_dir / citation_name
|
||||
if not citation_path.is_file() or _sha256_file(citation_path) != citation_hash.lower():
|
||||
raise DeliveryError("citation watermark text is missing or its checksum changed")
|
||||
receipt_path = burned.with_suffix(burned.suffix + ".citation.json")
|
||||
if not receipt_path.is_file():
|
||||
raise DeliveryError(f"citation watermark burn receipt is missing: {receipt_path}")
|
||||
receipt = _read_json(receipt_path)
|
||||
expected = {
|
||||
"output_file": burned.name,
|
||||
"output_sha256": _sha256_file(burned),
|
||||
"citation_sha256": citation_hash.lower(),
|
||||
"position": "top-left",
|
||||
}
|
||||
for key, value in expected.items():
|
||||
if receipt.get(key) != value:
|
||||
raise DeliveryError(f"citation watermark receipt has a stale or invalid {key}")
|
||||
|
||||
|
||||
def _existing_video_artifact(job_dir: Path, artifacts: dict[str, Any]) -> Path | None:
|
||||
records = [artifacts.get("lossless_mp4_master"), artifacts.get("intermediate")]
|
||||
fallback = artifacts.get("lossy_mp4_fallback")
|
||||
if isinstance(fallback, dict):
|
||||
records.append(fallback.get("created"))
|
||||
for record in records:
|
||||
path = _artifact_path(job_dir, record)
|
||||
if path is not None and path.is_file() and path.stat().st_size:
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
DELIVERABLES = ("full", "video", "subs", "bilingual-subs")
|
||||
|
||||
|
||||
def assess_delivery(download_manifest: Path) -> dict[str, Any]:
|
||||
download_manifest = download_manifest.expanduser().resolve()
|
||||
download = _read_json(download_manifest)
|
||||
configured_dir = download.get("output_directory")
|
||||
job_dir = (
|
||||
Path(configured_dir).expanduser().resolve()
|
||||
if isinstance(configured_dir, str)
|
||||
else download_manifest.parent
|
||||
)
|
||||
deliverable = download.get("deliverable")
|
||||
if deliverable not in DELIVERABLES:
|
||||
deliverable = "full"
|
||||
artifacts = download.get("artifacts")
|
||||
if not isinstance(artifacts, dict):
|
||||
raise DeliveryError("download manifest has no artifacts object")
|
||||
if deliverable in ("full", "video") and _existing_video_artifact(job_dir, artifacts) is None:
|
||||
raise DeliveryError("no declared video artifact exists on disk")
|
||||
|
||||
def complete(stage: str, **extra: Any) -> dict[str, Any]:
|
||||
return {
|
||||
"complete": True,
|
||||
"stage": stage,
|
||||
"deliverable": deliverable,
|
||||
"job_dir": str(job_dir),
|
||||
"missing": [],
|
||||
**extra,
|
||||
}
|
||||
|
||||
subtitle_record = artifacts.get("subtitle")
|
||||
subtitle = (
|
||||
_artifact_path(job_dir, subtitle_record.get("source_srt"))
|
||||
if isinstance(subtitle_record, dict)
|
||||
else None
|
||||
)
|
||||
if subtitle is not None and not subtitle.is_file():
|
||||
raise DeliveryError(f"declared source subtitle is missing: {subtitle}")
|
||||
if deliverable in ("subs", "bilingual-subs") and subtitle is None:
|
||||
raise DeliveryError(
|
||||
"a subtitle delivery was requested, but the manifest declares no source subtitle"
|
||||
)
|
||||
has_dialogue = (
|
||||
subtitle is not None
|
||||
and isinstance(subtitle_record, dict)
|
||||
and subtitle_record.get("dialogue") is not False
|
||||
)
|
||||
|
||||
if deliverable == "video":
|
||||
return complete("video_complete")
|
||||
if deliverable == "subs":
|
||||
return complete("subs_complete")
|
||||
if not has_dialogue:
|
||||
# full falls back to plain video; bilingual-subs still delivered the
|
||||
# source subtitle files even though nothing was translatable.
|
||||
stage = "video_only_complete" if deliverable == "full" else "subs_complete"
|
||||
return complete(stage)
|
||||
|
||||
subtitle_dir = job_dir / "subtitles"
|
||||
subtitle_manifest_path = subtitle_dir / "subtitle-manifest.json"
|
||||
if not subtitle_manifest_path.is_file():
|
||||
return {
|
||||
"complete": False,
|
||||
"stage": "subtitle_prepare_required",
|
||||
"job_dir": str(job_dir),
|
||||
"missing": [str(subtitle_manifest_path)],
|
||||
}
|
||||
|
||||
subtitle_manifest = _read_json(subtitle_manifest_path)
|
||||
batches = subtitle_manifest.get("translation_batches")
|
||||
if not isinstance(batches, list) or not batches:
|
||||
raise DeliveryError("subtitle manifest has no translation batches")
|
||||
output_dir_value = subtitle_manifest.get("translation_output_dir")
|
||||
translation_output_dir = (
|
||||
Path(output_dir_value)
|
||||
if isinstance(output_dir_value, str)
|
||||
else subtitle_dir / "translation-output"
|
||||
)
|
||||
missing_batches: list[str] = []
|
||||
for batch in batches:
|
||||
if not isinstance(batch, dict) or not isinstance(batch.get("path"), str):
|
||||
raise DeliveryError("subtitle manifest has an invalid translation batch")
|
||||
name = Path(batch["path"]).name
|
||||
if not (translation_output_dir / name).is_file():
|
||||
missing_batches.append(name)
|
||||
if missing_batches:
|
||||
return {
|
||||
"complete": False,
|
||||
"stage": "translation_required",
|
||||
"job_dir": str(job_dir),
|
||||
"missing": missing_batches,
|
||||
}
|
||||
|
||||
rendered_dir = subtitle_dir / "rendered"
|
||||
required_rendered = [rendered_dir / "bilingual.ass", rendered_dir / "validation.json"]
|
||||
missing_rendered = [str(path) for path in required_rendered if not path.is_file()]
|
||||
if missing_rendered:
|
||||
return {
|
||||
"complete": False,
|
||||
"stage": "render_required",
|
||||
"job_dir": str(job_dir),
|
||||
"missing": missing_rendered,
|
||||
}
|
||||
if deliverable == "bilingual-subs":
|
||||
return complete("bilingual_subs_complete", rendered_dir=str(rendered_dir))
|
||||
|
||||
delivery_names = download.get("delivery_names")
|
||||
burned_name = (
|
||||
delivery_names.get("bilingual_video")
|
||||
if isinstance(delivery_names, dict)
|
||||
else None
|
||||
)
|
||||
if isinstance(burned_name, str) and burned_name:
|
||||
if Path(burned_name).name != burned_name:
|
||||
raise DeliveryError("delivery_names.bilingual_video must be a plain filename")
|
||||
burned = job_dir / burned_name
|
||||
if burned.is_file() and burned.stat().st_size:
|
||||
_verify_citation_watermark(job_dir, download, burned)
|
||||
return complete("bilingual_complete", burned_video=str(burned))
|
||||
missing = [str(burned)]
|
||||
else:
|
||||
legacy = sorted(
|
||||
path
|
||||
for path in job_dir.glob("*.bilingual.mp4")
|
||||
if path.is_file() and path.stat().st_size
|
||||
)
|
||||
if legacy:
|
||||
_verify_citation_watermark(job_dir, download, legacy[-1])
|
||||
return complete("bilingual_complete", burned_video=str(legacy[-1]))
|
||||
missing = ["*.bilingual.mp4"]
|
||||
if missing:
|
||||
return {
|
||||
"complete": False,
|
||||
"stage": "burn_required",
|
||||
"job_dir": str(job_dir),
|
||||
"missing": missing,
|
||||
}
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Check whether a video job is video-only complete or bilingual complete."
|
||||
)
|
||||
parser.add_argument("download_manifest", type=Path)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
try:
|
||||
result = assess_delivery(args.download_manifest)
|
||||
except (DeliveryError, OSError) as exc:
|
||||
print(f"delivery verification error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
||||
return 0 if result["complete"] else 3
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user