Files
ai-manju/scripts/align_subtitles.py
T
rong a8e10b7b17 fix: 字幕按实测发声段对齐 + 静音镜/包装踩坑 (v1.2.1)
- 纠正 render.md/pitfalls.md「按设计时间缩放字幕」错误做法,改为只认实测发声段
- 新增 scripts/align_subtitles.py(泛化对齐工具,配置外置、支持 dump-segs/init-config/dry-run)
- render_native.py 加 --ass 参数,读对齐 ASS 不被内置设计时间轴覆盖
- pitfalls/native-av/README/SKILL 补六条踩坑;install.sh 清单同步
2026-11-19 17:14:14 +08:00

294 lines
13 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""align_subtitles.py — 台词镜字幕按【实测发声段】对齐(音画同出专用,后期零算力)。
为什么需要它:h3v3 [Audio] 里的秒数是写提示词时的设计值,H3 实际语速/停顿/起句延迟
与设计系统性错位,照抄(哪怕按 源实际/设计 缩放)都会让字幕早于口型 0.5-1.5s。
本脚本用 ffmpeg silencedetect 测真实发声段,把台词逐句贴到实际说话区间。
台词文本一字不改,只重排时间。不等同 ASR:AudioTranscribe 常超时,波形检测更稳。
配置放在项目里(典型 work/EPxx/align_config.json),含每镜 发声段→台词 的映射、
需静音的镜、可选包装层样式。用 --dump-segs 打印每镜发声段后再填 ALIGN。
用法:
# ① 看每镜实测发声段(填 ALIGN 用)
python3 tools/align_subtitles.py projects/EP01/30_shots.json --dump-segs
# ② 生成配置模板(按句数顺序一一映射,需人工核对)
python3 tools/align_subtitles.py projects/EP01/30_shots.json --init-config work/EP01/align_config.json
# ③ dry-run 核对每镜对齐时间轴
python3 tools/align_subtitles.py projects/EP01/30_shots.json --config work/EP01/align_config.json --dry-run
# ④ 生成对齐的 ep.ass(+ 可选 package.ass)
python3 tools/align_subtitles.py projects/EP01/30_shots.json --config work/EP01/align_config.json \
--out work/EP01/render_native/ep.ass
# ⑤ 用对齐的 ASS 缝合(render_native.py 需支持 --ass,见同版说明)
python3 tools/render_native.py projects/EP01/30_shots.json --ass work/EP01/render_native/ep.ass
"""
import argparse
import json
import os
import re
import subprocess
import sys
ROOT_ENV = "MANJU_ROOT"
def ff():
import imageio_ffmpeg
return imageio_ffmpeg.get_ffmpeg_exe()
def find_root(explicit=None):
"""工作空间根 = 显式 MANJU_ROOT → 从 shots 路径上溯找 config/pipeline.json → cwd。"""
if explicit:
return os.path.abspath(explicit)
env = os.environ.get(ROOT_ENV)
if env:
return os.path.abspath(env)
d = os.getcwd()
while True:
if os.path.exists(os.path.join(d, "config", "pipeline.json")):
return d
p = os.path.dirname(d)
if p == d:
break
d = p
return os.getcwd()
def run(args):
cmd = [ff(), "-hide_banner", "-loglevel", "error", "-y"] + args
r = subprocess.run(cmd, capture_output=True, text=True)
if r.returncode != 0:
print("FFMPEG FAILED:\n" + " ".join(cmd) + "\n" + r.stderr[-2000:], file=sys.stderr)
raise SystemExit(1)
return r
def probe(path):
r = subprocess.run([ff(), "-hide_banner", "-i", str(path)],
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
m = re.search(r"Duration:\s*(\d+):(\d+):([\d.]+)", r.stderr)
return (int(m.group(1)) * 3600 + int(m.group(2)) * 60 + float(m.group(3))) if m else None
def voiced_segments(path, noise, dmin):
"""从静音块反推发声段 [(start,end)](相对本镜)。"""
r = subprocess.run([ff(), "-hide_banner", "-i", path,
"-af", "silencedetect=noise=%s:d=%s" % (noise, dmin),
"-f", "null", "-"], capture_output=True, text=True)
total = probe(path)
sil = [(float(t) - float(du), float(t)) for t, du in
re.findall(r"silence_end:\s*([\d.]+)\s*\|\s*silence_duration:\s*([\d.]+)", r.stderr)]
segs, prev = [], 0.0
for a, b in sil:
if a > prev + 0.12:
segs.append((prev, a))
prev = max(prev, b)
if prev < total - 0.12:
segs.append((prev, total))
return segs
def design_lines(path):
t = open(path, encoding="utf-8").read()
if "[Audio]" not in t:
return []
seg = t.split("[Audio]")[1].split("[Narrative]")[0]
return [(float(a), float(b), txt) for a, b, txt in
re.findall(r'(\d+\.?\d*)s\s*-\s*(\d+\.?\d*)s\s*:\s*"([^"]+)"', seg)]
def ts(t):
h = int(t // 3600); m = int(t % 3600 // 60); s = t % 60
return "%d:%02d:%05.2f" % (h, m, s)
def wrap_line(text, maxlen=13):
punct = set(",。;!?、,.;!?")
parts = [p for p in re.split(r"([,。;!?、,.;!?])", text) if p]
lines, cur = [], ""
for p in parts:
if p in punct:
cur += p
elif len(cur) + len(p) <= maxlen or not cur:
cur += p
else:
lines.append(cur); cur = p
if cur:
lines.append(cur)
return "\\N".join(lines[:3])
SUB_STYLE = "Style: Default,PingFang SC,40,&H00FFFFFF,&H00FFFFFF,&H000C1420,&H90000000,1,0,0,0,100,100,0,0,1,3,1,2,40,40,120,1"
def write_ass(timeline, path, styles=None, events=None):
st = styles if styles is not None else [SUB_STYLE]
head = ("[Script Info]\nScriptType: v4.00+\nPlayResX: 576\nPlayResY: 1024\n"
"ScaledBorderAndShadow: yes\nWrapStyle: 2\n\n[V4+ Styles]\n"
"Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding\n"
+ "\n".join(st) + "\n\n[Events]\n"
"Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n")
body = ""
for a, b, txt in timeline:
body += "Dialogue: 0,%s,%s,Default,,0,0,0,,{\\fad(150,150)}%s\n" % (ts(a), ts(b), wrap_line(txt))
for a, b, style, text, fade in (events or []):
fx = ("{\\fad(%d,%d)}" % (fade, fade)) if fade else ""
body += "Dialogue: 0,%s,%s,%s,,0,0,0,,%s%s\n" % (ts(a), ts(b), style, fx, text)
with open(path, "w", encoding="utf-8") as f:
f.write(head + body)
def build_timeline(data, root, cfg, vid_dir=None):
noise = cfg.get("silence", {}).get("noise", "-32dB")
dmin = str(cfg.get("silence", {}).get("d", "0.12"))
align = cfg.get("align", {})
vdir = vid_dir or os.path.join(root, "work", shots_ep(data), "videos", "native")
timeline, report, cursor = [], [], 0.0
for s in data["shots"]:
sid = s["shotId"]
src = os.path.join(vdir, sid + ".mp4")
prompt = find_prompt(root, data, sid)
if not os.path.exists(src):
print("WARN 缺视频 %s,跳过对齐" % src, file=sys.stderr)
cursor += float(s.get("duration", 0)); continue
d = probe(src)
lines = design_lines(prompt)
segs = voiced_segments(src, noise, dmin)
groups = align.get(sid)
if not lines:
report.append((sid, cursor, d, segs, lines, []))
cursor += d; continue
if groups is None:
raise SystemExit("ERROR %s 无 ALIGN 配置,请先 --dump-segs + --init-config 生成" % sid)
if len(groups) != len(lines):
raise SystemExit("ERROR %s: 台词 %d 句但 ALIGN %d 组" % (sid, len(lines), len(groups)))
max_idx = max(i for g in groups for i in g) if groups else -1
if max_idx >= len(segs):
raise SystemExit("ERROR %s: ALIGN 引用段 %d 但只检测到 %d 段(阈值 %s/%s)"
% (sid, max_idx, len(segs), noise, dmin))
assigned = []
for gi, g in enumerate(groups):
d0, d1, txt = lines[gi]
a, b = (segs[g[0]][0], segs[g[-1]][1]) if g else (0.0, 0.0)
assigned.append([a, b, txt, d1 - d0, str(g)])
# 同段被多句引用(连读)按设计时长比例切
used = {}
for i, (a, b, *_rest) in enumerate(assigned):
used.setdefault((a, b), []).append(i)
for (a, b), idxs in used.items():
if len(idxs) > 1:
total_dw = sum(assigned[i][3] for i in idxs)
p = a
for i in idxs:
q = p + (b - a) * (assigned[i][3] / total_dw)
assigned[i][0], assigned[i][1] = p, q
p = q
shot_tl = []
for a, b, txt, _dw, _g in assigned:
a = max(0.0, a - 0.04); b = min(d, b + 0.04)
if b - a < 0.45:
b = min(d, a + 0.45)
shot_tl.append((cursor + a, cursor + b, txt))
timeline.append((cursor + a, cursor + b, txt))
report.append((sid, cursor, d, segs, lines, shot_tl))
cursor += d
return timeline, report, cursor
def shots_ep(shots):
return shots.get("episode", "EP00")
def find_prompt(root, shots, sid):
ep = shots_ep(shots)
for rel in ("projects/%s/40_prompts/%s.h3v3.txt" % (ep, sid),
"40_prompts/%s.h3v3.txt" % sid):
p = os.path.join(root, rel)
if os.path.exists(p):
return p
return os.path.join(root, "projects", ep, "40_prompts", sid + ".h3v3.txt")
def print_report(report):
for sid, cursor, d, segs, lines, tl in report:
sdesc = " ".join("%d:%.2f-%.2f(%.2f)" % (i, a, b, b - a) for i, (a, b) in enumerate(segs))
print("---- %s @%.2f-%.2fs 段=%d 句=%d" % (sid, cursor, cursor + d, len(segs), len(lines)))
print(" 发声段: %s" % sdesc)
for (a, b, txt), (d0, d1, _) in zip(tl, lines):
print(" %.2f-%.2f %s (设计 %.2f-%.2f)" % (a, b, txt, d0, d1))
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("shots", help="30_shots.json 路径")
ap.add_argument("--root", help="工作空间根(默认上溯找 config/pipeline.json)")
ap.add_argument("--config", help="对齐配置 JSON(work/EPxx/align_config.json)")
ap.add_argument("--dump-segs", action="store_true", help="只打印每镜实测发声段,供填 ALIGN")
ap.add_argument("--init-config", metavar="PATH", help="生成配置模板(按句数顺序一一映射)")
ap.add_argument("--dry-run", action="store_true", help="只打印对齐时间轴,不写文件")
ap.add_argument("--out", help="输出对齐后的 ep.ass 路径")
ap.add_argument("--package-out", help="输出包装层 package.ass 路径(配置含 package 时)")
a = ap.parse_args()
root = find_root(a.root)
shots = json.load(open(os.path.abspath(a.shots), encoding="utf-8"))
cfg = json.load(open(os.path.abspath(a.config), encoding="utf-8")) if a.config else {}
ep = shots_ep(shots)
vdir = os.path.join(root, "work", ep, "videos", "native")
if a.dump_segs:
noise = cfg.get("silence", {}).get("noise", "-32dB")
dmin = str(cfg.get("silence", {}).get("d", "0.12"))
for s in shots["shots"]:
sid = s["shotId"]
src = os.path.join(vdir, sid + ".mp4")
if not os.path.exists(src):
print("%s 缺视频 %s" % (sid, src)); continue
segs = voiced_segments(src, noise, dmin)
lines = design_lines(find_prompt(root, shots, sid))
print("%s 段=%d 句=%d %s" % (
sid, len(segs), len(lines),
" ".join("%d:%.2f-%.2f" % (i, x, y) for i, (x, y) in enumerate(segs))))
return 0
if a.init_config:
tpl = {"_readme": "ALIGN 把发声段索引分给各句;先 --dump-segs 看段,再人工校准(默认一一映射仅供参考)。",
"silence": cfg.get("silence", {"noise": "-32dB", "d": "0.12"}),
"align": {}, "mute_shots": []}
noise = tpl["silence"]["noise"]; dmin = str(tpl["silence"]["d"])
for s in shots["shots"]:
sid = s["shotId"]
src = os.path.join(vdir, sid + ".mp4")
nlines = len(design_lines(find_prompt(root, shots, sid)))
tpl["align"][sid] = [[i] for i in range(nlines)] if nlines else []
os.makedirs(os.path.dirname(os.path.abspath(a.init_config)), exist_ok=True)
with open(a.init_config, "w", encoding="utf-8") as f:
json.dump(tpl, f, ensure_ascii=False, indent=2)
print("配置模板 -> %s(务必 --dump-segs 后人工校准)" % a.init_config)
return 0
timeline, report, total = build_timeline(shots, root, cfg, vdir)
print("字幕 %d 句,总长 %.2fs" % (len(timeline), total), flush=True)
print_report(report)
if a.dry_run:
print("\n[DRY-RUN] 未写文件。静音镜:%s" % (cfg.get("mute_shots") or "无"))
return 0
if a.out:
os.makedirs(os.path.dirname(os.path.abspath(a.out)), exist_ok=True)
write_ass(timeline, a.out)
print("对齐字幕 -> %s" % a.out)
if a.package_out and cfg.get("package"):
pkg = cfg["package"]
write_ass([], a.package_out, styles=pkg.get("styles"), events=pkg.get("events"))
print("包装层 -> %s" % a.package_out)
if not a.out and not a.package_out:
print("提示:加 --out / --package-out 才会写 ASS;当前仅预览。")
return 0
if __name__ == "__main__":
sys.exit(main())