- 纠正 render.md/pitfalls.md「按设计时间缩放字幕」错误做法,改为只认实测发声段 - 新增 scripts/align_subtitles.py(泛化对齐工具,配置外置、支持 dump-segs/init-config/dry-run) - render_native.py 加 --ass 参数,读对齐 ASS 不被内置设计时间轴覆盖 - pitfalls/native-av/README/SKILL 补六条踩坑;install.sh 清单同步
294 lines
13 KiB
Python
294 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""align_subtitles.py — 台词镜字幕按【实测发声段】对齐(音画同出专用,后期零算力)。
|
||
|
||
为什么需要它:h3v3 [Audio] 里的秒数是写提示词时的设计值,H3 实际语速/停顿/起句延迟
|
||
与设计系统性错位,照抄(哪怕按 源实际/设计 缩放)都会让字幕早于口型 0.5-1.5s。
|
||
本脚本用 ffmpeg silencedetect 测真实发声段,把台词逐句贴到实际说话区间。
|
||
台词文本一字不改,只重排时间。不等同 ASR:AudioTranscribe 常超时,波形检测更稳。
|
||
|
||
配置放在项目里(典型 work/EPxx/align_config.json),含每镜 发声段→台词 的映射、
|
||
需静音的镜、可选包装层样式。用 --dump-segs 打印每镜发声段后再填 ALIGN。
|
||
|
||
用法:
|
||
# ① 看每镜实测发声段(填 ALIGN 用)
|
||
python3 tools/align_subtitles.py projects/EP01/30_shots.json --dump-segs
|
||
# ② 生成配置模板(按句数顺序一一映射,需人工核对)
|
||
python3 tools/align_subtitles.py projects/EP01/30_shots.json --init-config work/EP01/align_config.json
|
||
# ③ dry-run 核对每镜对齐时间轴
|
||
python3 tools/align_subtitles.py projects/EP01/30_shots.json --config work/EP01/align_config.json --dry-run
|
||
# ④ 生成对齐的 ep.ass(+ 可选 package.ass)
|
||
python3 tools/align_subtitles.py projects/EP01/30_shots.json --config work/EP01/align_config.json \
|
||
--out work/EP01/render_native/ep.ass
|
||
# ⑤ 用对齐的 ASS 缝合(render_native.py 需支持 --ass,见同版说明)
|
||
python3 tools/render_native.py projects/EP01/30_shots.json --ass work/EP01/render_native/ep.ass
|
||
"""
|
||
import argparse
|
||
import json
|
||
import os
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
|
||
ROOT_ENV = "MANJU_ROOT"
|
||
|
||
|
||
def ff():
|
||
import imageio_ffmpeg
|
||
return imageio_ffmpeg.get_ffmpeg_exe()
|
||
|
||
|
||
def find_root(explicit=None):
|
||
"""工作空间根 = 显式 MANJU_ROOT → 从 shots 路径上溯找 config/pipeline.json → cwd。"""
|
||
if explicit:
|
||
return os.path.abspath(explicit)
|
||
env = os.environ.get(ROOT_ENV)
|
||
if env:
|
||
return os.path.abspath(env)
|
||
d = os.getcwd()
|
||
while True:
|
||
if os.path.exists(os.path.join(d, "config", "pipeline.json")):
|
||
return d
|
||
p = os.path.dirname(d)
|
||
if p == d:
|
||
break
|
||
d = p
|
||
return os.getcwd()
|
||
|
||
|
||
def run(args):
|
||
cmd = [ff(), "-hide_banner", "-loglevel", "error", "-y"] + args
|
||
r = subprocess.run(cmd, capture_output=True, text=True)
|
||
if r.returncode != 0:
|
||
print("FFMPEG FAILED:\n" + " ".join(cmd) + "\n" + r.stderr[-2000:], file=sys.stderr)
|
||
raise SystemExit(1)
|
||
return r
|
||
|
||
|
||
def probe(path):
|
||
r = subprocess.run([ff(), "-hide_banner", "-i", str(path)],
|
||
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
|
||
m = re.search(r"Duration:\s*(\d+):(\d+):([\d.]+)", r.stderr)
|
||
return (int(m.group(1)) * 3600 + int(m.group(2)) * 60 + float(m.group(3))) if m else None
|
||
|
||
|
||
def voiced_segments(path, noise, dmin):
|
||
"""从静音块反推发声段 [(start,end)](相对本镜)。"""
|
||
r = subprocess.run([ff(), "-hide_banner", "-i", path,
|
||
"-af", "silencedetect=noise=%s:d=%s" % (noise, dmin),
|
||
"-f", "null", "-"], capture_output=True, text=True)
|
||
total = probe(path)
|
||
sil = [(float(t) - float(du), float(t)) for t, du in
|
||
re.findall(r"silence_end:\s*([\d.]+)\s*\|\s*silence_duration:\s*([\d.]+)", r.stderr)]
|
||
segs, prev = [], 0.0
|
||
for a, b in sil:
|
||
if a > prev + 0.12:
|
||
segs.append((prev, a))
|
||
prev = max(prev, b)
|
||
if prev < total - 0.12:
|
||
segs.append((prev, total))
|
||
return segs
|
||
|
||
|
||
def design_lines(path):
|
||
t = open(path, encoding="utf-8").read()
|
||
if "[Audio]" not in t:
|
||
return []
|
||
seg = t.split("[Audio]")[1].split("[Narrative]")[0]
|
||
return [(float(a), float(b), txt) for a, b, txt in
|
||
re.findall(r'(\d+\.?\d*)s\s*-\s*(\d+\.?\d*)s\s*:\s*"([^"]+)"', seg)]
|
||
|
||
|
||
def ts(t):
|
||
h = int(t // 3600); m = int(t % 3600 // 60); s = t % 60
|
||
return "%d:%02d:%05.2f" % (h, m, s)
|
||
|
||
|
||
def wrap_line(text, maxlen=13):
|
||
punct = set(",。;!?、,.;!?")
|
||
parts = [p for p in re.split(r"([,。;!?、,.;!?])", text) if p]
|
||
lines, cur = [], ""
|
||
for p in parts:
|
||
if p in punct:
|
||
cur += p
|
||
elif len(cur) + len(p) <= maxlen or not cur:
|
||
cur += p
|
||
else:
|
||
lines.append(cur); cur = p
|
||
if cur:
|
||
lines.append(cur)
|
||
return "\\N".join(lines[:3])
|
||
|
||
|
||
SUB_STYLE = "Style: Default,PingFang SC,40,&H00FFFFFF,&H00FFFFFF,&H000C1420,&H90000000,1,0,0,0,100,100,0,0,1,3,1,2,40,40,120,1"
|
||
|
||
|
||
def write_ass(timeline, path, styles=None, events=None):
|
||
st = styles if styles is not None else [SUB_STYLE]
|
||
head = ("[Script Info]\nScriptType: v4.00+\nPlayResX: 576\nPlayResY: 1024\n"
|
||
"ScaledBorderAndShadow: yes\nWrapStyle: 2\n\n[V4+ Styles]\n"
|
||
"Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding\n"
|
||
+ "\n".join(st) + "\n\n[Events]\n"
|
||
"Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n")
|
||
body = ""
|
||
for a, b, txt in timeline:
|
||
body += "Dialogue: 0,%s,%s,Default,,0,0,0,,{\\fad(150,150)}%s\n" % (ts(a), ts(b), wrap_line(txt))
|
||
for a, b, style, text, fade in (events or []):
|
||
fx = ("{\\fad(%d,%d)}" % (fade, fade)) if fade else ""
|
||
body += "Dialogue: 0,%s,%s,%s,,0,0,0,,%s%s\n" % (ts(a), ts(b), style, fx, text)
|
||
with open(path, "w", encoding="utf-8") as f:
|
||
f.write(head + body)
|
||
|
||
|
||
def build_timeline(data, root, cfg, vid_dir=None):
|
||
noise = cfg.get("silence", {}).get("noise", "-32dB")
|
||
dmin = str(cfg.get("silence", {}).get("d", "0.12"))
|
||
align = cfg.get("align", {})
|
||
vdir = vid_dir or os.path.join(root, "work", shots_ep(data), "videos", "native")
|
||
timeline, report, cursor = [], [], 0.0
|
||
for s in data["shots"]:
|
||
sid = s["shotId"]
|
||
src = os.path.join(vdir, sid + ".mp4")
|
||
prompt = find_prompt(root, data, sid)
|
||
if not os.path.exists(src):
|
||
print("WARN 缺视频 %s,跳过对齐" % src, file=sys.stderr)
|
||
cursor += float(s.get("duration", 0)); continue
|
||
d = probe(src)
|
||
lines = design_lines(prompt)
|
||
segs = voiced_segments(src, noise, dmin)
|
||
groups = align.get(sid)
|
||
if not lines:
|
||
report.append((sid, cursor, d, segs, lines, []))
|
||
cursor += d; continue
|
||
if groups is None:
|
||
raise SystemExit("ERROR %s 无 ALIGN 配置,请先 --dump-segs + --init-config 生成" % sid)
|
||
if len(groups) != len(lines):
|
||
raise SystemExit("ERROR %s: 台词 %d 句但 ALIGN %d 组" % (sid, len(lines), len(groups)))
|
||
max_idx = max(i for g in groups for i in g) if groups else -1
|
||
if max_idx >= len(segs):
|
||
raise SystemExit("ERROR %s: ALIGN 引用段 %d 但只检测到 %d 段(阈值 %s/%s)"
|
||
% (sid, max_idx, len(segs), noise, dmin))
|
||
|
||
assigned = []
|
||
for gi, g in enumerate(groups):
|
||
d0, d1, txt = lines[gi]
|
||
a, b = (segs[g[0]][0], segs[g[-1]][1]) if g else (0.0, 0.0)
|
||
assigned.append([a, b, txt, d1 - d0, str(g)])
|
||
# 同段被多句引用(连读)按设计时长比例切
|
||
used = {}
|
||
for i, (a, b, *_rest) in enumerate(assigned):
|
||
used.setdefault((a, b), []).append(i)
|
||
for (a, b), idxs in used.items():
|
||
if len(idxs) > 1:
|
||
total_dw = sum(assigned[i][3] for i in idxs)
|
||
p = a
|
||
for i in idxs:
|
||
q = p + (b - a) * (assigned[i][3] / total_dw)
|
||
assigned[i][0], assigned[i][1] = p, q
|
||
p = q
|
||
shot_tl = []
|
||
for a, b, txt, _dw, _g in assigned:
|
||
a = max(0.0, a - 0.04); b = min(d, b + 0.04)
|
||
if b - a < 0.45:
|
||
b = min(d, a + 0.45)
|
||
shot_tl.append((cursor + a, cursor + b, txt))
|
||
timeline.append((cursor + a, cursor + b, txt))
|
||
report.append((sid, cursor, d, segs, lines, shot_tl))
|
||
cursor += d
|
||
return timeline, report, cursor
|
||
|
||
|
||
def shots_ep(shots):
|
||
return shots.get("episode", "EP00")
|
||
|
||
|
||
def find_prompt(root, shots, sid):
|
||
ep = shots_ep(shots)
|
||
for rel in ("projects/%s/40_prompts/%s.h3v3.txt" % (ep, sid),
|
||
"40_prompts/%s.h3v3.txt" % sid):
|
||
p = os.path.join(root, rel)
|
||
if os.path.exists(p):
|
||
return p
|
||
return os.path.join(root, "projects", ep, "40_prompts", sid + ".h3v3.txt")
|
||
|
||
|
||
def print_report(report):
|
||
for sid, cursor, d, segs, lines, tl in report:
|
||
sdesc = " ".join("%d:%.2f-%.2f(%.2f)" % (i, a, b, b - a) for i, (a, b) in enumerate(segs))
|
||
print("---- %s @%.2f-%.2fs 段=%d 句=%d" % (sid, cursor, cursor + d, len(segs), len(lines)))
|
||
print(" 发声段: %s" % sdesc)
|
||
for (a, b, txt), (d0, d1, _) in zip(tl, lines):
|
||
print(" %.2f-%.2f %s (设计 %.2f-%.2f)" % (a, b, txt, d0, d1))
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
ap.add_argument("shots", help="30_shots.json 路径")
|
||
ap.add_argument("--root", help="工作空间根(默认上溯找 config/pipeline.json)")
|
||
ap.add_argument("--config", help="对齐配置 JSON(work/EPxx/align_config.json)")
|
||
ap.add_argument("--dump-segs", action="store_true", help="只打印每镜实测发声段,供填 ALIGN")
|
||
ap.add_argument("--init-config", metavar="PATH", help="生成配置模板(按句数顺序一一映射)")
|
||
ap.add_argument("--dry-run", action="store_true", help="只打印对齐时间轴,不写文件")
|
||
ap.add_argument("--out", help="输出对齐后的 ep.ass 路径")
|
||
ap.add_argument("--package-out", help="输出包装层 package.ass 路径(配置含 package 时)")
|
||
a = ap.parse_args()
|
||
|
||
root = find_root(a.root)
|
||
shots = json.load(open(os.path.abspath(a.shots), encoding="utf-8"))
|
||
cfg = json.load(open(os.path.abspath(a.config), encoding="utf-8")) if a.config else {}
|
||
ep = shots_ep(shots)
|
||
vdir = os.path.join(root, "work", ep, "videos", "native")
|
||
|
||
if a.dump_segs:
|
||
noise = cfg.get("silence", {}).get("noise", "-32dB")
|
||
dmin = str(cfg.get("silence", {}).get("d", "0.12"))
|
||
for s in shots["shots"]:
|
||
sid = s["shotId"]
|
||
src = os.path.join(vdir, sid + ".mp4")
|
||
if not os.path.exists(src):
|
||
print("%s 缺视频 %s" % (sid, src)); continue
|
||
segs = voiced_segments(src, noise, dmin)
|
||
lines = design_lines(find_prompt(root, shots, sid))
|
||
print("%s 段=%d 句=%d %s" % (
|
||
sid, len(segs), len(lines),
|
||
" ".join("%d:%.2f-%.2f" % (i, x, y) for i, (x, y) in enumerate(segs))))
|
||
return 0
|
||
|
||
if a.init_config:
|
||
tpl = {"_readme": "ALIGN 把发声段索引分给各句;先 --dump-segs 看段,再人工校准(默认一一映射仅供参考)。",
|
||
"silence": cfg.get("silence", {"noise": "-32dB", "d": "0.12"}),
|
||
"align": {}, "mute_shots": []}
|
||
noise = tpl["silence"]["noise"]; dmin = str(tpl["silence"]["d"])
|
||
for s in shots["shots"]:
|
||
sid = s["shotId"]
|
||
src = os.path.join(vdir, sid + ".mp4")
|
||
nlines = len(design_lines(find_prompt(root, shots, sid)))
|
||
tpl["align"][sid] = [[i] for i in range(nlines)] if nlines else []
|
||
os.makedirs(os.path.dirname(os.path.abspath(a.init_config)), exist_ok=True)
|
||
with open(a.init_config, "w", encoding="utf-8") as f:
|
||
json.dump(tpl, f, ensure_ascii=False, indent=2)
|
||
print("配置模板 -> %s(务必 --dump-segs 后人工校准)" % a.init_config)
|
||
return 0
|
||
|
||
timeline, report, total = build_timeline(shots, root, cfg, vdir)
|
||
print("字幕 %d 句,总长 %.2fs" % (len(timeline), total), flush=True)
|
||
print_report(report)
|
||
if a.dry_run:
|
||
print("\n[DRY-RUN] 未写文件。静音镜:%s" % (cfg.get("mute_shots") or "无"))
|
||
return 0
|
||
|
||
if a.out:
|
||
os.makedirs(os.path.dirname(os.path.abspath(a.out)), exist_ok=True)
|
||
write_ass(timeline, a.out)
|
||
print("对齐字幕 -> %s" % a.out)
|
||
if a.package_out and cfg.get("package"):
|
||
pkg = cfg["package"]
|
||
write_ass([], a.package_out, styles=pkg.get("styles"), events=pkg.get("events"))
|
||
print("包装层 -> %s" % a.package_out)
|
||
if not a.out and not a.package_out:
|
||
print("提示:加 --out / --package-out 才会写 ASS;当前仅预览。")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|