AI 漫剧流水线技能完整落地:文件驱动、可校验、单镜可重做。 七阶段:基线资产 → 策划叙事 → 分镜中枢 → 配音先行 → 出图/出视频 → 缝合 → 审片归档。 - scripts/: manju.py 编排、submit_t2i/submit_i2v 提交、cleanup_comfyui 清理 - assets/comfyui-manju-cleanup/: ComfyUI 清理扩展,归属校验 fail-closed - references/: models.md 模型节点图全表、workflow/director/storyboard/pitfalls - .gitignore: 排除 __pycache__ 与运行产物
312 lines
16 KiB
Python
312 lines
16 KiB
Python
#!/usr/bin/env python3
|
||
"""ai-manju 清理链路自检:扩展插件纯逻辑 + 客户端端到端 mock ComfyUI。
|
||
|
||
用法:python3 tools/selftest_cleanup.py
|
||
零第三方依赖;起一个本地 HTTPServer 模拟 ComfyUI 的标准接口与 manju-cleanup 扩展,
|
||
用 subprocess 跑 cleanup_comfyui.py 的 dry-run / apply,断言「只删自己发起的任务」。
|
||
"""
|
||
import importlib.util
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import threading
|
||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
CLEANUP = os.path.join(HERE, "cleanup_comfyui.py")
|
||
# 扩展插件本体在技能包的 assets/ 下;setup 复制脚本到工作空间 tools/ 时,
|
||
# assets/ 只有模板、没有扩展源码 —— 所以要往回找技能包,容错退化为跳过扩展用例。
|
||
PLUGIN_CANDIDATES = [
|
||
os.path.join(HERE, "..", "assets", "comfyui-manju-cleanup", "__init__.py"),
|
||
os.path.join(HERE, "..", "..", "assets", "comfyui-manju-cleanup", "__init__.py"),
|
||
os.path.join(HERE, "..", "..", "..", "assets", "comfyui-manju-cleanup", "__init__.py"),
|
||
]
|
||
PLUGIN = next((p for p in PLUGIN_CANDIDATES if os.path.isfile(p)), PLUGIN_CANDIDATES[0])
|
||
FAIL = []
|
||
|
||
|
||
def check(cond, msg):
|
||
if not cond:
|
||
FAIL.append(msg)
|
||
print(" [FAIL] " + msg)
|
||
else:
|
||
print(" [ok] " + msg)
|
||
|
||
|
||
def test_plugin():
|
||
print("\n== 扩展插件纯逻辑 ==")
|
||
spec = importlib.util.spec_from_file_location("manju_cleanup_node", PLUGIN)
|
||
m = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(m)
|
||
root = tempfile.mkdtemp(prefix="manju-selftest-")
|
||
out, inp = os.path.join(root, "output"), os.path.join(root, "input")
|
||
os.makedirs(os.path.join(out, "video"))
|
||
os.makedirs(inp)
|
||
roots = {"output": out, "input": inp, "temp": os.path.join(root, "temp")}
|
||
os.makedirs(roots["temp"])
|
||
paths = [os.path.join(out, "a.mp4"), os.path.join(out, "video", "b.mp4"),
|
||
os.path.join(inp, "head.png")]
|
||
for p in paths:
|
||
open(p, "wb").write(b"x" * 512)
|
||
|
||
f = lambda n, s="", t="output": {"filename": n, "subfolder": s, "type": t}
|
||
|
||
def item(cid, outs, imgs=()):
|
||
g = {str(i): {"class_type": "LoadImage", "inputs": {"image": n}}
|
||
for i, n in enumerate(imgs)}
|
||
return {"prompt": [1, "pid", g, {"client_id": cid}, {}],
|
||
"outputs": {"9": {"images": outs}}}
|
||
|
||
own = item("manju-i2v-ab12", [f("a.mp4"), f("b.mp4", "video"),
|
||
f("head.png", "", "input")], ["head.png"])
|
||
other = item("xv-someone", [f("a.mp4")])
|
||
hist = {"p1": own, "p2": other}
|
||
check(m.resolve(roots, f("../etc/passwd"))[1] == "path escape refused", "路径 .. 逃逸被拒")
|
||
check(m.resolve(roots, f("/etc/passwd"))[1] == "absolute path refused", "绝对路径被拒")
|
||
check(m.resolve(roots, f("b.mp4", "video"))[1] == "", "正常子目录可解析")
|
||
r = m.build_report(hist, roots, ["p1", "p2"], client_ids=["manju-i2v-ab12"], dry_run=True)
|
||
check(r["summary"]["shared"] == 1, "被别人引用的产物判 shared")
|
||
check(r["summary"]["skipped-input"] == 1, "purge_input 默认关闭")
|
||
check(r["summary"]["refused"] == 1, "未归属 pid 的文件被拒")
|
||
check(all(os.path.isfile(p) for p in paths), "dry-run 零副作用")
|
||
r = m.build_report(hist, roots, ["p1"], client_ids=["manju-i2v-ab12"], dry_run=False)
|
||
check(os.path.isfile(paths[0]) and not os.path.isfile(paths[1]), "只删独占产物")
|
||
check(os.path.isfile(paths[2]), "上传素材默认保留")
|
||
r = m.build_report({"p1": own}, roots, ["p1"], client_ids=["manju-i2v-ab12"],
|
||
dry_run=False, purge_input=True)
|
||
check(not os.path.isfile(paths[2]), "purge_input=true 才删输入")
|
||
check(not m.build_report(hist, roots, ["p1"], dry_run=False)["pids"]["p1"]["owned"],
|
||
"无归属凭证 fail-closed")
|
||
check(m.build_report(hist, roots, ["p1"], prefixes=["manju-"], dry_run=True)
|
||
["pids"]["p1"]["owned"], "前缀白名单可用")
|
||
check(not m.build_report({}, roots, ["nope"], client_ids=["x"], dry_run=False)
|
||
["pids"]["nope"]["exists"], "history 无记录时拒绝猜测文件名")
|
||
|
||
# 回归(2026-11-12):ComfyUI 0.45 内存态 history 布局 = [extra_data, number, prompt],
|
||
# client_id 在 prompt[0];HTTP 序列化布局 = [number, prompt_id, prompt, extra_data, [...]],
|
||
# client_id 在 prompt[3]。扩展两种布局都要能读,否则 0.45 上归属校验全失败、清不掉。
|
||
open(os.path.join(out, "mem.mp4"), "wb").write(b"x" * 512) # 专属文件,避免被前面用例删掉
|
||
item_mem = {"prompt": [{"client_id": "manju-i2v-mem01", "create_time": 1}, 0,
|
||
{"1": {"class_type": "LoadImage"}}],
|
||
"outputs": {"9": {"images": [{"filename": "mem.mp4", "type": "output"}]}}}
|
||
item_http = {"prompt": [0, "pid", {"1": {"class_type": "LoadImage"}},
|
||
{"client_id": "manju-i2v-http01"}, ["9"]],
|
||
"outputs": {"9": {"images": [{"filename": "http.mp4", "type": "output"}]}}}
|
||
check(m.client_id_of(item_mem) == "manju-i2v-mem01", "内存态[0]布局能读 client_id")
|
||
check(m.client_id_of(item_http) == "manju-i2v-http01", "HTTP态[3]布局能读 client_id")
|
||
r_mem = m.build_report({"m1": item_mem}, roots, ["m1"], prefixes=["manju-"], dry_run=True)
|
||
check(r_mem["pids"]["m1"]["owned"] is True, "内存态布局归属判定通过")
|
||
check(r_mem["pids"]["m1"]["files"][0]["status"] == "dry-run", "内存态布局产物进入预览")
|
||
r_reject = m.build_report({"m1": item_mem}, roots, ["m1"], prefixes=["other-"], dry_run=True)
|
||
check(r_reject["pids"]["m1"]["owned"] is False, "非 manju- 前缀仍拒绝")
|
||
|
||
|
||
class Server:
|
||
"""mock ComfyUI:官方 /api/* + manju-cleanup 扩展。"""
|
||
|
||
def __init__(self):
|
||
self.state = {"deleted": set(), "purge": [], "history_posts": [],
|
||
"free": 0, "node": True}
|
||
self.items = {
|
||
"own-pid": {"prompt": [1, "own-pid", {}, {"client_id": "manju-i2v-ab12"}, {}],
|
||
"outputs": {"9": {"images": [
|
||
{"filename": "own.mp4", "subfolder": "video", "type": "output"}]}}},
|
||
"other-pid": {"prompt": [1, "other-pid", {}, {"client_id": "xv-someone"}, {}],
|
||
"outputs": {"9": {"images": [
|
||
{"filename": "other.mp4", "subfolder": "video", "type": "output"}]}}},
|
||
}
|
||
h = self
|
||
|
||
class H(BaseHTTPRequestHandler):
|
||
def log_message(self, *a):
|
||
pass
|
||
|
||
def _send(self, code, obj=None):
|
||
body = b"" if obj is None else json.dumps(obj).encode()
|
||
self.send_response(code)
|
||
self.send_header("Content-Type", "application/json")
|
||
self.send_header("Content-Length", str(len(body)))
|
||
self.end_headers()
|
||
if body:
|
||
self.wfile.write(body)
|
||
|
||
def do_GET(self):
|
||
p = self.path
|
||
if p == "/api/system_stats":
|
||
return self._send(200, {"system": {"comfyui_version": "test"}})
|
||
if p == "/api/manju/cleanup/ping" and h.state["node"]:
|
||
return self._send(200, {"ok": True, "version": "1.0.0",
|
||
"roots": {"output": "/tmp/fake"}})
|
||
if p == "/api/queue":
|
||
return self._send(200, {"queue_running": [], "queue_pending": []})
|
||
if p.startswith("/api/history/"):
|
||
pid = p.rsplit("/", 1)[-1]
|
||
if pid in h.state["deleted"] or pid not in h.items:
|
||
return self._send(200, {})
|
||
return self._send(200, {pid: h.items[pid]})
|
||
return self._send(404, {})
|
||
|
||
def do_POST(self):
|
||
n = int(self.headers.get("Content-Length") or 0)
|
||
raw = self.rfile.read(n) if n else b"{}"
|
||
try:
|
||
body = json.loads(raw)
|
||
except ValueError:
|
||
body = {}
|
||
p = self.path
|
||
if p == "/api/manju/cleanup/purge" and h.state["node"]:
|
||
pids = body.get("pids") or []
|
||
allowed = set(body.get("client_ids") or [])
|
||
prefixes = body.get("owner_prefixes") or []
|
||
h.state["purge"].append({"dry_run": body.get("dry_run"),
|
||
"pids": list(pids)})
|
||
out = {}
|
||
for pid in pids:
|
||
it = h.items.get(pid, {})
|
||
cid = it.get("prompt", [None] * 4)[3].get("client_id", "")
|
||
owned = cid in allowed or any(cid.startswith(x) for x in prefixes)
|
||
rows = []
|
||
for fn in ("own.mp4", "other.mp4"):
|
||
rows.append({"type": "output", "subfolder": "video",
|
||
"filename": fn, "bytes": 100,
|
||
"status": "dry-run" if body.get("dry_run") else "deleted",
|
||
"reason": ""})
|
||
out[pid] = {"exists": bool(it), "client_id": cid, "owned": owned,
|
||
"owner_reason": "" if owned else "not in whitelist",
|
||
"files": rows[:1]}
|
||
return self._send(200, {"ok": True, "pids": out, "summary": {}})
|
||
if p == "/api/history":
|
||
h.state["history_posts"].append(list(body.get("delete") or []))
|
||
h.state["deleted"].update(body.get("delete") or [])
|
||
return self._send(200)
|
||
if p == "/api/queue":
|
||
return self._send(200)
|
||
if p == "/api/free":
|
||
h.state["free"] += 1
|
||
return self._send(200)
|
||
return self._send(404, {})
|
||
|
||
self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), H)
|
||
self.port = self.httpd.server_address[1]
|
||
self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
|
||
|
||
def __enter__(self):
|
||
self.thread.start()
|
||
return self
|
||
|
||
def __exit__(self, *a):
|
||
self.httpd.shutdown()
|
||
self.httpd.server_close()
|
||
|
||
|
||
def run_clean(root, *args):
|
||
env = dict(os.environ, MANJU_ROOT=root)
|
||
return subprocess.run([sys.executable, CLEANUP] + list(args),
|
||
env=env, capture_output=True, text=True, timeout=60)
|
||
|
||
|
||
def test_client():
|
||
print("\n== 客户端端到端(本地 mock ComfyUI)==")
|
||
root = tempfile.mkdtemp(prefix="manju-selftest-ws-")
|
||
os.makedirs(os.path.join(root, "config"))
|
||
vid = os.path.join(root, "work", "EP01", "videos")
|
||
os.makedirs(vid)
|
||
for name in ("S01.mp4", "S02.mp4"):
|
||
open(os.path.join(vid, name), "wb").write(b"x" * 2048)
|
||
with Server() as srv:
|
||
host = "http://127.0.0.1:%d" % srv.port
|
||
cfg = {"videoModel": {"endpoint": host},
|
||
"cleanup": {"minBytes": 100, "apiBase": "/api",
|
||
"ownerPrefixes": ["manju-"], "requireOwner": True}}
|
||
json.dump(cfg, open(os.path.join(root, "config", "pipeline.json"), "w"))
|
||
json.dump({"S01": {"out": "work/EP01/videos/S01.mp4", "promptId": "own-pid",
|
||
"clientId": "manju-i2v-ab12"},
|
||
"S02": {"out": "work/EP01/videos/S02.mp4", "promptId": "other-pid"}},
|
||
open(os.path.join(vid, "a14b_runs.json"), "w"))
|
||
|
||
dry = run_clean(root, "--ep", "EP01")
|
||
check(dry.returncode == 0, "dry-run 退出码 0:" + dry.stderr[:120])
|
||
check(not srv.state["deleted"] and not srv.state["history_posts"],
|
||
"dry-run 不删任何记录")
|
||
check(all(x["dry_run"] is True for x in srv.state["purge"]),
|
||
"dry-run 只发 purge(dry_run=true)")
|
||
check("⛔ 拒绝(非本人任务)" in dry.stdout, "非本人任务被明确标注拒绝")
|
||
check("✅ 自己的任务" in dry.stdout, "本人任务通过归属校验")
|
||
|
||
srv.state["purge"].clear()
|
||
app = run_clean(root, "--ep", "EP01", "--apply", "--free")
|
||
check(app.returncode == 0, "apply 退出码 0:" + app.stderr[:120])
|
||
check(srv.state["deleted"] == {"own-pid"},
|
||
"只删自己的 history 记录(实际 %s)" % sorted(srv.state["deleted"]))
|
||
check(srv.state["history_posts"] and srv.state["history_posts"][0] == ["own-pid"],
|
||
"POST /api/history 的 delete 列表不含别人的 pid")
|
||
real = [x for x in srv.state["purge"] if x["dry_run"] is False]
|
||
check(len(real) == 1 and real[0]["pids"] == ["own-pid"],
|
||
"purge 真删只对本人 pid 调用一次")
|
||
order = [("purge" if x["dry_run"] is False else "preview") for x in srv.state["purge"]]
|
||
check(order and order[-1] == "purge", "purge 在删记录之前(依赖 history 反查)")
|
||
check(srv.state["free"] == 1, "/free 整批只调一次")
|
||
check("已确认(删前有 → 删后无)" in app.stdout, "删后复核通过")
|
||
|
||
# 未装扩展:自动降级,不报错
|
||
srv.state["node"] = False
|
||
srv.state["deleted"].clear()
|
||
srv.state["history_posts"].clear()
|
||
srv.state["purge"].clear()
|
||
json.dump({"S01": {"out": "work/EP01/videos/S01.mp4", "promptId": "own-pid",
|
||
"clientId": "manju-i2v-ab12"}},
|
||
open(os.path.join(vid, "a14b_runs.json"), "w"))
|
||
deg = run_clean(root, "--ep", "EP01")
|
||
check(deg.returncode == 0, "未装扩展时降级不报错:" + deg.stderr[:120])
|
||
check("未安装 → 降级" in deg.stdout, "明确提示降级路线")
|
||
check(not srv.state["purge"], "未装扩展不发 purge 请求")
|
||
|
||
|
||
def test_anchor_ledger():
|
||
"""回归:裸 --out 出的锚图,台账写在图片同目录,清理必须能扫到。
|
||
|
||
2026-11-13 修的漏洞:ledger_paths() 只认固定路径 work/<ep>/images/t2i_runs.json,
|
||
而 submit_t2i.py 把台账写在 os.path.dirname(out) —— 锚图的台账落在
|
||
assets/characters/t2i_runs.json,于是锚图产物永远留在服务端。
|
||
"""
|
||
print("\n== 锚图台账扫描(回归) ==")
|
||
root = tempfile.mkdtemp(prefix="manju-selftest-anchor-")
|
||
os.makedirs(os.path.join(root, "config"))
|
||
os.makedirs(os.path.join(root, "assets", "characters"))
|
||
open(os.path.join(root, "assets", "characters", "ling.png"), "wb").write(b"x" * 2048)
|
||
host = "http://127.0.0.1:1" # 不真的连,只看扫没扫到
|
||
cfg = {"imageModel": {"endpoint": host},
|
||
"cleanup": {"minBytes": 100, "apiBase": "/api",
|
||
"ownerPrefixes": ["manju-"], "requireOwner": True}}
|
||
json.dump(cfg, open(os.path.join(root, "config", "pipeline.json"), "w"))
|
||
json.dump({"ling.png": {"out": "assets/characters/ling.png", "promptId": "anchor-pid",
|
||
"clientId": "manju-t2i-abcd1234", "endpoint": host}},
|
||
open(os.path.join(root, "assets", "characters", "t2i_runs.json"), "w"))
|
||
|
||
spec = importlib.util.spec_from_file_location("cc_anchor", CLEANUP)
|
||
m = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(m)
|
||
found = [p for _, p, _ in m.ledger_paths(root, "")]
|
||
check(any(p.endswith(os.path.join("assets", "characters", "t2i_runs.json")) for p in found),
|
||
"锚图台账(图片同目录)能被扫到")
|
||
|
||
jobs, skipped = m.gather(root, cfg, cfg.get("cleanup", cfg), "", set(), [])
|
||
check(any(j["pid"] == "anchor-pid" for j in jobs),
|
||
"无 --ep 时锚图任务进入待清列表")
|
||
check(any(j["pid"] == "anchor-pid" for j in jobs) and
|
||
all(j["host"].rstrip("/") == host.rstrip("/") for j in jobs if j["pid"] == "anchor-pid"),
|
||
"端点取台账里记录的实际端点,不靠配置猜")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
if os.path.isfile(PLUGIN):
|
||
test_plugin()
|
||
else:
|
||
print("\n== 扩展插件纯逻辑 ==\n [skip] 扩展源码不在技能包内(工作空间副本),只跑客户端用例")
|
||
test_client()
|
||
test_anchor_ledger()
|
||
print("\n%s (%d 项失败)" % ("ALL PASS" if not FAIL else "FAILED", len(FAIL)))
|
||
sys.exit(1 if FAIL else 0)
|