Files
ai-manju/scripts/cleanup_comfyui.py
admin 7753939a3b feat: ai-manju skill 初始提交
AI 漫剧流水线技能完整落地:文件驱动、可校验、单镜可重做。

七阶段:基线资产 → 策划叙事 → 分镜中枢 → 配音先行 → 出图/出视频 → 缝合 → 审片归档。

- scripts/: manju.py 编排、submit_t2i/submit_i2v 提交、cleanup_comfyui 清理
- assets/comfyui-manju-cleanup/: ComfyUI 清理扩展,归属校验 fail-closed
- references/: models.md 模型节点图全表、workflow/director/storyboard/pitfalls
- .gitignore: 排除 __pycache__ 与运行产物
2026-11-14 17:49:39 +08:00

884 lines
39 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""manju 生成后清理:只删「自己发起的任务」的 ComfyUI 记录与服务端产物。
归属模型(本版本核心):
* 本地台账里的 promptId 只可能是本技能提交的任务 —— 这是第一层凭证;
* ComfyUI 把 client_id 持久化在 history 项 prompt[3]["client_id"] 里,
提交脚本用可识别的 "manju-i2v-*" / "manju-t2i-*",并把 clientId 写进台账。
清理时按「台账 clientId 精确匹配 > ownerPrefixes 前缀白名单」判归属,
未通过一律拒绝,**不会调用任何删除接口**(fail-closed)。
* 服务端扩展再做一次独立归属校验,双保险。
标准接口(ComfyUI 0.38 / 0.45 实机核对 + 官方 server.py / docs.comfy.org):
* GET /api/system_stats 探测 /api 前缀是否可用
* GET /api/history/<pid> 抽产物清单 + client_id(顺序铁律)
* POST /api/history {"delete": [pid, ...]} 批量删生成记录
* POST /api/queue {"delete": [pid, ...]} 批量删队列残留
* POST /api/free {"unload_models": true, "free_memory": true} 释放模型/显存
* GET /api/history/<pid> + GET /api/queue 删后复核「从有到无」
官方**没有**删除 output/ 产物文件的 HTTP 路由(DELETE /userdata 只管 ComfyUI/user/,
/api/assets 删的是数据库引用且现网 503)。所以本技能自带扩展
assets/comfyui-manju-cleanup/,装到 ComfyUI/custom_nodes 后提供:
* GET /manju/cleanup/ping 能力探测(官方 add_routes 自动镜像 /api 前缀)
* POST /manju/cleanup/purge 按 pid 从服务端 history 反查产物并安全删除
顺序铁律(装了扩展时):
预览 purge(dry_run=true) → purge(dry_run=false) 删文件 → 官方 POST /history|/queue
删记录 → 复核 → 整批 /free。
purge 依赖 history 反查文件名,**必须先删文件再删记录**;反序会永远查不到产物。
没装扩展时自动降级到社区 DELETE 插件(deleteApi)或 cleanup.commands 文件系统模板;
此时顺序仍是「先抽清单 → 删记录 → 复核 → 删文件」。
安全闸门:本地导出的成片必须存在且 >= cleanup.minBytes 才允许清理;默认 dry-run,
必须 --apply 才真删。只按 prompt_id 精确删,禁用 {"clear": true}。本地资产永不删除。
用法:
python3 tools/cleanup_comfyui.py --ep EP01 # 预演
python3 tools/cleanup_comfyui.py --ep EP01 --apply # 真清
python3 tools/cleanup_comfyui.py --ep EP01 --shot S03 --apply # 只清一镜
python3 tools/cleanup_comfyui.py --prompt-id <pid> --endpoint http://... --apply
python3 tools/cleanup_comfyui.py --ep EP01 --apply --purge-input --free
"""
import argparse
import json
import os
import shlex
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
# K8s 三个端点的默认降级模板:动态取 pod 名,不重启 Pod 也能精确删产物。
# 与 assets/config.template.json 的 cleanup.commands 保持一致;工作空间配置可覆盖。
_K8S_RM_TMPL = (
"KUBECONFIG=$HOME/.kube/kubeconfig-wuguirong kubectl -n wuguirong exec "
"$(KUBECONFIG=$HOME/.kube/kubeconfig-wuguirong kubectl -n wuguirong get pods "
"-l app={app} -o jsonpath='{.items[0].metadata.name}') -c comfyui -- "
"sh -c \"cd {root} && rm -f -- {files} {temp_files}\"")
DEFAULT_CLEANUP = {
"auto": False, # 提交脚本导出成功后是否自动调用
"minBytes": 100000, # 本地成片小于此值视为导出失败,拒绝清理
"commands": { # {endpoint: "shell 模板"},占位符见 render_command
# 2026-11-13 起三个端口都指向同一个 pod(comfyui-wan-h3image,ComfyUI 0.45):
# 旧的 comfyui-wan-a14b / comfyui-wan-h3 deployment 已 scale 到 0,
# 它们的 Service selector 改指 app=comfyui-wan-h3image,NodePort 保持不变。
# 因此三条模板的 pod 选择器必须统一,否则会取到空 pod 名。
"http://192.168.1.30:30112": _K8S_RM_TMPL.replace("{app}", "comfyui-wan-h3image"),
"http://192.168.1.30:30111": _K8S_RM_TMPL.replace("{app}", "comfyui-wan-h3image"),
# H3-Image 文生图端点:未装 manju-cleanup 扩展,走 kubectl exec 降级
"http://192.168.1.30:30113": _K8S_RM_TMPL.replace("{app}", "comfyui-wan-h3image"),
},
"command": "", # 通用回退模板
"outputRoot": "/comfyui/output", # 服务端 output 根目录(K8s 是 PVC 软链)
"inputRoot": "/comfyui/input",
"purgeInput": False, # 上传的首帧图是否一起删(多镜可能共用,默认不删)
"free": False, # 整批清理结束后是否 POST /free(别每镜调用)
"apiBase": "/api", # 官方新版标准前缀;探测失败自动回退 ""
"deleteApi": "", # 社区自定义节点:comfyui_extra_api | api-tools | ""
"nodeApi": "", # 扩展路径前缀;留空自动探测 /api/manju/cleanup
"clientIds": [], # 额外 client_id 精确白名单(补老台账没记 clientId 的任务)
"ownerPrefixes": ["manju-"], # client_id 前缀白名单
"requireOwner": True, # 归属校验失败一律拒绝;改成 false 是危险的
"token": "", # 扩展设了 MANJU_CLEANUP_TOKEN 时填这里
}
# 社区自定义节点提供的标准 DELETE 路由(现网未安装,需显式配置才启用)
DELETE_APIS = {
"comfyui_extra_api": (
"/comfyapi/v1/output-images",
"/comfyapi/v1/output-images/{name}?temp=false"),
"api-tools": (
"/api-tools/v1/images/output",
"/api-tools/v1/images/output/{name}?temp=false"),
}
IMAGE_EXTS = (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp")
_API_BASE_CACHE = {}
_NODE_CACHE = {}
def find_root():
env = os.environ.get("MANJU_ROOT")
if env:
return os.path.abspath(os.path.expanduser(env))
d = os.getcwd()
while True:
if os.path.isfile(os.path.join(d, "config", "pipeline.json")):
return d
nd = os.path.dirname(d)
if nd == d:
break
d = nd
return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
def load_cfg(root):
p = os.path.join(root, "config", "pipeline.json")
raw = json.load(open(p, encoding="utf-8")) if os.path.isfile(p) else {}
c = dict(DEFAULT_CLEANUP)
c.update(raw.get("cleanup", {}))
return raw, c
def http(host, path, payload=None, timeout=30, method=None):
"""返回 (status, json|None)。HTTPError 不当异常抛,返回码交给调用方判断。
ComfyUI 的 delete / free / interrupt 路由返回 200 空 body;空 body 不解析。
"""
if method is None:
method = "POST" if payload is not None else "GET"
data, headers = None, {}
if payload is not None:
data = json.dumps(payload).encode()
headers["Content-Type"] = "application/json"
req = urllib.request.Request(host + path, data=data, headers=headers, method=method)
try:
with opener.open(req, timeout=timeout) as r:
body, status = r.read(), r.status
except urllib.error.HTTPError as e:
return e.code, None
if not body:
return status, None
try:
return status, json.loads(body)
except ValueError:
return status, None
def api_base(host, cfg):
"""探测 /api 前缀(官方新版标准路径),不可用则回退无前缀的旧路径。"""
want = str(cfg.get("apiBase", "/api")).rstrip("/")
key = (host, want)
if key in _API_BASE_CACHE:
return _API_BASE_CACHE[key]
pref = ""
if want:
try:
st, d = http(host, want + "/system_stats", timeout=10)
if st == 200 and isinstance(d, dict) and ("system" in d or "devices" in d):
pref = want
except Exception:
pass
_API_BASE_CACHE[key] = pref
return pref
def ap(host, cfg, path):
return api_base(host, cfg) + path
def rel_file(f):
"""history outputs 里的一条 -> (type, subfolder, filename)"""
return (f.get("type") or "output",
(f.get("subfolder") or "").strip("/\\"),
f.get("filename") or "")
def collect_outputs(item):
"""从 history 条目的 outputs 抽全部产物,含 SaveImage / SaveVideo 多输出节点。"""
found = []
for node in (item.get("outputs") or {}).values():
for key in ("images", "videos", "gifs", "audio"):
for f in (node.get(key) or []):
t, sub, name = rel_file(f)
if name:
found.append({"type": t, "subfolder": sub, "filename": name})
return found
def collect_inputs(item):
"""从 history 的 prompt 节点图里找 LoadImage 用到的上传图(input 目录)。"""
found = []
prompt = item.get("prompt") or []
graph = prompt[2] if len(prompt) > 2 and isinstance(prompt[2], dict) else {}
for node in graph.values():
if not isinstance(node, dict):
continue
ct = node.get("class_type", "")
if "LoadImage" not in ct and "ImageLoader" not in ct:
continue
name = (node.get("inputs") or {}).get("image")
if isinstance(name, str) and name:
sub, _, base = name.rpartition("/")
found.append({"type": "input", "subfolder": sub, "filename": base or name})
return found
def client_id_of(item):
"""ComfyUI 把 client_id 持久化在 history 项 prompt[3](HTTP 序列化态)或 prompt[0]
(内存态原始布局 [extra_data, number, prompt])。两种都兼容,再退化递归找。"""
p = item.get("prompt") if isinstance(item, dict) else None
if isinstance(p, (list, tuple)):
for idx in (3, 0):
if len(p) > idx and isinstance(p[idx], dict) and "client_id" in p[idx]:
return str(p[idx].get("client_id") or "")
r = _find_client_id(item)
return str(r) if r else ""
def _find_client_id(o):
if isinstance(o, dict):
if "client_id" in o:
return o["client_id"]
for v in o.values():
r = _find_client_id(v)
if r is not None:
return r
elif isinstance(o, (list, tuple)):
for v in o:
r = _find_client_id(v)
if r is not None:
return r
return None
def credentials_for(job, cfg):
"""本次任务的归属凭证:台账 clientId + 配置精确白名单 + 前缀白名单。"""
ids = [x for x in [job.get("clientId")] + list(cfg.get("clientIds") or []) if x]
return {"client_ids": list(dict.fromkeys(str(x) for x in ids)),
"prefixes": list(cfg.get("ownerPrefixes") or []),
"require_owner": bool(cfg.get("requireOwner", True)),
"token": str(cfg.get("token") or "")}
def owner_of(cid, creds):
"""(owned, reason)。没有任何凭证时 fail-closed,绝不因为「传了 pid」就放行。"""
if cid and cid in creds["client_ids"]:
return True, "台账 clientId 精确匹配"
if cid and any(cid.startswith(p) for p in creds["prefixes"] if p):
return True, "client_id 前缀白名单匹配"
if not creds["require_owner"]:
return True, "requireOwner=false(显式关闭归属校验,不推荐)"
return False, ("client_id=%r 不在白名单:台账无 clientId、clientIds 未列、"
"ownerPrefixes 不匹配" % cid)
def node_api(host, cfg):
"""探测 manju-cleanup 扩展,返回可用路径前缀('' 表示未安装)。"""
key = (host, str(cfg.get("nodeApi") or ""))
if key in _NODE_CACHE:
return _NODE_CACHE[key]
custom = str(cfg.get("nodeApi") or "").rstrip("/")
cands = [custom] if custom else [ap(host, cfg, "/manju/cleanup"),
"/manju/cleanup"]
for p in dict.fromkeys([x for x in cands if x]):
try:
st, d = http(host, p + "/ping", timeout=8)
if st == 200 and isinstance(d, dict) and d.get("ok"):
_NODE_CACHE[key] = p
return p
except Exception:
pass
_NODE_CACHE[key] = ""
return ""
def node_purge(host, cfg, prefix, pids, creds, dry_run, purge_input):
"""调扩展的 purge;服务端按 pid 自己从 history 反查文件并二次校验归属。"""
body = {"pids": list(pids),
"client_ids": creds["client_ids"],
"owner_prefixes": creds["prefixes"],
"require_owner": creds["require_owner"],
"dry_run": bool(dry_run), "purge_input": bool(purge_input)}
if creds["token"]:
body["token"] = creds["token"]
try:
st, d = http(host, prefix + "/purge", body, timeout=90)
except Exception as e:
return {"ok": False, "error": "请求失败:%s" % e, "pids": {}}
if st != 200 or not isinstance(d, dict):
return {"ok": False, "error": "HTTP %d" % st, "pids": {}}
return d
def node_files_text(entry):
"""把扩展返回的单个 pid 报告压成一行人话。"""
if not entry:
return "扩展未返回该 pid"
if not entry.get("exists"):
return "拒绝:history 已无该条,服务端不猜测文件名"
if not entry.get("owned"):
return "拒绝:扩展归属校验未通过(%s)" % entry.get("owner_reason", "")
counts = {}
for f in entry.get("files") or []:
counts[f["status"]] = counts.get(f["status"], 0) + 1
return "扩展 purge:%s" % (", ".join("%s=%d" % kv for kv in sorted(counts.items()))
or "无文件")
def inspect_pid(pid, host, cfg, creds, purge_input=False):
"""阶段 A:先从 history 抽产物清单与 client_id,删了记录就再也拿不到。
扩展的 dry_run 预览由调用方按 host 批量执行(node_purge),不在这里发请求。
"""
rep = {"pid": pid, "host": host, "outputs": [], "inputs": [],
"history": "?", "queue": "?", "verify": "?", "files": "?",
"commands": [], "had": False, "client_id": "", "owned": False,
"owner_reason": "", "creds": creds, "node": "", "node_entry": {}}
rep["node"] = node_api(host, cfg)
try:
st, item = http(host, ap(host, cfg, "/history/" + urllib.parse.quote(pid)), timeout=30)
d = item or {}
if st != 200:
rep["verify"] = "history 读取 HTTP %d,不做文件清理" % st
return rep
entry = d.get(pid, d) if isinstance(d, dict) else {}
rep["had"] = bool(d) and (pid in d if isinstance(d, dict) else True)
all_out = collect_outputs(entry) if isinstance(entry, dict) else []
# 有些节点(LoadVideo / 预览类)会在 outputs 里回带 type=input 的源素材。
rep["outputs"] = [f for f in all_out if f["type"] in ("output", "temp")]
extra = [f for f in all_out if f["type"] == "input"]
seen = {(f["type"], f["subfolder"], f["filename"]) for f in extra}
rep["inputs"] = extra + [f for f in collect_inputs(entry)
if (f["type"], f["subfolder"], f["filename"]) not in seen]
rep["client_id"] = client_id_of(entry)
except Exception as e:
rep["verify"] = "history 读取失败:%s" % e
rep["owned"], rep["owner_reason"] = owner_of(rep["client_id"], creds)
if not rep["had"]:
rep["note"] = "history 已无该条(可能重启/被清过),不猜测文件名"
return rep
def merge_node_preview(rep, resp):
"""把服务端 dry_run 预览的归属结论并进客户端报告(服务端优先否决)。"""
if not (isinstance(resp, dict) and resp.get("ok")):
rep["node_error"] = (resp or {}).get("error", "预览失败")
return
e = (resp.get("pids") or {}).get(rep["pid"])
rep["node_entry"] = e or {}
if e and not e.get("owned"):
rep["owned"] = False
rep["owner_reason"] = "扩展否决:%s" % e.get("owner_reason", "")
def delete_records(host, pids, cfg, had_map):
"""按 host 批量调标准接口删记录,再逐 pid 复核「从有到无」。
200 空 body 只代表请求被接受且幂等完成,不代表记录真的消失。
"""
out = {pid: {} for pid in pids}
for path, key in (("/history", "history"), ("/queue", "queue")):
try:
st, _ = http(host, ap(host, cfg, path), {"delete": list(pids)}, timeout=30)
ok = st == 200
for pid in pids:
out[pid][key] = ("HTTP 200(空 body)" if ok else "FAILED HTTP %d" % st)
except Exception as e:
for pid in pids:
out[pid][key] = "FAILED: %s" % e
for pid in pids:
try:
st, d = http(host, ap(host, cfg, "/history/" + urllib.parse.quote(pid)), timeout=20)
still = bool(d) and (pid in d if isinstance(d, dict) else True)
if st == 200 and not still and had_map.get(pid):
out[pid]["verify"] = "已确认(删前有 → 删后无)"
elif not had_map.get(pid):
out[pid]["verify"] = "幂等 200,但删前就无此记录,不宣称删除成功"
elif still:
out[pid]["verify"] = "FAILED:复核时记录仍在"
else:
out[pid]["verify"] = "未确认(HTTP %d)" % st
except Exception as e:
out[pid]["verify"] = "复核失败:%s" % e
try:
st, q = http(host, ap(host, cfg, "/queue"), timeout=20)
if st == 200 and isinstance(q, dict):
live = set()
for arr in ("queue_running", "queue_pending"):
for x in q.get(arr) or []:
if isinstance(x, list) and len(x) > 1:
live.add(str(x[1]))
for pid in pids:
out[pid]["queue_verify"] = "queue 已无该 pid" if pid not in live else \
"FAILED:queue 里仍有该 pid"
except Exception:
pass
return out
def render_command(template, files, root, input_files, input_root):
"""把模板里的占位符替换成 shell-safe 的路径列表。"""
def q(fs):
return " ".join(shlex.quote(os.path.join(f["subfolder"], f["filename"]))
for f in fs) if fs else ""
return (template
.replace("{files}", q([f for f in files if f["type"] == "output"]))
.replace("{temp_files}", q([f for f in files if f["type"] == "temp"]))
.replace("{input_files}", q(input_files))
.replace("{root}", shlex.quote(root) if root else ".")
.replace("{input_root}", shlex.quote(input_root) if input_root else "."))
def pick_command(cfg, host):
return (cfg.get("commands") or {}).get(host.rstrip("/")) or cfg.get("command") or ""
def try_http_delete(host, cfg, files):
"""社区方案:若端点装了自定义节点,用标准 HTTP DELETE 删 output 图片。"""
name = str(cfg.get("deleteApi") or "").strip()
if not name or name not in DELETE_APIS:
return [], files, ""
list_path, del_tpl = DELETE_APIS[name]
try:
st, d = http(host, list_path, timeout=15)
if st != 200 or not isinstance(d, (list, dict)):
return [], files, "deleteApi=%s 未就绪(HTTP %d),回退文件模板" % (name, st)
except Exception as e:
return [], files, "deleteApi=%s 探测失败:%s" % (name, e)
deleted, remaining, fails = [], [], []
for f in files:
path = os.path.join(f["subfolder"], f["filename"])
eligible = (f["type"] == "output" and not f["subfolder"]
and f["filename"].lower().endswith(IMAGE_EXTS))
if not eligible:
remaining.append(f)
continue
url = del_tpl.replace("{name}", urllib.parse.quote(f["filename"]))
try:
st, _ = http(host, url, timeout=20, method="DELETE")
(deleted if st in (200, 204) else remaining).append(f)
if st not in (200, 204):
fails.append("%s → HTTP %d" % (path, st))
except Exception as e:
remaining.append(f)
fails.append("%s → %s" % (path, e))
note = "标准 DELETE 删掉 %d 个%s" % (len(deleted), (";失败:" + ", ".join(fails[:3])) if fails else "")
return deleted, remaining, note
def delete_files(host, cfg, rep, apply, purge_input=False):
"""降级路线:优先社区 DELETE,其余走服务端文件系统模板。
dry-run 绝不发任何 DELETE / 任何 shell 命令,只列清单与将要执行的命令。
"""
files = rep["outputs"] + (rep["inputs"] if purge_input else [])
inputs = rep["inputs"] if purge_input else []
if not files:
rep["files"] = "无产物文件需删"
return rep
tpl = pick_command(cfg, host)
cmd = render_command(tpl, files, cfg.get("outputRoot", ""),
inputs, cfg.get("inputRoot", "")) if tpl else ""
if not apply:
parts = ["dry-run 不删(%d 个文件)" % len(files)]
name = str(cfg.get("deleteApi") or "").strip()
if name in DELETE_APIS:
st, d = http(host, DELETE_APIS[name][0], timeout=15)
n_img = sum(1 for f in files if f["type"] == "output" and not f["subfolder"]
and f["filename"].lower().endswith(IMAGE_EXTS))
parts.append("deleteApi=%s %s,%d 个图片将走标准 DELETE"
% (name, "可用" if st == 200 and isinstance(d, (list, dict)) else "未就绪",
n_img))
if cmd:
rep["commands"].append(cmd)
else:
parts.append("未配 cleanup.commands[%s],服务端文件只能手工清" % host)
rep["pending"] = files
rep["files"] = " | ".join(parts)
return rep
deleted, remaining, note = try_http_delete(host, cfg, files)
parts = [note] if note else []
if remaining:
if not tpl:
parts.append("未配 cleanup.commands[%s],%d 个文件未删(见清单)"
% (host, len(remaining)))
rep["pending"] = remaining
else:
cmd = render_command(tpl, remaining, cfg.get("outputRoot", ""),
inputs, cfg.get("inputRoot", ""))
rep["commands"].append(cmd)
try:
p = subprocess.run(cmd, shell=True, capture_output=True,
text=True, timeout=120)
parts.append("模板 %s (rc=%d)" % ("ok" if p.returncode == 0 else "FAILED",
p.returncode))
if p.returncode != 0:
parts.append(p.stderr.strip()[:200])
except Exception as e:
parts.append("模板 FAILED: %s" % e)
if not parts:
parts.append("全部走标准 DELETE")
rep["files"] = " | ".join(parts)
return rep
def free_models(host, cfg):
"""标准接口释放模型/显存。它只是设置队列 flag,真正卸载发生在下一个队列循环。"""
try:
st, _ = http(host, ap(host, cfg, "/free"),
{"unload_models": True, "free_memory": True}, timeout=30)
return "HTTP %d(空 body,异步生效)" % st if st == 200 else "FAILED HTTP %d" % st
except Exception as e:
return "FAILED: %s" % e
LEDGER_NAMES = ("t2i_runs.json", "a14b_runs.json")
def ledger_paths(root, ep):
"""列出该扫描范围内的台账(去重)。
submit_t2i.py 把台账写在 **os.path.dirname(out)**,所以台账位置随输出路径变:
--ep/--shot 出图 → work/<EP>/images/t2i_runs.json
裸 --out 出锚图 → assets/characters/t2i_runs.json(跟图片同目录!)
因此不能只猜固定路径 —— 漏扫 = 锚图产物永远留在服务端。
这里直接递归扫工作空间下所有同名台账,--ep 时额外限定只取该集内的。
"""
found, seen = [], set()
def add(label, path):
rp = os.path.realpath(path)
if rp in seen:
return
seen.add(rp)
found.append((label, path, "videoModel" if label == "a14b_runs.json" else "imageModel"))
for base, dirs, files in os.walk(root):
dirs[:] = [d for d in dirs if d not in (".git", "__pycache__", "node_modules")]
for fn in files:
if fn not in LEDGER_NAMES:
continue
path = os.path.join(base, fn)
rel = os.path.relpath(path, root)
if ep:
# 带 --ep 时:只要台账路径里含该集号,或台账在集外(锚图目录)
in_ep = (os.sep + ep + os.sep) in (os.sep + rel)
outside_ep = not any(
part.startswith("EP") and os.path.isdir(os.path.join(root, "work", part))
for part in rel.split(os.sep)
)
if not in_ep and not outside_ep:
continue
add(fn, path)
return found
def gather(root, raw, cfg, ep, shots, prompt_ids):
"""收集待清理任务,逐条过导出闸门。返回 (jobs, skipped)。
prompt_ids 元素兼容 6 元组与 7 元组(末尾多一个 clientId)。
"""
jobs, skipped = [], []
for p in prompt_ids:
pid, host, key, ledger, ledger_key, local = p[:6]
cid = p[6] if len(p) > 6 else ""
jobs.append({"pid": pid, "host": host, "key": key, "ledger": ledger,
"ledgerKey": ledger_key, "local": local, "clientId": cid,
"reason": "--prompt-id 显式指定,跳过导出闸门"})
# 扫描台账:无 --ep 时扫工作空间根的台账(裸 --out 的锚图),
# 有 --ep 时扫该集台账 + 根级台账(锚图不属于任何集)。
for label, path, cfg_key in ledger_paths(root, ep):
if not os.path.isfile(path):
continue
runs = json.load(open(path, encoding="utf-8"))
host = str(raw.get(cfg_key, {}).get("endpoint", "")).rstrip("/")
for key, rec in runs.items():
if shots and key not in shots and key.split("/")[-1] not in shots:
continue
pid = rec.get("promptId")
if not pid:
continue # 已清过,跳过
# 台账里的 endpoint 是该任务**实际**提交到的端点,优先于配置里的全局值
# (锚图与首帧可能不在同一个端点,别清错地方)
task_host = str(rec.get("endpoint") or host).rstrip("/")
local = os.path.join(root, rec.get("out", ""))
ok = os.path.isfile(local) and os.path.getsize(local) >= int(cfg["minBytes"])
j = {"pid": pid, "host": task_host, "key": key, "ledger": path,
"ledgerKey": key, "local": local,
"clientId": rec.get("clientId", "")}
if ok:
jobs.append(j)
else:
j["reason"] = "本地导出文件缺失或过小:%s" % local
skipped.append(j)
seen, uniq = set(), []
for j in jobs:
if j["pid"] in seen:
continue
seen.add(j["pid"])
uniq.append(j)
return uniq, skipped
def update_ledger(jobs, skipped, apply, purge_ledger):
"""默认移除服务端可关联的 promptId 并留清理时间;--purge-ledger 则删整条。"""
if not apply:
return 0
touched = set()
for j in jobs:
p = j.get("ledger")
if not p or not os.path.isfile(p) or p in touched:
continue
touched.add(p)
runs = json.load(open(p, encoding="utf-8"))
for job in jobs:
if job.get("ledger") != p:
continue
k = job.get("ledgerKey")
if k not in runs:
continue
if purge_ledger:
runs.pop(k, None)
else:
runs[k].pop("promptId", None)
runs[k]["cleanedAt"] = time.strftime("%Y-%m-%d %H:%M:%S")
with open(p, "w", encoding="utf-8") as f:
json.dump(runs, f, ensure_ascii=False, indent=2)
return len(touched)
def group_by(pairs, key):
g = {}
for item in pairs:
g.setdefault(key(item), []).append(item)
return g
def cleanup_pid(pid, host, apply, cfg, creds=None, purge_input=False):
"""单 pid 完整清理,供提交脚本的 after_export 钩子调用。"""
creds = creds or credentials_for({}, cfg)
rep = inspect_pid(pid, host, cfg, creds, purge_input)
prefix = rep["node"]
if prefix:
prev = node_purge(host, cfg, prefix, [pid], creds, True, purge_input)
merge_node_preview(rep, prev)
if not rep["owned"]:
rep["history"] = rep["queue"] = "拒绝:归属校验未通过"
rep["files"] = "拒绝:不删任何服务端文件"
return rep
if not apply:
rep["history"] = "dry-run 不删"
rep["queue"] = "dry-run 不删"
if prefix:
rep["files"] = node_files_text(rep["node_entry"])
return rep
return delete_files(host, cfg, rep, False, purge_input)
if prefix:
res = node_purge(host, cfg, prefix, [pid], creds, False, purge_input)
rep["node_entry"] = (res.get("pids") or {}).get(pid, {})
rep["files"] = node_files_text(rep["node_entry"])
if not res.get("ok"):
rep["files"] += " | purge HTTP 异常:%s" % res.get("error", "")
res = delete_records(host, [pid], cfg, {pid: rep["had"]})[pid]
rep["history"] = "%s | %s" % (res["history"], res["verify"])
rep["queue"] = res["queue"] + (" | " + res["queue_verify"] if res.get("queue_verify") else "")
if not prefix:
rep = delete_files(host, cfg, rep, True, purge_input)
return rep
def after_export(root, pid, host, ledger, ledger_key, local, client_id=""):
"""提交脚本钩子:成片已落本地后,清掉服务端记录与产物。
调用方保证下载成功。这里再独立过一次导出闸门 + 归属闸门,任何异常都只影响
清理、不影响已成片的产物。整批做完再单独 --free,不要每镜卸载模型。
"""
raw, cfg = load_cfg(root)
host = host.rstrip("/")
if not os.path.isfile(local) or os.path.getsize(local) < int(cfg["minBytes"]):
return "SKIP 本地成片缺失/过小,未清理:%s" % local
if not client_id and ledger and os.path.isfile(ledger):
try:
client_id = (json.load(open(ledger, encoding="utf-8"))
.get(ledger_key, {}).get("clientId", ""))
except Exception:
pass
creds = credentials_for({"clientId": client_id}, cfg)
rep = cleanup_pid(pid, host, True, cfg, creds, bool(cfg.get("purgeInput")))
if not rep["owned"]:
return "REFUSE 归属校验未通过(client_id=%s):未删任何记录/文件" % rep["client_id"]
update_ledger([{"ledger": ledger, "ledgerKey": ledger_key, "pid": pid}],
[], True, bool(cfg.get("purgeLedger")))
tail = ""
if "未删" in rep["files"] or "未配" in rep["files"] or "拒绝" in rep["files"]:
tail = " | ⚠ 仍有服务端文件待清"
return ("history=%s queue=%s files=%s 产物%d个 归属=%s%s"
% (rep["history"], rep["queue"], rep["files"], len(rep["outputs"]),
rep["client_id"] or "-", tail))
def print_report(j, rep, cfg):
print("\n[%s] %s @ %s" % (j["key"], j["pid"], j["host"]))
print(" 归属 : %s | client_id=%s | %s" %
("✅ 自己的任务" if rep["owned"] else "⛔ 拒绝(非本人任务)",
rep["client_id"] or "-", rep["owner_reason"]))
print(" 扩展 : %s" % (rep["node"] + "/purge" if rep["node"] else
"未安装 → 降级 deleteApi / commands"))
print(" history : %s" % rep["history"])
print(" queue : %s" % rep["queue"])
print(" 产物 : %d 个文件" % len(rep["outputs"]))
for f in rep["outputs"]:
print(" - [%s] %s" % (f["type"], os.path.join(f["subfolder"], f["filename"])))
if rep["node_entry"]:
for f in rep["node_entry"].get("files") or []:
print(" · %s → %s%s" % (os.path.join(f["subfolder"], f["filename"]),
f["status"],
("(" + f["reason"] + ")") if f.get("reason") else ""))
if rep["inputs"]:
print(" 上传图 : %d 个%s" % (len(rep["inputs"]),
"(--purge-input 未开,列出不删)" if not cfg.get("purgeInput") else ""))
print(" 文件删除: %s" % rep["files"])
for c in rep["commands"]:
print(" 命令 : %s" % c)
if rep.get("node_error"):
print(" 扩展错误: %s" % rep["node_error"])
if rep.get("note"):
print(" 备注 : %s" % rep["note"])
def main():
ap_ = argparse.ArgumentParser(description="只清自己发起的任务:ComfyUI 记录 + 服务端产物")
ap_.add_argument("--ep", default="", help="集号,如 EP01;读该集两个台账")
ap_.add_argument("--shot", default="", help="只清指定镜,逗号分隔")
ap_.add_argument("--prompt-id", default="", help="直接指定 prompt_id,逗号分隔")
ap_.add_argument("--endpoint", default="", help="--prompt-id 对应的服务地址")
ap_.add_argument("--client-id", default="", help="--prompt-id 对应任务的 client_id(归属凭证)")
ap_.add_argument("--apply", action="store_true", help="真的执行;默认只预演")
ap_.add_argument("--purge-input", action="store_true", help="连上传的首帧图一起删")
ap_.add_argument("--purge-ledger", action="store_true", help="删掉本地台账整条记录")
ap_.add_argument("--free", action="store_true", help="整批结束后 POST /free 释放模型")
ap_.add_argument("--force", action="store_true", help="跳过本地导出文件闸门(危险)")
a = ap_.parse_args()
root = find_root()
raw, cfg = load_cfg(root)
if a.force:
cfg["minBytes"] = 0
if a.purge_input:
cfg["purgeInput"] = True
shots = set(x.strip() for x in a.shot.split(",") if x.strip())
explicit = []
if a.prompt_id:
host = (a.endpoint or raw.get("videoModel", {}).get("endpoint", "")).rstrip("/")
if not host:
return "用 --prompt-id 时必须给 --endpoint 或在 config 里配 videoModel.endpoint"
for pid in [x.strip() for x in a.prompt_id.split(",") if x.strip()]:
explicit.append((pid, host, pid, "", pid, "", a.client_id))
jobs, skipped = gather(root, raw, cfg, a.ep, shots, explicit)
if not jobs and not skipped:
return "没有待清理的 prompt(台账里无 promptId,或都已被清过)"
print("模式:%s | 归属校验:%s" % (
"APPLY 真删" if a.apply else "DRY-RUN 预演",
"开启(fail-closed)" if cfg.get("requireOwner", True) else "⚠ 已关闭"))
print("待清 %d 条,导出闸门拦下 %d 条" % (len(jobs), len(skipped)))
for s in skipped:
print(" [SKIP] %s %s" % (s["key"], s["reason"]))
# 阶段 A:抽清单 + 客户端归属判定 + 探测扩展
reports = []
for j in jobs:
creds = credentials_for(j, cfg)
rep = inspect_pid(j["pid"], j["host"], cfg, creds, bool(cfg.get("purgeInput")))
reports.append((j, rep, creds))
# 阶段 A2:扩展 dry_run 预览(按 host 批量),服务端归属结论优先否决
for host, triples in group_by(reports, lambda t: t[1]["host"]).items():
with_node = [t for t in triples if t[1]["node"]]
if not with_node:
continue
merged = {"client_ids": [], "prefixes": [], "require_owner": True, "token": ""}
for _, _, c in with_node:
merged["client_ids"] += c["client_ids"]
merged["prefixes"] += c["prefixes"]
merged["token"] = merged["token"] or c["token"]
merged["client_ids"] = list(dict.fromkeys(merged["client_ids"]))
merged["prefixes"] = list(dict.fromkeys(merged["prefixes"]))
prev = node_purge(host, cfg, with_node[0][1]["node"],
[t[1]["pid"] for t in with_node], merged, True,
bool(cfg.get("purgeInput")))
for _, rep, _ in with_node:
merge_node_preview(rep, prev)
# 归属闸门:不归属的 pid 绝不进入任何删除操作
eligible = [t for t in reports if t[1]["owned"]]
for _, rep, _ in reports:
if not rep["owned"]:
rep["history"] = rep["queue"] = "拒绝:归属校验未通过"
rep["files"] = "拒绝:不删任何服务端文件"
rep["commands"] = []
# 阶段 B:扩展真实删文件(必须在删记录之前:purge 依赖 history 反查)
for host, triples in group_by(eligible, lambda t: t[1]["host"]).items():
with_node = [t for t in triples if t[1]["node"]]
if not (a.apply and with_node):
continue
merged = {"client_ids": [], "prefixes": [], "require_owner": True, "token": ""}
for _, _, c in with_node:
merged["client_ids"] += c["client_ids"]
merged["prefixes"] += c["prefixes"]
merged["token"] = merged["token"] or c["token"]
merged["client_ids"] = list(dict.fromkeys(merged["client_ids"]))
merged["prefixes"] = list(dict.fromkeys(merged["prefixes"]))
res = node_purge(host, cfg, with_node[0][1]["node"],
[t[1]["pid"] for t in with_node], merged, False,
bool(cfg.get("purgeInput")))
for _, rep, _ in with_node:
rep["node_entry"] = (res.get("pids") or {}).get(rep["pid"], {})
rep["files"] = node_files_text(rep["node_entry"])
if not res.get("ok"):
rep["files"] += " | purge 异常:%s" % res.get("error", "")
# 阶段 C:官方标准接口批量删记录 + 复核(只传 eligible 的 pid)
if a.apply:
for host, triples in group_by(eligible, lambda t: t[1]["host"]).items():
pids = [t[1]["pid"] for t in triples]
res = delete_records(host, pids, cfg,
{t[1]["pid"]: t[1]["had"] for t in triples})
for _, rep, _ in triples:
r = res[rep["pid"]]
rep["history"] = "%s | %s" % (r["history"], r["verify"])
rep["queue"] = r["queue"] + (" | " + r["queue_verify"]
if r.get("queue_verify") else "")
else:
for _, rep, _ in eligible:
rep["history"] = "dry-run 不删"
rep["queue"] = "dry-run 不删"
# 阶段 D:未装扩展的走降级路线(社区 DELETE / 文件系统模板)
for _, rep, _ in eligible:
if rep["node"]:
if not a.apply:
rep["files"] = node_files_text(rep["node_entry"])
else:
delete_files(rep["host"], cfg, rep, a.apply, bool(cfg.get("purgeInput")))
for j, rep, _ in reports:
print_report(j, rep, cfg)
if a.apply:
n = update_ledger([j for j, rep, _ in eligible if rep["owned"]],
skipped, True, a.purge_ledger)
print("\n台账更新:%d 个文件(%s)" % (
n, "整条删除" if a.purge_ledger else "移除 promptId + 记 cleanedAt"))
if a.free or cfg.get("free"):
for host in sorted(set(j["host"] for j, _, _ in eligible)):
print(" /free %s -> %s" % (host, free_models(host, cfg)))
else:
print("\n这是预演,未删除任何东西。确认清单后加 --apply。")
return 0
if __name__ == "__main__":
try:
sys.exit(main())
except Exception as e:
print("ERROR: %s" % e, file=sys.stderr, flush=True)
sys.exit(1)