AI 漫剧流水线技能完整落地:文件驱动、可校验、单镜可重做。 七阶段:基线资产 → 策划叙事 → 分镜中枢 → 配音先行 → 出图/出视频 → 缝合 → 审片归档。 - scripts/: manju.py 编排、submit_t2i/submit_i2v 提交、cleanup_comfyui 清理 - assets/comfyui-manju-cleanup/: ComfyUI 清理扩展,归属校验 fail-closed - references/: models.md 模型节点图全表、workflow/director/storyboard/pitfalls - .gitignore: 排除 __pycache__ 与运行产物
884 lines
39 KiB
Python
884 lines
39 KiB
Python
#!/usr/bin/env python3
|
||
"""manju 生成后清理:只删「自己发起的任务」的 ComfyUI 记录与服务端产物。
|
||
|
||
归属模型(本版本核心):
|
||
* 本地台账里的 promptId 只可能是本技能提交的任务 —— 这是第一层凭证;
|
||
* ComfyUI 把 client_id 持久化在 history 项 prompt[3]["client_id"] 里,
|
||
提交脚本用可识别的 "manju-i2v-*" / "manju-t2i-*",并把 clientId 写进台账。
|
||
清理时按「台账 clientId 精确匹配 > ownerPrefixes 前缀白名单」判归属,
|
||
未通过一律拒绝,**不会调用任何删除接口**(fail-closed)。
|
||
* 服务端扩展再做一次独立归属校验,双保险。
|
||
|
||
标准接口(ComfyUI 0.38 / 0.45 实机核对 + 官方 server.py / docs.comfy.org):
|
||
* GET /api/system_stats 探测 /api 前缀是否可用
|
||
* GET /api/history/<pid> 抽产物清单 + client_id(顺序铁律)
|
||
* POST /api/history {"delete": [pid, ...]} 批量删生成记录
|
||
* POST /api/queue {"delete": [pid, ...]} 批量删队列残留
|
||
* POST /api/free {"unload_models": true, "free_memory": true} 释放模型/显存
|
||
* GET /api/history/<pid> + GET /api/queue 删后复核「从有到无」
|
||
|
||
官方**没有**删除 output/ 产物文件的 HTTP 路由(DELETE /userdata 只管 ComfyUI/user/,
|
||
/api/assets 删的是数据库引用且现网 503)。所以本技能自带扩展
|
||
assets/comfyui-manju-cleanup/,装到 ComfyUI/custom_nodes 后提供:
|
||
* GET /manju/cleanup/ping 能力探测(官方 add_routes 自动镜像 /api 前缀)
|
||
* POST /manju/cleanup/purge 按 pid 从服务端 history 反查产物并安全删除
|
||
|
||
顺序铁律(装了扩展时):
|
||
预览 purge(dry_run=true) → purge(dry_run=false) 删文件 → 官方 POST /history|/queue
|
||
删记录 → 复核 → 整批 /free。
|
||
purge 依赖 history 反查文件名,**必须先删文件再删记录**;反序会永远查不到产物。
|
||
|
||
没装扩展时自动降级到社区 DELETE 插件(deleteApi)或 cleanup.commands 文件系统模板;
|
||
此时顺序仍是「先抽清单 → 删记录 → 复核 → 删文件」。
|
||
|
||
安全闸门:本地导出的成片必须存在且 >= cleanup.minBytes 才允许清理;默认 dry-run,
|
||
必须 --apply 才真删。只按 prompt_id 精确删,禁用 {"clear": true}。本地资产永不删除。
|
||
|
||
用法:
|
||
python3 tools/cleanup_comfyui.py --ep EP01 # 预演
|
||
python3 tools/cleanup_comfyui.py --ep EP01 --apply # 真清
|
||
python3 tools/cleanup_comfyui.py --ep EP01 --shot S03 --apply # 只清一镜
|
||
python3 tools/cleanup_comfyui.py --prompt-id <pid> --endpoint http://... --apply
|
||
python3 tools/cleanup_comfyui.py --ep EP01 --apply --purge-input --free
|
||
"""
|
||
import argparse
|
||
import json
|
||
import os
|
||
import shlex
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
|
||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||
|
||
# K8s 三个端点的默认降级模板:动态取 pod 名,不重启 Pod 也能精确删产物。
|
||
# 与 assets/config.template.json 的 cleanup.commands 保持一致;工作空间配置可覆盖。
|
||
_K8S_RM_TMPL = (
|
||
"KUBECONFIG=$HOME/.kube/kubeconfig-wuguirong kubectl -n wuguirong exec "
|
||
"$(KUBECONFIG=$HOME/.kube/kubeconfig-wuguirong kubectl -n wuguirong get pods "
|
||
"-l app={app} -o jsonpath='{.items[0].metadata.name}') -c comfyui -- "
|
||
"sh -c \"cd {root} && rm -f -- {files} {temp_files}\"")
|
||
|
||
DEFAULT_CLEANUP = {
|
||
"auto": False, # 提交脚本导出成功后是否自动调用
|
||
"minBytes": 100000, # 本地成片小于此值视为导出失败,拒绝清理
|
||
"commands": { # {endpoint: "shell 模板"},占位符见 render_command
|
||
# 2026-11-13 起三个端口都指向同一个 pod(comfyui-wan-h3image,ComfyUI 0.45):
|
||
# 旧的 comfyui-wan-a14b / comfyui-wan-h3 deployment 已 scale 到 0,
|
||
# 它们的 Service selector 改指 app=comfyui-wan-h3image,NodePort 保持不变。
|
||
# 因此三条模板的 pod 选择器必须统一,否则会取到空 pod 名。
|
||
"http://192.168.1.30:30112": _K8S_RM_TMPL.replace("{app}", "comfyui-wan-h3image"),
|
||
"http://192.168.1.30:30111": _K8S_RM_TMPL.replace("{app}", "comfyui-wan-h3image"),
|
||
# H3-Image 文生图端点:未装 manju-cleanup 扩展,走 kubectl exec 降级
|
||
"http://192.168.1.30:30113": _K8S_RM_TMPL.replace("{app}", "comfyui-wan-h3image"),
|
||
},
|
||
"command": "", # 通用回退模板
|
||
"outputRoot": "/comfyui/output", # 服务端 output 根目录(K8s 是 PVC 软链)
|
||
"inputRoot": "/comfyui/input",
|
||
"purgeInput": False, # 上传的首帧图是否一起删(多镜可能共用,默认不删)
|
||
"free": False, # 整批清理结束后是否 POST /free(别每镜调用)
|
||
"apiBase": "/api", # 官方新版标准前缀;探测失败自动回退 ""
|
||
"deleteApi": "", # 社区自定义节点:comfyui_extra_api | api-tools | ""
|
||
"nodeApi": "", # 扩展路径前缀;留空自动探测 /api/manju/cleanup
|
||
"clientIds": [], # 额外 client_id 精确白名单(补老台账没记 clientId 的任务)
|
||
"ownerPrefixes": ["manju-"], # client_id 前缀白名单
|
||
"requireOwner": True, # 归属校验失败一律拒绝;改成 false 是危险的
|
||
"token": "", # 扩展设了 MANJU_CLEANUP_TOKEN 时填这里
|
||
}
|
||
|
||
# 社区自定义节点提供的标准 DELETE 路由(现网未安装,需显式配置才启用)
|
||
DELETE_APIS = {
|
||
"comfyui_extra_api": (
|
||
"/comfyapi/v1/output-images",
|
||
"/comfyapi/v1/output-images/{name}?temp=false"),
|
||
"api-tools": (
|
||
"/api-tools/v1/images/output",
|
||
"/api-tools/v1/images/output/{name}?temp=false"),
|
||
}
|
||
IMAGE_EXTS = (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp")
|
||
|
||
_API_BASE_CACHE = {}
|
||
_NODE_CACHE = {}
|
||
|
||
|
||
def find_root():
|
||
env = os.environ.get("MANJU_ROOT")
|
||
if env:
|
||
return os.path.abspath(os.path.expanduser(env))
|
||
d = os.getcwd()
|
||
while True:
|
||
if os.path.isfile(os.path.join(d, "config", "pipeline.json")):
|
||
return d
|
||
nd = os.path.dirname(d)
|
||
if nd == d:
|
||
break
|
||
d = nd
|
||
return os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
|
||
|
||
def load_cfg(root):
|
||
p = os.path.join(root, "config", "pipeline.json")
|
||
raw = json.load(open(p, encoding="utf-8")) if os.path.isfile(p) else {}
|
||
c = dict(DEFAULT_CLEANUP)
|
||
c.update(raw.get("cleanup", {}))
|
||
return raw, c
|
||
|
||
|
||
def http(host, path, payload=None, timeout=30, method=None):
|
||
"""返回 (status, json|None)。HTTPError 不当异常抛,返回码交给调用方判断。
|
||
|
||
ComfyUI 的 delete / free / interrupt 路由返回 200 空 body;空 body 不解析。
|
||
"""
|
||
if method is None:
|
||
method = "POST" if payload is not None else "GET"
|
||
data, headers = None, {}
|
||
if payload is not None:
|
||
data = json.dumps(payload).encode()
|
||
headers["Content-Type"] = "application/json"
|
||
req = urllib.request.Request(host + path, data=data, headers=headers, method=method)
|
||
try:
|
||
with opener.open(req, timeout=timeout) as r:
|
||
body, status = r.read(), r.status
|
||
except urllib.error.HTTPError as e:
|
||
return e.code, None
|
||
if not body:
|
||
return status, None
|
||
try:
|
||
return status, json.loads(body)
|
||
except ValueError:
|
||
return status, None
|
||
|
||
|
||
def api_base(host, cfg):
|
||
"""探测 /api 前缀(官方新版标准路径),不可用则回退无前缀的旧路径。"""
|
||
want = str(cfg.get("apiBase", "/api")).rstrip("/")
|
||
key = (host, want)
|
||
if key in _API_BASE_CACHE:
|
||
return _API_BASE_CACHE[key]
|
||
pref = ""
|
||
if want:
|
||
try:
|
||
st, d = http(host, want + "/system_stats", timeout=10)
|
||
if st == 200 and isinstance(d, dict) and ("system" in d or "devices" in d):
|
||
pref = want
|
||
except Exception:
|
||
pass
|
||
_API_BASE_CACHE[key] = pref
|
||
return pref
|
||
|
||
|
||
def ap(host, cfg, path):
|
||
return api_base(host, cfg) + path
|
||
|
||
|
||
def rel_file(f):
|
||
"""history outputs 里的一条 -> (type, subfolder, filename)"""
|
||
return (f.get("type") or "output",
|
||
(f.get("subfolder") or "").strip("/\\"),
|
||
f.get("filename") or "")
|
||
|
||
|
||
def collect_outputs(item):
|
||
"""从 history 条目的 outputs 抽全部产物,含 SaveImage / SaveVideo 多输出节点。"""
|
||
found = []
|
||
for node in (item.get("outputs") or {}).values():
|
||
for key in ("images", "videos", "gifs", "audio"):
|
||
for f in (node.get(key) or []):
|
||
t, sub, name = rel_file(f)
|
||
if name:
|
||
found.append({"type": t, "subfolder": sub, "filename": name})
|
||
return found
|
||
|
||
|
||
def collect_inputs(item):
|
||
"""从 history 的 prompt 节点图里找 LoadImage 用到的上传图(input 目录)。"""
|
||
found = []
|
||
prompt = item.get("prompt") or []
|
||
graph = prompt[2] if len(prompt) > 2 and isinstance(prompt[2], dict) else {}
|
||
for node in graph.values():
|
||
if not isinstance(node, dict):
|
||
continue
|
||
ct = node.get("class_type", "")
|
||
if "LoadImage" not in ct and "ImageLoader" not in ct:
|
||
continue
|
||
name = (node.get("inputs") or {}).get("image")
|
||
if isinstance(name, str) and name:
|
||
sub, _, base = name.rpartition("/")
|
||
found.append({"type": "input", "subfolder": sub, "filename": base or name})
|
||
return found
|
||
|
||
|
||
def client_id_of(item):
|
||
"""ComfyUI 把 client_id 持久化在 history 项 prompt[3](HTTP 序列化态)或 prompt[0]
|
||
(内存态原始布局 [extra_data, number, prompt])。两种都兼容,再退化递归找。"""
|
||
p = item.get("prompt") if isinstance(item, dict) else None
|
||
if isinstance(p, (list, tuple)):
|
||
for idx in (3, 0):
|
||
if len(p) > idx and isinstance(p[idx], dict) and "client_id" in p[idx]:
|
||
return str(p[idx].get("client_id") or "")
|
||
r = _find_client_id(item)
|
||
return str(r) if r else ""
|
||
|
||
|
||
def _find_client_id(o):
|
||
if isinstance(o, dict):
|
||
if "client_id" in o:
|
||
return o["client_id"]
|
||
for v in o.values():
|
||
r = _find_client_id(v)
|
||
if r is not None:
|
||
return r
|
||
elif isinstance(o, (list, tuple)):
|
||
for v in o:
|
||
r = _find_client_id(v)
|
||
if r is not None:
|
||
return r
|
||
return None
|
||
|
||
|
||
def credentials_for(job, cfg):
|
||
"""本次任务的归属凭证:台账 clientId + 配置精确白名单 + 前缀白名单。"""
|
||
ids = [x for x in [job.get("clientId")] + list(cfg.get("clientIds") or []) if x]
|
||
return {"client_ids": list(dict.fromkeys(str(x) for x in ids)),
|
||
"prefixes": list(cfg.get("ownerPrefixes") or []),
|
||
"require_owner": bool(cfg.get("requireOwner", True)),
|
||
"token": str(cfg.get("token") or "")}
|
||
|
||
|
||
def owner_of(cid, creds):
|
||
"""(owned, reason)。没有任何凭证时 fail-closed,绝不因为「传了 pid」就放行。"""
|
||
if cid and cid in creds["client_ids"]:
|
||
return True, "台账 clientId 精确匹配"
|
||
if cid and any(cid.startswith(p) for p in creds["prefixes"] if p):
|
||
return True, "client_id 前缀白名单匹配"
|
||
if not creds["require_owner"]:
|
||
return True, "requireOwner=false(显式关闭归属校验,不推荐)"
|
||
return False, ("client_id=%r 不在白名单:台账无 clientId、clientIds 未列、"
|
||
"ownerPrefixes 不匹配" % cid)
|
||
|
||
|
||
def node_api(host, cfg):
|
||
"""探测 manju-cleanup 扩展,返回可用路径前缀('' 表示未安装)。"""
|
||
key = (host, str(cfg.get("nodeApi") or ""))
|
||
if key in _NODE_CACHE:
|
||
return _NODE_CACHE[key]
|
||
custom = str(cfg.get("nodeApi") or "").rstrip("/")
|
||
cands = [custom] if custom else [ap(host, cfg, "/manju/cleanup"),
|
||
"/manju/cleanup"]
|
||
for p in dict.fromkeys([x for x in cands if x]):
|
||
try:
|
||
st, d = http(host, p + "/ping", timeout=8)
|
||
if st == 200 and isinstance(d, dict) and d.get("ok"):
|
||
_NODE_CACHE[key] = p
|
||
return p
|
||
except Exception:
|
||
pass
|
||
_NODE_CACHE[key] = ""
|
||
return ""
|
||
|
||
|
||
def node_purge(host, cfg, prefix, pids, creds, dry_run, purge_input):
|
||
"""调扩展的 purge;服务端按 pid 自己从 history 反查文件并二次校验归属。"""
|
||
body = {"pids": list(pids),
|
||
"client_ids": creds["client_ids"],
|
||
"owner_prefixes": creds["prefixes"],
|
||
"require_owner": creds["require_owner"],
|
||
"dry_run": bool(dry_run), "purge_input": bool(purge_input)}
|
||
if creds["token"]:
|
||
body["token"] = creds["token"]
|
||
try:
|
||
st, d = http(host, prefix + "/purge", body, timeout=90)
|
||
except Exception as e:
|
||
return {"ok": False, "error": "请求失败:%s" % e, "pids": {}}
|
||
if st != 200 or not isinstance(d, dict):
|
||
return {"ok": False, "error": "HTTP %d" % st, "pids": {}}
|
||
return d
|
||
|
||
|
||
def node_files_text(entry):
|
||
"""把扩展返回的单个 pid 报告压成一行人话。"""
|
||
if not entry:
|
||
return "扩展未返回该 pid"
|
||
if not entry.get("exists"):
|
||
return "拒绝:history 已无该条,服务端不猜测文件名"
|
||
if not entry.get("owned"):
|
||
return "拒绝:扩展归属校验未通过(%s)" % entry.get("owner_reason", "")
|
||
counts = {}
|
||
for f in entry.get("files") or []:
|
||
counts[f["status"]] = counts.get(f["status"], 0) + 1
|
||
return "扩展 purge:%s" % (", ".join("%s=%d" % kv for kv in sorted(counts.items()))
|
||
or "无文件")
|
||
|
||
|
||
def inspect_pid(pid, host, cfg, creds, purge_input=False):
|
||
"""阶段 A:先从 history 抽产物清单与 client_id,删了记录就再也拿不到。
|
||
|
||
扩展的 dry_run 预览由调用方按 host 批量执行(node_purge),不在这里发请求。
|
||
"""
|
||
rep = {"pid": pid, "host": host, "outputs": [], "inputs": [],
|
||
"history": "?", "queue": "?", "verify": "?", "files": "?",
|
||
"commands": [], "had": False, "client_id": "", "owned": False,
|
||
"owner_reason": "", "creds": creds, "node": "", "node_entry": {}}
|
||
rep["node"] = node_api(host, cfg)
|
||
try:
|
||
st, item = http(host, ap(host, cfg, "/history/" + urllib.parse.quote(pid)), timeout=30)
|
||
d = item or {}
|
||
if st != 200:
|
||
rep["verify"] = "history 读取 HTTP %d,不做文件清理" % st
|
||
return rep
|
||
entry = d.get(pid, d) if isinstance(d, dict) else {}
|
||
rep["had"] = bool(d) and (pid in d if isinstance(d, dict) else True)
|
||
all_out = collect_outputs(entry) if isinstance(entry, dict) else []
|
||
# 有些节点(LoadVideo / 预览类)会在 outputs 里回带 type=input 的源素材。
|
||
rep["outputs"] = [f for f in all_out if f["type"] in ("output", "temp")]
|
||
extra = [f for f in all_out if f["type"] == "input"]
|
||
seen = {(f["type"], f["subfolder"], f["filename"]) for f in extra}
|
||
rep["inputs"] = extra + [f for f in collect_inputs(entry)
|
||
if (f["type"], f["subfolder"], f["filename"]) not in seen]
|
||
rep["client_id"] = client_id_of(entry)
|
||
except Exception as e:
|
||
rep["verify"] = "history 读取失败:%s" % e
|
||
rep["owned"], rep["owner_reason"] = owner_of(rep["client_id"], creds)
|
||
if not rep["had"]:
|
||
rep["note"] = "history 已无该条(可能重启/被清过),不猜测文件名"
|
||
return rep
|
||
|
||
|
||
def merge_node_preview(rep, resp):
|
||
"""把服务端 dry_run 预览的归属结论并进客户端报告(服务端优先否决)。"""
|
||
if not (isinstance(resp, dict) and resp.get("ok")):
|
||
rep["node_error"] = (resp or {}).get("error", "预览失败")
|
||
return
|
||
e = (resp.get("pids") or {}).get(rep["pid"])
|
||
rep["node_entry"] = e or {}
|
||
if e and not e.get("owned"):
|
||
rep["owned"] = False
|
||
rep["owner_reason"] = "扩展否决:%s" % e.get("owner_reason", "")
|
||
|
||
|
||
def delete_records(host, pids, cfg, had_map):
|
||
"""按 host 批量调标准接口删记录,再逐 pid 复核「从有到无」。
|
||
|
||
200 空 body 只代表请求被接受且幂等完成,不代表记录真的消失。
|
||
"""
|
||
out = {pid: {} for pid in pids}
|
||
for path, key in (("/history", "history"), ("/queue", "queue")):
|
||
try:
|
||
st, _ = http(host, ap(host, cfg, path), {"delete": list(pids)}, timeout=30)
|
||
ok = st == 200
|
||
for pid in pids:
|
||
out[pid][key] = ("HTTP 200(空 body)" if ok else "FAILED HTTP %d" % st)
|
||
except Exception as e:
|
||
for pid in pids:
|
||
out[pid][key] = "FAILED: %s" % e
|
||
|
||
for pid in pids:
|
||
try:
|
||
st, d = http(host, ap(host, cfg, "/history/" + urllib.parse.quote(pid)), timeout=20)
|
||
still = bool(d) and (pid in d if isinstance(d, dict) else True)
|
||
if st == 200 and not still and had_map.get(pid):
|
||
out[pid]["verify"] = "已确认(删前有 → 删后无)"
|
||
elif not had_map.get(pid):
|
||
out[pid]["verify"] = "幂等 200,但删前就无此记录,不宣称删除成功"
|
||
elif still:
|
||
out[pid]["verify"] = "FAILED:复核时记录仍在"
|
||
else:
|
||
out[pid]["verify"] = "未确认(HTTP %d)" % st
|
||
except Exception as e:
|
||
out[pid]["verify"] = "复核失败:%s" % e
|
||
try:
|
||
st, q = http(host, ap(host, cfg, "/queue"), timeout=20)
|
||
if st == 200 and isinstance(q, dict):
|
||
live = set()
|
||
for arr in ("queue_running", "queue_pending"):
|
||
for x in q.get(arr) or []:
|
||
if isinstance(x, list) and len(x) > 1:
|
||
live.add(str(x[1]))
|
||
for pid in pids:
|
||
out[pid]["queue_verify"] = "queue 已无该 pid" if pid not in live else \
|
||
"FAILED:queue 里仍有该 pid"
|
||
except Exception:
|
||
pass
|
||
return out
|
||
|
||
|
||
def render_command(template, files, root, input_files, input_root):
|
||
"""把模板里的占位符替换成 shell-safe 的路径列表。"""
|
||
def q(fs):
|
||
return " ".join(shlex.quote(os.path.join(f["subfolder"], f["filename"]))
|
||
for f in fs) if fs else ""
|
||
return (template
|
||
.replace("{files}", q([f for f in files if f["type"] == "output"]))
|
||
.replace("{temp_files}", q([f for f in files if f["type"] == "temp"]))
|
||
.replace("{input_files}", q(input_files))
|
||
.replace("{root}", shlex.quote(root) if root else ".")
|
||
.replace("{input_root}", shlex.quote(input_root) if input_root else "."))
|
||
|
||
|
||
def pick_command(cfg, host):
|
||
return (cfg.get("commands") or {}).get(host.rstrip("/")) or cfg.get("command") or ""
|
||
|
||
|
||
def try_http_delete(host, cfg, files):
|
||
"""社区方案:若端点装了自定义节点,用标准 HTTP DELETE 删 output 图片。"""
|
||
name = str(cfg.get("deleteApi") or "").strip()
|
||
if not name or name not in DELETE_APIS:
|
||
return [], files, ""
|
||
list_path, del_tpl = DELETE_APIS[name]
|
||
try:
|
||
st, d = http(host, list_path, timeout=15)
|
||
if st != 200 or not isinstance(d, (list, dict)):
|
||
return [], files, "deleteApi=%s 未就绪(HTTP %d),回退文件模板" % (name, st)
|
||
except Exception as e:
|
||
return [], files, "deleteApi=%s 探测失败:%s" % (name, e)
|
||
|
||
deleted, remaining, fails = [], [], []
|
||
for f in files:
|
||
path = os.path.join(f["subfolder"], f["filename"])
|
||
eligible = (f["type"] == "output" and not f["subfolder"]
|
||
and f["filename"].lower().endswith(IMAGE_EXTS))
|
||
if not eligible:
|
||
remaining.append(f)
|
||
continue
|
||
url = del_tpl.replace("{name}", urllib.parse.quote(f["filename"]))
|
||
try:
|
||
st, _ = http(host, url, timeout=20, method="DELETE")
|
||
(deleted if st in (200, 204) else remaining).append(f)
|
||
if st not in (200, 204):
|
||
fails.append("%s → HTTP %d" % (path, st))
|
||
except Exception as e:
|
||
remaining.append(f)
|
||
fails.append("%s → %s" % (path, e))
|
||
note = "标准 DELETE 删掉 %d 个%s" % (len(deleted), (";失败:" + ", ".join(fails[:3])) if fails else "")
|
||
return deleted, remaining, note
|
||
|
||
|
||
def delete_files(host, cfg, rep, apply, purge_input=False):
|
||
"""降级路线:优先社区 DELETE,其余走服务端文件系统模板。
|
||
|
||
dry-run 绝不发任何 DELETE / 任何 shell 命令,只列清单与将要执行的命令。
|
||
"""
|
||
files = rep["outputs"] + (rep["inputs"] if purge_input else [])
|
||
inputs = rep["inputs"] if purge_input else []
|
||
if not files:
|
||
rep["files"] = "无产物文件需删"
|
||
return rep
|
||
|
||
tpl = pick_command(cfg, host)
|
||
cmd = render_command(tpl, files, cfg.get("outputRoot", ""),
|
||
inputs, cfg.get("inputRoot", "")) if tpl else ""
|
||
|
||
if not apply:
|
||
parts = ["dry-run 不删(%d 个文件)" % len(files)]
|
||
name = str(cfg.get("deleteApi") or "").strip()
|
||
if name in DELETE_APIS:
|
||
st, d = http(host, DELETE_APIS[name][0], timeout=15)
|
||
n_img = sum(1 for f in files if f["type"] == "output" and not f["subfolder"]
|
||
and f["filename"].lower().endswith(IMAGE_EXTS))
|
||
parts.append("deleteApi=%s %s,%d 个图片将走标准 DELETE"
|
||
% (name, "可用" if st == 200 and isinstance(d, (list, dict)) else "未就绪",
|
||
n_img))
|
||
if cmd:
|
||
rep["commands"].append(cmd)
|
||
else:
|
||
parts.append("未配 cleanup.commands[%s],服务端文件只能手工清" % host)
|
||
rep["pending"] = files
|
||
rep["files"] = " | ".join(parts)
|
||
return rep
|
||
|
||
deleted, remaining, note = try_http_delete(host, cfg, files)
|
||
parts = [note] if note else []
|
||
|
||
if remaining:
|
||
if not tpl:
|
||
parts.append("未配 cleanup.commands[%s],%d 个文件未删(见清单)"
|
||
% (host, len(remaining)))
|
||
rep["pending"] = remaining
|
||
else:
|
||
cmd = render_command(tpl, remaining, cfg.get("outputRoot", ""),
|
||
inputs, cfg.get("inputRoot", ""))
|
||
rep["commands"].append(cmd)
|
||
try:
|
||
p = subprocess.run(cmd, shell=True, capture_output=True,
|
||
text=True, timeout=120)
|
||
parts.append("模板 %s (rc=%d)" % ("ok" if p.returncode == 0 else "FAILED",
|
||
p.returncode))
|
||
if p.returncode != 0:
|
||
parts.append(p.stderr.strip()[:200])
|
||
except Exception as e:
|
||
parts.append("模板 FAILED: %s" % e)
|
||
if not parts:
|
||
parts.append("全部走标准 DELETE")
|
||
rep["files"] = " | ".join(parts)
|
||
return rep
|
||
|
||
|
||
def free_models(host, cfg):
|
||
"""标准接口释放模型/显存。它只是设置队列 flag,真正卸载发生在下一个队列循环。"""
|
||
try:
|
||
st, _ = http(host, ap(host, cfg, "/free"),
|
||
{"unload_models": True, "free_memory": True}, timeout=30)
|
||
return "HTTP %d(空 body,异步生效)" % st if st == 200 else "FAILED HTTP %d" % st
|
||
except Exception as e:
|
||
return "FAILED: %s" % e
|
||
|
||
|
||
LEDGER_NAMES = ("t2i_runs.json", "a14b_runs.json")
|
||
|
||
|
||
def ledger_paths(root, ep):
|
||
"""列出该扫描范围内的台账(去重)。
|
||
|
||
submit_t2i.py 把台账写在 **os.path.dirname(out)**,所以台账位置随输出路径变:
|
||
--ep/--shot 出图 → work/<EP>/images/t2i_runs.json
|
||
裸 --out 出锚图 → assets/characters/t2i_runs.json(跟图片同目录!)
|
||
因此不能只猜固定路径 —— 漏扫 = 锚图产物永远留在服务端。
|
||
这里直接递归扫工作空间下所有同名台账,--ep 时额外限定只取该集内的。
|
||
"""
|
||
found, seen = [], set()
|
||
|
||
def add(label, path):
|
||
rp = os.path.realpath(path)
|
||
if rp in seen:
|
||
return
|
||
seen.add(rp)
|
||
found.append((label, path, "videoModel" if label == "a14b_runs.json" else "imageModel"))
|
||
|
||
for base, dirs, files in os.walk(root):
|
||
dirs[:] = [d for d in dirs if d not in (".git", "__pycache__", "node_modules")]
|
||
for fn in files:
|
||
if fn not in LEDGER_NAMES:
|
||
continue
|
||
path = os.path.join(base, fn)
|
||
rel = os.path.relpath(path, root)
|
||
if ep:
|
||
# 带 --ep 时:只要台账路径里含该集号,或台账在集外(锚图目录)
|
||
in_ep = (os.sep + ep + os.sep) in (os.sep + rel)
|
||
outside_ep = not any(
|
||
part.startswith("EP") and os.path.isdir(os.path.join(root, "work", part))
|
||
for part in rel.split(os.sep)
|
||
)
|
||
if not in_ep and not outside_ep:
|
||
continue
|
||
add(fn, path)
|
||
return found
|
||
|
||
|
||
def gather(root, raw, cfg, ep, shots, prompt_ids):
|
||
"""收集待清理任务,逐条过导出闸门。返回 (jobs, skipped)。
|
||
|
||
prompt_ids 元素兼容 6 元组与 7 元组(末尾多一个 clientId)。
|
||
"""
|
||
jobs, skipped = [], []
|
||
for p in prompt_ids:
|
||
pid, host, key, ledger, ledger_key, local = p[:6]
|
||
cid = p[6] if len(p) > 6 else ""
|
||
jobs.append({"pid": pid, "host": host, "key": key, "ledger": ledger,
|
||
"ledgerKey": ledger_key, "local": local, "clientId": cid,
|
||
"reason": "--prompt-id 显式指定,跳过导出闸门"})
|
||
# 扫描台账:无 --ep 时扫工作空间根的台账(裸 --out 的锚图),
|
||
# 有 --ep 时扫该集台账 + 根级台账(锚图不属于任何集)。
|
||
for label, path, cfg_key in ledger_paths(root, ep):
|
||
if not os.path.isfile(path):
|
||
continue
|
||
runs = json.load(open(path, encoding="utf-8"))
|
||
host = str(raw.get(cfg_key, {}).get("endpoint", "")).rstrip("/")
|
||
for key, rec in runs.items():
|
||
if shots and key not in shots and key.split("/")[-1] not in shots:
|
||
continue
|
||
pid = rec.get("promptId")
|
||
if not pid:
|
||
continue # 已清过,跳过
|
||
# 台账里的 endpoint 是该任务**实际**提交到的端点,优先于配置里的全局值
|
||
# (锚图与首帧可能不在同一个端点,别清错地方)
|
||
task_host = str(rec.get("endpoint") or host).rstrip("/")
|
||
local = os.path.join(root, rec.get("out", ""))
|
||
ok = os.path.isfile(local) and os.path.getsize(local) >= int(cfg["minBytes"])
|
||
j = {"pid": pid, "host": task_host, "key": key, "ledger": path,
|
||
"ledgerKey": key, "local": local,
|
||
"clientId": rec.get("clientId", "")}
|
||
if ok:
|
||
jobs.append(j)
|
||
else:
|
||
j["reason"] = "本地导出文件缺失或过小:%s" % local
|
||
skipped.append(j)
|
||
seen, uniq = set(), []
|
||
for j in jobs:
|
||
if j["pid"] in seen:
|
||
continue
|
||
seen.add(j["pid"])
|
||
uniq.append(j)
|
||
return uniq, skipped
|
||
|
||
|
||
def update_ledger(jobs, skipped, apply, purge_ledger):
|
||
"""默认移除服务端可关联的 promptId 并留清理时间;--purge-ledger 则删整条。"""
|
||
if not apply:
|
||
return 0
|
||
touched = set()
|
||
for j in jobs:
|
||
p = j.get("ledger")
|
||
if not p or not os.path.isfile(p) or p in touched:
|
||
continue
|
||
touched.add(p)
|
||
runs = json.load(open(p, encoding="utf-8"))
|
||
for job in jobs:
|
||
if job.get("ledger") != p:
|
||
continue
|
||
k = job.get("ledgerKey")
|
||
if k not in runs:
|
||
continue
|
||
if purge_ledger:
|
||
runs.pop(k, None)
|
||
else:
|
||
runs[k].pop("promptId", None)
|
||
runs[k]["cleanedAt"] = time.strftime("%Y-%m-%d %H:%M:%S")
|
||
with open(p, "w", encoding="utf-8") as f:
|
||
json.dump(runs, f, ensure_ascii=False, indent=2)
|
||
return len(touched)
|
||
|
||
|
||
def group_by(pairs, key):
|
||
g = {}
|
||
for item in pairs:
|
||
g.setdefault(key(item), []).append(item)
|
||
return g
|
||
|
||
|
||
def cleanup_pid(pid, host, apply, cfg, creds=None, purge_input=False):
|
||
"""单 pid 完整清理,供提交脚本的 after_export 钩子调用。"""
|
||
creds = creds or credentials_for({}, cfg)
|
||
rep = inspect_pid(pid, host, cfg, creds, purge_input)
|
||
prefix = rep["node"]
|
||
if prefix:
|
||
prev = node_purge(host, cfg, prefix, [pid], creds, True, purge_input)
|
||
merge_node_preview(rep, prev)
|
||
if not rep["owned"]:
|
||
rep["history"] = rep["queue"] = "拒绝:归属校验未通过"
|
||
rep["files"] = "拒绝:不删任何服务端文件"
|
||
return rep
|
||
if not apply:
|
||
rep["history"] = "dry-run 不删"
|
||
rep["queue"] = "dry-run 不删"
|
||
if prefix:
|
||
rep["files"] = node_files_text(rep["node_entry"])
|
||
return rep
|
||
return delete_files(host, cfg, rep, False, purge_input)
|
||
|
||
if prefix:
|
||
res = node_purge(host, cfg, prefix, [pid], creds, False, purge_input)
|
||
rep["node_entry"] = (res.get("pids") or {}).get(pid, {})
|
||
rep["files"] = node_files_text(rep["node_entry"])
|
||
if not res.get("ok"):
|
||
rep["files"] += " | purge HTTP 异常:%s" % res.get("error", "")
|
||
res = delete_records(host, [pid], cfg, {pid: rep["had"]})[pid]
|
||
rep["history"] = "%s | %s" % (res["history"], res["verify"])
|
||
rep["queue"] = res["queue"] + (" | " + res["queue_verify"] if res.get("queue_verify") else "")
|
||
if not prefix:
|
||
rep = delete_files(host, cfg, rep, True, purge_input)
|
||
return rep
|
||
|
||
|
||
def after_export(root, pid, host, ledger, ledger_key, local, client_id=""):
|
||
"""提交脚本钩子:成片已落本地后,清掉服务端记录与产物。
|
||
|
||
调用方保证下载成功。这里再独立过一次导出闸门 + 归属闸门,任何异常都只影响
|
||
清理、不影响已成片的产物。整批做完再单独 --free,不要每镜卸载模型。
|
||
"""
|
||
raw, cfg = load_cfg(root)
|
||
host = host.rstrip("/")
|
||
if not os.path.isfile(local) or os.path.getsize(local) < int(cfg["minBytes"]):
|
||
return "SKIP 本地成片缺失/过小,未清理:%s" % local
|
||
if not client_id and ledger and os.path.isfile(ledger):
|
||
try:
|
||
client_id = (json.load(open(ledger, encoding="utf-8"))
|
||
.get(ledger_key, {}).get("clientId", ""))
|
||
except Exception:
|
||
pass
|
||
creds = credentials_for({"clientId": client_id}, cfg)
|
||
rep = cleanup_pid(pid, host, True, cfg, creds, bool(cfg.get("purgeInput")))
|
||
if not rep["owned"]:
|
||
return "REFUSE 归属校验未通过(client_id=%s):未删任何记录/文件" % rep["client_id"]
|
||
update_ledger([{"ledger": ledger, "ledgerKey": ledger_key, "pid": pid}],
|
||
[], True, bool(cfg.get("purgeLedger")))
|
||
tail = ""
|
||
if "未删" in rep["files"] or "未配" in rep["files"] or "拒绝" in rep["files"]:
|
||
tail = " | ⚠ 仍有服务端文件待清"
|
||
return ("history=%s queue=%s files=%s 产物%d个 归属=%s%s"
|
||
% (rep["history"], rep["queue"], rep["files"], len(rep["outputs"]),
|
||
rep["client_id"] or "-", tail))
|
||
|
||
|
||
def print_report(j, rep, cfg):
|
||
print("\n[%s] %s @ %s" % (j["key"], j["pid"], j["host"]))
|
||
print(" 归属 : %s | client_id=%s | %s" %
|
||
("✅ 自己的任务" if rep["owned"] else "⛔ 拒绝(非本人任务)",
|
||
rep["client_id"] or "-", rep["owner_reason"]))
|
||
print(" 扩展 : %s" % (rep["node"] + "/purge" if rep["node"] else
|
||
"未安装 → 降级 deleteApi / commands"))
|
||
print(" history : %s" % rep["history"])
|
||
print(" queue : %s" % rep["queue"])
|
||
print(" 产物 : %d 个文件" % len(rep["outputs"]))
|
||
for f in rep["outputs"]:
|
||
print(" - [%s] %s" % (f["type"], os.path.join(f["subfolder"], f["filename"])))
|
||
if rep["node_entry"]:
|
||
for f in rep["node_entry"].get("files") or []:
|
||
print(" · %s → %s%s" % (os.path.join(f["subfolder"], f["filename"]),
|
||
f["status"],
|
||
("(" + f["reason"] + ")") if f.get("reason") else ""))
|
||
if rep["inputs"]:
|
||
print(" 上传图 : %d 个%s" % (len(rep["inputs"]),
|
||
"(--purge-input 未开,列出不删)" if not cfg.get("purgeInput") else ""))
|
||
print(" 文件删除: %s" % rep["files"])
|
||
for c in rep["commands"]:
|
||
print(" 命令 : %s" % c)
|
||
if rep.get("node_error"):
|
||
print(" 扩展错误: %s" % rep["node_error"])
|
||
if rep.get("note"):
|
||
print(" 备注 : %s" % rep["note"])
|
||
|
||
|
||
def main():
|
||
ap_ = argparse.ArgumentParser(description="只清自己发起的任务:ComfyUI 记录 + 服务端产物")
|
||
ap_.add_argument("--ep", default="", help="集号,如 EP01;读该集两个台账")
|
||
ap_.add_argument("--shot", default="", help="只清指定镜,逗号分隔")
|
||
ap_.add_argument("--prompt-id", default="", help="直接指定 prompt_id,逗号分隔")
|
||
ap_.add_argument("--endpoint", default="", help="--prompt-id 对应的服务地址")
|
||
ap_.add_argument("--client-id", default="", help="--prompt-id 对应任务的 client_id(归属凭证)")
|
||
ap_.add_argument("--apply", action="store_true", help="真的执行;默认只预演")
|
||
ap_.add_argument("--purge-input", action="store_true", help="连上传的首帧图一起删")
|
||
ap_.add_argument("--purge-ledger", action="store_true", help="删掉本地台账整条记录")
|
||
ap_.add_argument("--free", action="store_true", help="整批结束后 POST /free 释放模型")
|
||
ap_.add_argument("--force", action="store_true", help="跳过本地导出文件闸门(危险)")
|
||
a = ap_.parse_args()
|
||
|
||
root = find_root()
|
||
raw, cfg = load_cfg(root)
|
||
if a.force:
|
||
cfg["minBytes"] = 0
|
||
if a.purge_input:
|
||
cfg["purgeInput"] = True
|
||
shots = set(x.strip() for x in a.shot.split(",") if x.strip())
|
||
explicit = []
|
||
if a.prompt_id:
|
||
host = (a.endpoint or raw.get("videoModel", {}).get("endpoint", "")).rstrip("/")
|
||
if not host:
|
||
return "用 --prompt-id 时必须给 --endpoint 或在 config 里配 videoModel.endpoint"
|
||
for pid in [x.strip() for x in a.prompt_id.split(",") if x.strip()]:
|
||
explicit.append((pid, host, pid, "", pid, "", a.client_id))
|
||
|
||
jobs, skipped = gather(root, raw, cfg, a.ep, shots, explicit)
|
||
if not jobs and not skipped:
|
||
return "没有待清理的 prompt(台账里无 promptId,或都已被清过)"
|
||
|
||
print("模式:%s | 归属校验:%s" % (
|
||
"APPLY 真删" if a.apply else "DRY-RUN 预演",
|
||
"开启(fail-closed)" if cfg.get("requireOwner", True) else "⚠ 已关闭"))
|
||
print("待清 %d 条,导出闸门拦下 %d 条" % (len(jobs), len(skipped)))
|
||
for s in skipped:
|
||
print(" [SKIP] %s %s" % (s["key"], s["reason"]))
|
||
|
||
# 阶段 A:抽清单 + 客户端归属判定 + 探测扩展
|
||
reports = []
|
||
for j in jobs:
|
||
creds = credentials_for(j, cfg)
|
||
rep = inspect_pid(j["pid"], j["host"], cfg, creds, bool(cfg.get("purgeInput")))
|
||
reports.append((j, rep, creds))
|
||
|
||
# 阶段 A2:扩展 dry_run 预览(按 host 批量),服务端归属结论优先否决
|
||
for host, triples in group_by(reports, lambda t: t[1]["host"]).items():
|
||
with_node = [t for t in triples if t[1]["node"]]
|
||
if not with_node:
|
||
continue
|
||
merged = {"client_ids": [], "prefixes": [], "require_owner": True, "token": ""}
|
||
for _, _, c in with_node:
|
||
merged["client_ids"] += c["client_ids"]
|
||
merged["prefixes"] += c["prefixes"]
|
||
merged["token"] = merged["token"] or c["token"]
|
||
merged["client_ids"] = list(dict.fromkeys(merged["client_ids"]))
|
||
merged["prefixes"] = list(dict.fromkeys(merged["prefixes"]))
|
||
prev = node_purge(host, cfg, with_node[0][1]["node"],
|
||
[t[1]["pid"] for t in with_node], merged, True,
|
||
bool(cfg.get("purgeInput")))
|
||
for _, rep, _ in with_node:
|
||
merge_node_preview(rep, prev)
|
||
|
||
# 归属闸门:不归属的 pid 绝不进入任何删除操作
|
||
eligible = [t for t in reports if t[1]["owned"]]
|
||
for _, rep, _ in reports:
|
||
if not rep["owned"]:
|
||
rep["history"] = rep["queue"] = "拒绝:归属校验未通过"
|
||
rep["files"] = "拒绝:不删任何服务端文件"
|
||
rep["commands"] = []
|
||
|
||
# 阶段 B:扩展真实删文件(必须在删记录之前:purge 依赖 history 反查)
|
||
for host, triples in group_by(eligible, lambda t: t[1]["host"]).items():
|
||
with_node = [t for t in triples if t[1]["node"]]
|
||
if not (a.apply and with_node):
|
||
continue
|
||
merged = {"client_ids": [], "prefixes": [], "require_owner": True, "token": ""}
|
||
for _, _, c in with_node:
|
||
merged["client_ids"] += c["client_ids"]
|
||
merged["prefixes"] += c["prefixes"]
|
||
merged["token"] = merged["token"] or c["token"]
|
||
merged["client_ids"] = list(dict.fromkeys(merged["client_ids"]))
|
||
merged["prefixes"] = list(dict.fromkeys(merged["prefixes"]))
|
||
res = node_purge(host, cfg, with_node[0][1]["node"],
|
||
[t[1]["pid"] for t in with_node], merged, False,
|
||
bool(cfg.get("purgeInput")))
|
||
for _, rep, _ in with_node:
|
||
rep["node_entry"] = (res.get("pids") or {}).get(rep["pid"], {})
|
||
rep["files"] = node_files_text(rep["node_entry"])
|
||
if not res.get("ok"):
|
||
rep["files"] += " | purge 异常:%s" % res.get("error", "")
|
||
|
||
# 阶段 C:官方标准接口批量删记录 + 复核(只传 eligible 的 pid)
|
||
if a.apply:
|
||
for host, triples in group_by(eligible, lambda t: t[1]["host"]).items():
|
||
pids = [t[1]["pid"] for t in triples]
|
||
res = delete_records(host, pids, cfg,
|
||
{t[1]["pid"]: t[1]["had"] for t in triples})
|
||
for _, rep, _ in triples:
|
||
r = res[rep["pid"]]
|
||
rep["history"] = "%s | %s" % (r["history"], r["verify"])
|
||
rep["queue"] = r["queue"] + (" | " + r["queue_verify"]
|
||
if r.get("queue_verify") else "")
|
||
else:
|
||
for _, rep, _ in eligible:
|
||
rep["history"] = "dry-run 不删"
|
||
rep["queue"] = "dry-run 不删"
|
||
|
||
# 阶段 D:未装扩展的走降级路线(社区 DELETE / 文件系统模板)
|
||
for _, rep, _ in eligible:
|
||
if rep["node"]:
|
||
if not a.apply:
|
||
rep["files"] = node_files_text(rep["node_entry"])
|
||
else:
|
||
delete_files(rep["host"], cfg, rep, a.apply, bool(cfg.get("purgeInput")))
|
||
|
||
for j, rep, _ in reports:
|
||
print_report(j, rep, cfg)
|
||
|
||
if a.apply:
|
||
n = update_ledger([j for j, rep, _ in eligible if rep["owned"]],
|
||
skipped, True, a.purge_ledger)
|
||
print("\n台账更新:%d 个文件(%s)" % (
|
||
n, "整条删除" if a.purge_ledger else "移除 promptId + 记 cleanedAt"))
|
||
if a.free or cfg.get("free"):
|
||
for host in sorted(set(j["host"] for j, _, _ in eligible)):
|
||
print(" /free %s -> %s" % (host, free_models(host, cfg)))
|
||
else:
|
||
print("\n这是预演,未删除任何东西。确认清单后加 --apply。")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
sys.exit(main())
|
||
except Exception as e:
|
||
print("ERROR: %s" % e, file=sys.stderr, flush=True)
|
||
sys.exit(1)
|