两件事: 1) 验证脚本统一从 /api/v1/me 拉能力目录(新增 tests/me_catalog.py 共享模块)。 之前 slot×模型、apix 条目写死在脚本里——配置是管理台动态的,改配置后探针 要么漏验新上游、要么拿旧模型名误报。现在 test_models 按 upstreams×models 全展开(both 双格式并发),apix_check 按目录逐条探;拉不到目录退出码 2 (验证失效 ≠ 上游没问题,不许假绿)。 三态口径:❌ 网关侧故障(错误体归属 + HTTP 码 + error.code 值三重判据—— 智谱 401 体也是 {"error":{code,message}} 形状,光看形态会把欠费误判成发版 故障);⚠️ 上游/厂商侧(欠费 402、不可达 502/504、WAF、200 软错误) **如实汇总反馈但不染红发版**;网关 429 限流按 GW_WARN 处理。 live_check.sh 退出码语义固定:0 过 / 1 网关侧故障 / 2 验证不可用。 分类器 15 个边界场景离线单测全对(撞形状/码集合/软错误/不误伤正常 200)。 2) agent.sh release 改 4/4 步:验证失败不再停在原地—— 抓 PRE_IMG + update job 的备份路径 → live_check → rc=1 时自动 POST /rollback {tag, restore_backup} 回滚镜像并恢复 data/audit,回滚后 再跑 verify 自检;rc=2(验证不可用)**不回滚**(盲回滚比带病观察危险)。 ipass-agent 1.0.2:rollback 本地无镜像自动 pull(自动路径不许卡「先手动拉」); 新增 restore_data——先 stop 容器(防周期落盘把污染写回)、备份 tar 解包顶替 data/audit、现目录改名 .broken-<ts> 留证、只认 ~/ipass/backup/ 下的 tar。 restore_data 本地单测过(恢复/留证/staging 清理/越界拒绝)。 release 开头加控制面版本闸门:<1.0.2 会静默忽略 restore_backup → 中止 (IPASS_FORCE_RELEASE=1 可跳过),⚠️ 线上控制面还是 1.0.1,发版前先上 .11 跑 §12.5 自更新。 附带修 bash 老坑:`$VAR` 紧跟全角字符被吞进变量名(unbound variable, live_check A 路径实测炸出),统一 ${VAR}。 线上实测:live_check 三步全绿(LLM 10/10 目录全展开 + apix 3/3 + 验收)。 只动 tests/deploy,不进镜像;ipass-agent 属控制面(跑宿主),需单独自更新。
162 lines
6.7 KiB
Python
162 lines
6.7 KiB
Python
#!/usr/bin/env python3
|
||
"""apix 上线后探针 —— 条目从 /api/v1/me 动态拉,逐个验 vendor / passthrough 转发。
|
||
|
||
与 test_models.py 共用 me_catalog.py(目录拉取 + 三态判定),口径完全一致:
|
||
❌ 网关侧故障(location 没挂 / 鉴权没生效 / 连不上)→ 退出码 1,发版要抓;
|
||
⚠️ 上游/供应商侧问题(欠费 402、服务不可达 502、WAF 拦截、探针参数被拒)
|
||
→ **如实打印并汇总列出**,但不染红发版验证。
|
||
|
||
探针构造(参数名/路径都跟着目录走,不写死):
|
||
vendor → `GET /apix/<slug>?<p>=<占位>`,参数名取 `params[0]`,没有就从
|
||
`desc`/`usage_hint` 正则找第一个 `?xx=`(如 accel 的 `?url=`)。
|
||
passthrough → `GET /apixp/<slug>/health`(轻量,不触发 GPU 懒加载)。
|
||
tail 是 `/apixp/<slug>` **之后**的全部:上游 `/health` 对应
|
||
`/apixp/ocr/health`;usage_hint 里的 `/<tail>` 要换成 slug,别照抄。
|
||
|
||
每条探针前还做一次**无效 Key 预检**(期望 401 + 网关体):证明 location 挂着、
|
||
鉴权生效——与上游/供应商是否抽风无关。若连无效 Key 都没被网关拦下,判 ❌。
|
||
|
||
用法:
|
||
python3 tests/apix_check.py <key> # 探针全部 enabled 条目
|
||
python3 tests/apix_check.py <key> --slug ocr # 只探某几个(逗号分隔)
|
||
python3 tests/apix_check.py <key> --timeout 40 --no-retry
|
||
|
||
退出码:有 ❌ → 1;只有 ✅/⚠️ → 0;拉不到目录 → 2(探针失效,不是上游没问题)。
|
||
"""
|
||
|
||
import os
|
||
import re
|
||
import sys
|
||
import time
|
||
import urllib.parse
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
import me_catalog as mc # noqa: E402
|
||
|
||
# vendor 参数占位值:让请求「格式合法」即可。accel 这类只放行白名单 host
|
||
# (GitHub/Gitee/GitLab)——用最小 raw 文件(~485B),拿 200 又不吃额度。
|
||
PLACEHOLDER = {
|
||
"q": "ping", "query": "ping", "keyword": "ping", "text": "ping",
|
||
"url": "https://raw.githubusercontent.com/octocat/Hello-World/master/README",
|
||
}
|
||
|
||
# 无效 Key 预检:形态合法(sk_ 开头)但查不到,网关应返 401
|
||
BAD_KEY = "sk_" + "0" * 48
|
||
|
||
# 只有「疑似限流/抖动」值得重试(souyi 边缘偶发拦截);404 重试没意义
|
||
RETRYABLE = (403, 429, 0)
|
||
RETRY_WAIT_SEC = 15
|
||
|
||
|
||
def build_url(item: dict):
|
||
"""按 mode 构造探针 URL / 方法。返回 (method, url, body, ctype) 或 None。"""
|
||
slug = item.get("slug")
|
||
mode = item.get("mode") or "vendor"
|
||
if mode == "passthrough":
|
||
return "GET", f"{mc.LM}/apixp/{slug}/health", None, None
|
||
params = item.get("params") or []
|
||
p = str(params[0]) if params else None
|
||
if not p:
|
||
hint = f"{item.get('desc') or ''} {item.get('usage_hint') or ''}"
|
||
m = re.search(r"[?&](\w+)=", hint)
|
||
p = m.group(1) if m else None
|
||
query = f"?{urllib.parse.urlencode({p: PLACEHOLDER.get(p, 'test')})}" if p else ""
|
||
return "GET", f"{mc.LM}/apix/{slug}{query}", None, None
|
||
|
||
|
||
def probe(item: dict, key: str, timeout: int) -> dict:
|
||
built = build_url(item)
|
||
slug = item.get("slug") or "?"
|
||
mode = item.get("mode") or "vendor"
|
||
if not built:
|
||
return {"slug": slug, "mode": mode, "url": "-", "code": 0,
|
||
"status": mc.GW_FAIL, "why": "探针 URL 构造失败", "elapsed": 0,
|
||
"raw": "", "auth_ok": False, "auth_code": 0}
|
||
method, url, body, ctype = built
|
||
code, raw, elapsed = mc.http(method, url, key, timeout, body, ctype)
|
||
status, why = mc.classify_apix(code, raw)
|
||
|
||
# 无效 Key 预检(期望 401 且归属是网关,双判据见 me_catalog.from_gateway)
|
||
acode, araw, _ = mc.http(method, url, BAD_KEY, timeout, body, ctype)
|
||
auth_ok = (acode == 401 and mc.from_gateway(acode, araw)[0])
|
||
if not auth_ok:
|
||
status = mc.GW_FAIL
|
||
why = f"网关鉴权未生效(无效 Key 返回 HTTP {acode},期望 401 网关体)"
|
||
|
||
return {"slug": slug, "mode": mode, "url": url, "code": code, "status": status,
|
||
"why": why, "elapsed": elapsed, "raw": raw[:160].replace("\n", " "),
|
||
"auth_ok": auth_ok, "auth_code": acode}
|
||
|
||
|
||
def parse_args(argv):
|
||
key, timeout, only, retry = None, 30, None, True
|
||
i = 1
|
||
while i < len(argv):
|
||
a = argv[i]
|
||
if a.startswith("--timeout"):
|
||
timeout = int(a.split("=", 1)[1]) if "=" in a else int(argv[i + 1])
|
||
i += 1 if "=" not in a else 0
|
||
elif a == "--no-retry":
|
||
retry = False
|
||
elif a.startswith("--slug"):
|
||
raw = a.split("=", 1)[1] if "=" in a else argv[i + 1]
|
||
i += 1 if "=" not in a else 0
|
||
only = set(raw.split(","))
|
||
elif key is None:
|
||
key = a
|
||
i += 1
|
||
if key is None:
|
||
print(__doc__)
|
||
sys.exit(1)
|
||
return key, timeout, only, retry
|
||
|
||
|
||
def main():
|
||
key, timeout, only, retry = parse_args(sys.argv)
|
||
me, err = mc.fetch_me(key, timeout)
|
||
if me is None:
|
||
print(f"❌ 拉 /api/v1/me 失败:{err}")
|
||
print(" 拉不到目录 = 探针不知道要验什么(不是上游没问题),退出")
|
||
sys.exit(2)
|
||
|
||
items = me.get("apix") or []
|
||
todo = [it for it in items if it.get("enabled")]
|
||
if only:
|
||
todo = [it for it in todo if it.get("slug") in only]
|
||
disabled = [it.get("slug") for it in items if not it.get("enabled")]
|
||
|
||
print(f"🌐 {mc.LM} ⏱ timeout={timeout}s 📦 apix 条目 {len(todo)} 个待探"
|
||
+ (f"(停用跳过:{', '.join(map(str, disabled))})" if disabled else ""))
|
||
print("-" * 66)
|
||
|
||
results = []
|
||
for it in todo:
|
||
print(f"⏳ {it.get('slug'):<10} ... ", end="", flush=True)
|
||
r = probe(it, key, timeout)
|
||
if retry and r["status"] in (mc.UPSTREAM, mc.GW_WARN) and r["code"] in RETRYABLE:
|
||
time.sleep(RETRY_WAIT_SEC)
|
||
r2 = probe(it, key, timeout)
|
||
if r2["status"] not in (mc.UPSTREAM, mc.GW_WARN):
|
||
r2["retried"] = True
|
||
r = r2
|
||
results.append(r)
|
||
icon = mc.ICONS[r["status"]]
|
||
mark = "鉴权✓" if r["auth_ok"] else f"鉴权✗(HTTP {r['auth_code']})"
|
||
msg = f"{icon} [{r['mode']:<11}] {r['slug']:<10} {r['elapsed']:6.2f}s HTTP {r['code']} {mark}"
|
||
if r.get("retried"):
|
||
msg += " (重试后)"
|
||
if r["why"]:
|
||
msg += f" | {r['why']}"
|
||
print(msg)
|
||
if r["code"] != 0 and r["status"] != mc.PASS and r["raw"]:
|
||
print(f" └ {r['raw']}")
|
||
|
||
if not results:
|
||
print("· 目录里没有 enabled 的 apix 条目 —— 无事可做")
|
||
sys.exit(0)
|
||
sys.exit(mc.summarize(results, "apix 探针"))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|