Files
ipass__ipass/tests/test_models.py
T
admin 762d308745 test+deploy: 探针改目录驱动 + 三态判定 + 发版失败自动回滚(镜像+数据)
两件事:

1) 验证脚本统一从 /api/v1/me 拉能力目录(新增 tests/me_catalog.py 共享模块)。
   之前 slot×模型、apix 条目写死在脚本里——配置是管理台动态的,改配置后探针
   要么漏验新上游、要么拿旧模型名误报。现在 test_models 按 upstreams×models
   全展开(both 双格式并发),apix_check 按目录逐条探;拉不到目录退出码 2
   (验证失效 ≠ 上游没问题,不许假绿)。
   三态口径:❌ 网关侧故障(错误体归属 + HTTP 码 + error.code 值三重判据——
   智谱 401 体也是 {"error":{code,message}} 形状,光看形态会把欠费误判成发版
   故障);⚠️ 上游/厂商侧(欠费 402、不可达 502/504、WAF、200 软错误)
   **如实汇总反馈但不染红发版**;网关 429 限流按 GW_WARN 处理。
   live_check.sh 退出码语义固定:0 过 / 1 网关侧故障 / 2 验证不可用。
   分类器 15 个边界场景离线单测全对(撞形状/码集合/软错误/不误伤正常 200)。

2) agent.sh release 改 4/4 步:验证失败不再停在原地——
   抓 PRE_IMG + update job 的备份路径 → live_check → rc=1 时自动
   POST /rollback {tag, restore_backup} 回滚镜像并恢复 data/audit,回滚后
   再跑 verify 自检;rc=2(验证不可用)**不回滚**(盲回滚比带病观察危险)。
   ipass-agent 1.0.2:rollback 本地无镜像自动 pull(自动路径不许卡「先手动拉」);
   新增 restore_data——先 stop 容器(防周期落盘把污染写回)、备份 tar 解包顶替
   data/audit、现目录改名 .broken-<ts> 留证、只认 ~/ipass/backup/ 下的 tar。
   restore_data 本地单测过(恢复/留证/staging 清理/越界拒绝)。
   release 开头加控制面版本闸门:<1.0.2 会静默忽略 restore_backup → 中止
   (IPASS_FORCE_RELEASE=1 可跳过),⚠️ 线上控制面还是 1.0.1,发版前先上 .11
   跑 §12.5 自更新。
   附带修 bash 老坑:`$VAR` 紧跟全角字符被吞进变量名(unbound variable,
   live_check A 路径实测炸出),统一 ${VAR}。

线上实测:live_check 三步全绿(LLM 10/10 目录全展开 + apix 3/3 + 验收)。
只动 tests/deploy,不进镜像;ipass-agent 属控制面(跑宿主),需单独自更新。
2026-09-17 08:41:48 +08:00

225 lines
9.0 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""LLM 真实流量探针 —— 用例目录从 /api/v1/me 动态拉,挨个真调。
历史上这里写死过一组 (slot, model),问题:slot/模型名/双协议地址都是管理台
**动态配置**,改配置后探针要么静默漏验新上游、要么拿着旧模型名误报。1.0.16 起
`/api/v1/me` 一处下发全能力目录,探针跟着目录走(共享逻辑在 me_catalog.py)。
判定是三态(口径同 me_catalog.py 文件头):
✅ 2xx 真通;❌ 网关侧故障(发版要抓,退出码 1);
⚠️ 上游/厂商侧问题(欠费 402、不可达 502/504、厂商鉴权/限流、探针参数被拒)
—— **如实反馈但不染红发版**,网关工作正常、账号没钱不是镜像的锅。
用法:
python3 test_models.py <api_key> # 目录全展开,both + 并发
python3 test_models.py <api_key> --api openai # 只调 OpenAI 格式
python3 test_models.py <api_key> --model deepseek-flash,GLM-5.3-Flash
python3 test_models.py <api_key> --list-models # 只验各 slot 的 /v1/models
python3 test_models.py <api_key> --timeout 60 --sequential
退出码:有 ❌ → 1;只有 ✅/⚠️ → 0;拉不到目录 → 2(验证本身失效,别假绿)。
"""
import json
import os
import sys
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import me_catalog as mc # noqa: E402
PROMPT = "用一句话介绍你自己。"
API_KINDS = ("anthropic", "openai", "both")
def mask_key(k: str) -> str:
if len(k) <= 12:
return "*" * len(k)
return k[:6] + "..." + k[-4:]
def build_payload(api_kind: str, model: str) -> bytes:
if api_kind == "anthropic":
body = {"model": model, "max_tokens": 256,
"messages": [{"role": "user", "content": PROMPT}]}
else:
body = {"model": model, "max_tokens": 256,
"messages": [{"role": "user", "content": PROMPT}]}
return json.dumps(body).encode("utf-8")
def extra_headers(api_kind: str, key: str) -> dict:
# anthropic 走 x-api-key + 版本头;me_catalog.http 默认发 Bearer,这里补差异
if api_kind == "anthropic":
return {"x-api-key": key, "anthropic-version": "2023-06-01"}
return {}
def call_one(api_key: str, case: dict, api_kind: str, timeout: int) -> dict:
url = case["anthropic"] if api_kind == "anthropic" else case["openai"]
headers = {"User-Agent": mc.DEFAULT_UA, "Content-Type": "application/json"}
headers.update(extra_headers(api_kind, api_key))
if api_kind != "anthropic":
headers["Authorization"] = f"Bearer {api_key}"
req = urllib.request.Request(url, data=build_payload(api_kind, case["model"]),
headers=headers, method="POST")
start = time.time()
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
code = resp.status
raw = resp.read().decode("utf-8", "replace")
elapsed = time.time() - start
except urllib.error.HTTPError as e:
code = e.code
raw = e.read().decode("utf-8", "replace") if e.fp else ""
elapsed = time.time() - start
except Exception as e: # 超时 / DNS / TLS → classify_llm 会归为网关侧
code, raw, elapsed = 0, str(e), time.time() - start
status, why = mc.classify_llm(code, raw)
if status == mc.PASS:
# HTTP 200 ≠ 成功(实测:智谱把 NPE 软包成 200 + body 业务 500)——必须查 body
soft = mc.llm_soft_error(api_kind, raw)
if soft:
status, why = mc.UPSTREAM, f"厂商 200 软错误(链路通但没干活):{soft[:90]}"
text = ""
if status == mc.PASS:
try:
d = json.loads(raw)
if api_kind == "anthropic":
text = "".join(b.get("text", "") for b in d.get("content") or []
if isinstance(b, dict))
else:
ch = d.get("choices") or []
text = (ch[0].get("message") or {}).get("content", "") if ch else ""
except Exception:
pass
return {"name": f"{api_kind} {case['slot']}/{case['model'] or '(无模型名)'}",
"slot": case["slot"], "model": case["model"], "api": api_kind,
"code": code, "status": status, "why": why,
"elapsed": elapsed, "text": text.strip(), "raw": raw[:160].replace("\n", " ")}
def check_models_list(api_key: str, cases: list, timeout: int) -> list:
"""验每个 slot 的 /v1/models(模型发现走网关本地应答,不碰厂商)。"""
seen, results = set(), []
for c in cases:
if c["slot"] in seen:
continue
seen.add(c["slot"])
code, raw, elapsed = mc.http("GET", c["models_list"], api_key, timeout)
status, why = mc.classify_llm(code, raw)
ids = []
if status == mc.PASS:
try:
ids = [it.get("id") for it in (json.loads(raw).get("data") or [])]
except Exception:
why = "响应不是 OpenAI 列表形状"
status = mc.GW_FAIL
results.append({"name": f"models {c['slot']}", "slot": c["slot"],
"code": code, "status": status, "why": why, "elapsed": elapsed,
"ids": ids})
return results
def print_line(r: dict):
icon = mc.ICONS[r["status"]]
msg = f"{icon} {r['name']:<44} {r['elapsed']:6.2f}s HTTP {r['code']}"
if r["status"] == mc.PASS and r.get("text"):
msg += f" | {r['text'].replace(chr(10), ' ')[:40]}"
if r["why"]:
msg += f" | {r['why'][:80]}"
print(msg, flush=True)
def parse_args(argv):
api_key, timeout, only = None, 60, None
api_kind, concurrent, list_models = "both", True, False
i = 1
while i < len(argv):
a = argv[i]
if a == "--concurrent":
concurrent = True
elif a == "--sequential":
concurrent = False
elif a == "--list-models":
list_models = True
elif a.startswith("--api"):
v = a.split("=", 1)[1] if "=" in a else argv[i + 1]
if v not in API_KINDS:
print(f"--api 必须是 {API_KINDS} 之一"); sys.exit(2)
api_kind = v
i += 0 if "=" in a else 1
elif a.startswith("--timeout"):
timeout = int(a.split("=", 1)[1]) if "=" in a else int(argv[i + 1])
i += 0 if "=" in a else 1
elif a.startswith("--model"):
raw = a.split("=", 1)[1] if "=" in a else argv[i + 1]
i += 0 if "=" in a else 1
only = set(raw.split(","))
elif api_key is None:
api_key = a
i += 1
if api_key is None:
print(__doc__)
sys.exit(1)
return api_key, api_kind, timeout, only, concurrent, list_models
def main():
api_key, api_kind, timeout, only, concurrent, list_models = parse_args(sys.argv)
# 1) 拉能力目录(拉不到就退 2 —— 探针失效不能假绿,也别拿写死数据凑)
me, err = mc.fetch_me(api_key, timeout)
if me is None:
print(f"❌ 拉 /api/v1/me 失败:{err}")
print(" 目录拉不到 = 探针不知道要验什么,退出(不是上游问题)")
sys.exit(2)
cases = mc.llm_cases(me)
if only:
cases = [c for c in cases if c["model"] in only]
skipped = [u.get("slot") for u in me.get("upstreams") or [] if not u.get("enabled")]
print(f"🔑 key={mask_key(api_key)} 📦 目录={len(cases)} 个用例(上游 {len(me.get('upstreams') or [])} 个"
+ (f",停用跳过 {','.join(map(str, skipped))}" if skipped else "")
+ f") ⏱ timeout={timeout}s 🔌 {api_kind} ⚙ {'并发' if concurrent else '串行'}")
print("-" * 66)
if not cases:
print("· 目录里没有可验的上游/模型(全部停用或 models 为空)—— 请核对管理台配置")
sys.exit(0)
# 2) /v1/models 模型发现(--list-models 时只跑这个;成本低、不碰厂商)
results = []
if list_models:
ml_results = check_models_list(api_key, cases, timeout)
for r in ml_results:
print_line(r)
if r.get("ids"):
print(f" models: {', '.join(str(x) for x in r['ids'])}")
sys.exit(mc.summarize(ml_results, "模型发现"))
kinds = (["openai", "anthropic"] if api_kind == "both" else [api_kind])
combos = [(k, c) for k in kinds for c in cases]
if concurrent:
with ThreadPoolExecutor(max_workers=min(8, len(combos))) as ex:
futs = {ex.submit(call_one, api_key, c, k, timeout): (k, c) for k, c in combos}
for fut in as_completed(futs):
r = fut.result()
results.append(r)
print_line(r)
else:
for k, c in combos:
r = call_one(api_key, c, k, timeout)
results.append(r)
print_line(r)
sys.exit(mc.summarize(results, "LLM 真实调用"))
if __name__ == "__main__":
main()