两件事: 1) 验证脚本统一从 /api/v1/me 拉能力目录(新增 tests/me_catalog.py 共享模块)。 之前 slot×模型、apix 条目写死在脚本里——配置是管理台动态的,改配置后探针 要么漏验新上游、要么拿旧模型名误报。现在 test_models 按 upstreams×models 全展开(both 双格式并发),apix_check 按目录逐条探;拉不到目录退出码 2 (验证失效 ≠ 上游没问题,不许假绿)。 三态口径:❌ 网关侧故障(错误体归属 + HTTP 码 + error.code 值三重判据—— 智谱 401 体也是 {"error":{code,message}} 形状,光看形态会把欠费误判成发版 故障);⚠️ 上游/厂商侧(欠费 402、不可达 502/504、WAF、200 软错误) **如实汇总反馈但不染红发版**;网关 429 限流按 GW_WARN 处理。 live_check.sh 退出码语义固定:0 过 / 1 网关侧故障 / 2 验证不可用。 分类器 15 个边界场景离线单测全对(撞形状/码集合/软错误/不误伤正常 200)。 2) agent.sh release 改 4/4 步:验证失败不再停在原地—— 抓 PRE_IMG + update job 的备份路径 → live_check → rc=1 时自动 POST /rollback {tag, restore_backup} 回滚镜像并恢复 data/audit,回滚后 再跑 verify 自检;rc=2(验证不可用)**不回滚**(盲回滚比带病观察危险)。 ipass-agent 1.0.2:rollback 本地无镜像自动 pull(自动路径不许卡「先手动拉」); 新增 restore_data——先 stop 容器(防周期落盘把污染写回)、备份 tar 解包顶替 data/audit、现目录改名 .broken-<ts> 留证、只认 ~/ipass/backup/ 下的 tar。 restore_data 本地单测过(恢复/留证/staging 清理/越界拒绝)。 release 开头加控制面版本闸门:<1.0.2 会静默忽略 restore_backup → 中止 (IPASS_FORCE_RELEASE=1 可跳过),⚠️ 线上控制面还是 1.0.1,发版前先上 .11 跑 §12.5 自更新。 附带修 bash 老坑:`$VAR` 紧跟全角字符被吞进变量名(unbound variable, live_check A 路径实测炸出),统一 ${VAR}。 线上实测:live_check 三步全绿(LLM 10/10 目录全展开 + apix 3/3 + 验收)。 只动 tests/deploy,不进镜像;ipass-agent 属控制面(跑宿主),需单独自更新。
225 lines
9.0 KiB
Python
225 lines
9.0 KiB
Python
#!/usr/bin/env python3
|
||
"""LLM 真实流量探针 —— 用例目录从 /api/v1/me 动态拉,挨个真调。
|
||
|
||
历史上这里写死过一组 (slot, model),问题:slot/模型名/双协议地址都是管理台
|
||
**动态配置**,改配置后探针要么静默漏验新上游、要么拿着旧模型名误报。1.0.16 起
|
||
`/api/v1/me` 一处下发全能力目录,探针跟着目录走(共享逻辑在 me_catalog.py)。
|
||
|
||
判定是三态(口径同 me_catalog.py 文件头):
|
||
✅ 2xx 真通;❌ 网关侧故障(发版要抓,退出码 1);
|
||
⚠️ 上游/厂商侧问题(欠费 402、不可达 502/504、厂商鉴权/限流、探针参数被拒)
|
||
—— **如实反馈但不染红发版**,网关工作正常、账号没钱不是镜像的锅。
|
||
|
||
用法:
|
||
python3 test_models.py <api_key> # 目录全展开,both + 并发
|
||
python3 test_models.py <api_key> --api openai # 只调 OpenAI 格式
|
||
python3 test_models.py <api_key> --model deepseek-flash,GLM-5.3-Flash
|
||
python3 test_models.py <api_key> --list-models # 只验各 slot 的 /v1/models
|
||
python3 test_models.py <api_key> --timeout 60 --sequential
|
||
|
||
退出码:有 ❌ → 1;只有 ✅/⚠️ → 0;拉不到目录 → 2(验证本身失效,别假绿)。
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
import me_catalog as mc # noqa: E402
|
||
|
||
PROMPT = "用一句话介绍你自己。"
|
||
API_KINDS = ("anthropic", "openai", "both")
|
||
|
||
|
||
def mask_key(k: str) -> str:
|
||
if len(k) <= 12:
|
||
return "*" * len(k)
|
||
return k[:6] + "..." + k[-4:]
|
||
|
||
|
||
def build_payload(api_kind: str, model: str) -> bytes:
|
||
if api_kind == "anthropic":
|
||
body = {"model": model, "max_tokens": 256,
|
||
"messages": [{"role": "user", "content": PROMPT}]}
|
||
else:
|
||
body = {"model": model, "max_tokens": 256,
|
||
"messages": [{"role": "user", "content": PROMPT}]}
|
||
return json.dumps(body).encode("utf-8")
|
||
|
||
|
||
def extra_headers(api_kind: str, key: str) -> dict:
|
||
# anthropic 走 x-api-key + 版本头;me_catalog.http 默认发 Bearer,这里补差异
|
||
if api_kind == "anthropic":
|
||
return {"x-api-key": key, "anthropic-version": "2023-06-01"}
|
||
return {}
|
||
|
||
|
||
def call_one(api_key: str, case: dict, api_kind: str, timeout: int) -> dict:
|
||
url = case["anthropic"] if api_kind == "anthropic" else case["openai"]
|
||
headers = {"User-Agent": mc.DEFAULT_UA, "Content-Type": "application/json"}
|
||
headers.update(extra_headers(api_kind, api_key))
|
||
if api_kind != "anthropic":
|
||
headers["Authorization"] = f"Bearer {api_key}"
|
||
req = urllib.request.Request(url, data=build_payload(api_kind, case["model"]),
|
||
headers=headers, method="POST")
|
||
start = time.time()
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||
code = resp.status
|
||
raw = resp.read().decode("utf-8", "replace")
|
||
elapsed = time.time() - start
|
||
except urllib.error.HTTPError as e:
|
||
code = e.code
|
||
raw = e.read().decode("utf-8", "replace") if e.fp else ""
|
||
elapsed = time.time() - start
|
||
except Exception as e: # 超时 / DNS / TLS → classify_llm 会归为网关侧
|
||
code, raw, elapsed = 0, str(e), time.time() - start
|
||
status, why = mc.classify_llm(code, raw)
|
||
if status == mc.PASS:
|
||
# HTTP 200 ≠ 成功(实测:智谱把 NPE 软包成 200 + body 业务 500)——必须查 body
|
||
soft = mc.llm_soft_error(api_kind, raw)
|
||
if soft:
|
||
status, why = mc.UPSTREAM, f"厂商 200 软错误(链路通但没干活):{soft[:90]}"
|
||
text = ""
|
||
if status == mc.PASS:
|
||
try:
|
||
d = json.loads(raw)
|
||
if api_kind == "anthropic":
|
||
text = "".join(b.get("text", "") for b in d.get("content") or []
|
||
if isinstance(b, dict))
|
||
else:
|
||
ch = d.get("choices") or []
|
||
text = (ch[0].get("message") or {}).get("content", "") if ch else ""
|
||
except Exception:
|
||
pass
|
||
return {"name": f"{api_kind} {case['slot']}/{case['model'] or '(无模型名)'}",
|
||
"slot": case["slot"], "model": case["model"], "api": api_kind,
|
||
"code": code, "status": status, "why": why,
|
||
"elapsed": elapsed, "text": text.strip(), "raw": raw[:160].replace("\n", " ")}
|
||
|
||
|
||
def check_models_list(api_key: str, cases: list, timeout: int) -> list:
|
||
"""验每个 slot 的 /v1/models(模型发现走网关本地应答,不碰厂商)。"""
|
||
seen, results = set(), []
|
||
for c in cases:
|
||
if c["slot"] in seen:
|
||
continue
|
||
seen.add(c["slot"])
|
||
code, raw, elapsed = mc.http("GET", c["models_list"], api_key, timeout)
|
||
status, why = mc.classify_llm(code, raw)
|
||
ids = []
|
||
if status == mc.PASS:
|
||
try:
|
||
ids = [it.get("id") for it in (json.loads(raw).get("data") or [])]
|
||
except Exception:
|
||
why = "响应不是 OpenAI 列表形状"
|
||
status = mc.GW_FAIL
|
||
results.append({"name": f"models {c['slot']}", "slot": c["slot"],
|
||
"code": code, "status": status, "why": why, "elapsed": elapsed,
|
||
"ids": ids})
|
||
return results
|
||
|
||
|
||
def print_line(r: dict):
|
||
icon = mc.ICONS[r["status"]]
|
||
msg = f"{icon} {r['name']:<44} {r['elapsed']:6.2f}s HTTP {r['code']}"
|
||
if r["status"] == mc.PASS and r.get("text"):
|
||
msg += f" | {r['text'].replace(chr(10), ' ')[:40]}"
|
||
if r["why"]:
|
||
msg += f" | {r['why'][:80]}"
|
||
print(msg, flush=True)
|
||
|
||
|
||
def parse_args(argv):
|
||
api_key, timeout, only = None, 60, None
|
||
api_kind, concurrent, list_models = "both", True, False
|
||
i = 1
|
||
while i < len(argv):
|
||
a = argv[i]
|
||
if a == "--concurrent":
|
||
concurrent = True
|
||
elif a == "--sequential":
|
||
concurrent = False
|
||
elif a == "--list-models":
|
||
list_models = True
|
||
elif a.startswith("--api"):
|
||
v = a.split("=", 1)[1] if "=" in a else argv[i + 1]
|
||
if v not in API_KINDS:
|
||
print(f"--api 必须是 {API_KINDS} 之一"); sys.exit(2)
|
||
api_kind = v
|
||
i += 0 if "=" in a else 1
|
||
elif a.startswith("--timeout"):
|
||
timeout = int(a.split("=", 1)[1]) if "=" in a else int(argv[i + 1])
|
||
i += 0 if "=" in a else 1
|
||
elif a.startswith("--model"):
|
||
raw = a.split("=", 1)[1] if "=" in a else argv[i + 1]
|
||
i += 0 if "=" in a else 1
|
||
only = set(raw.split(","))
|
||
elif api_key is None:
|
||
api_key = a
|
||
i += 1
|
||
if api_key is None:
|
||
print(__doc__)
|
||
sys.exit(1)
|
||
return api_key, api_kind, timeout, only, concurrent, list_models
|
||
|
||
|
||
def main():
|
||
api_key, api_kind, timeout, only, concurrent, list_models = parse_args(sys.argv)
|
||
|
||
# 1) 拉能力目录(拉不到就退 2 —— 探针失效不能假绿,也别拿写死数据凑)
|
||
me, err = mc.fetch_me(api_key, timeout)
|
||
if me is None:
|
||
print(f"❌ 拉 /api/v1/me 失败:{err}")
|
||
print(" 目录拉不到 = 探针不知道要验什么,退出(不是上游问题)")
|
||
sys.exit(2)
|
||
cases = mc.llm_cases(me)
|
||
if only:
|
||
cases = [c for c in cases if c["model"] in only]
|
||
skipped = [u.get("slot") for u in me.get("upstreams") or [] if not u.get("enabled")]
|
||
|
||
print(f"🔑 key={mask_key(api_key)} 📦 目录={len(cases)} 个用例(上游 {len(me.get('upstreams') or [])} 个"
|
||
+ (f",停用跳过 {','.join(map(str, skipped))}" if skipped else "")
|
||
+ f") ⏱ timeout={timeout}s 🔌 {api_kind} ⚙ {'并发' if concurrent else '串行'}")
|
||
print("-" * 66)
|
||
|
||
if not cases:
|
||
print("· 目录里没有可验的上游/模型(全部停用或 models 为空)—— 请核对管理台配置")
|
||
sys.exit(0)
|
||
|
||
# 2) /v1/models 模型发现(--list-models 时只跑这个;成本低、不碰厂商)
|
||
results = []
|
||
if list_models:
|
||
ml_results = check_models_list(api_key, cases, timeout)
|
||
for r in ml_results:
|
||
print_line(r)
|
||
if r.get("ids"):
|
||
print(f" models: {', '.join(str(x) for x in r['ids'])}")
|
||
sys.exit(mc.summarize(ml_results, "模型发现"))
|
||
|
||
kinds = (["openai", "anthropic"] if api_kind == "both" else [api_kind])
|
||
combos = [(k, c) for k in kinds for c in cases]
|
||
|
||
if concurrent:
|
||
with ThreadPoolExecutor(max_workers=min(8, len(combos))) as ex:
|
||
futs = {ex.submit(call_one, api_key, c, k, timeout): (k, c) for k, c in combos}
|
||
for fut in as_completed(futs):
|
||
r = fut.result()
|
||
results.append(r)
|
||
print_line(r)
|
||
else:
|
||
for k, c in combos:
|
||
r = call_one(api_key, c, k, timeout)
|
||
results.append(r)
|
||
print_line(r)
|
||
|
||
sys.exit(mc.summarize(results, "LLM 真实调用"))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|