Files
ipass__ipass/tests/test_models.py
T
admin 6081151410 fix(release): 发版链路三处缺陷(1.1.1 发版误回滚暴露)
1. me_catalog.py 把 .env 约定的**裸主机名**当完整 URL 用:release 里
   agent.sh source .env 后子进程继承,live_check 两步探针 ValueError 崩退 1,
   被三态口径判成「网关侧故障」触发自动回滚(1.1.1 实测,健康版本差点被回)。
   现在补 scheme + IPASS_PUBLIC_PORT;探针脚本未捕获异常统一退 2(验证不可用),
   崩溃永远不该触发回滚。
2. rollback 参数键位漂移:远端控制面对 release 发的 {"tag":...} 报「需要显式传
   tag」→ 自动回滚 job 失败。agent.sh 的 rollback/回滚体改 python 拼 JSON 并
   tag/image 双写;ipass-agent 1.0.4 的 op_rollback 两种键都认、短 tag/裸版本
   用受管容器仓库段补全;release 版本闸门抬到 1.0.4(自动回滚必须先证明可用)。
3. 线上默认 tag 迁出 .env → deploy/image.conf(非秘密、入库):release 成功
   自动回写 + commit,不再需要人肉同步;.env 的 IPASS_IMAGE_TAG 仅临时覆盖。
   另修 agent.sh upstream-test:旧契约(slot 塞 body、无探测体)换成面板同款
   (query slot + 第一个登记模型),实测 minimax 200。
2026-09-17 18:52:22 +08:00

234 lines
9.6 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""LLM 真实流量探针 —— 用例目录从 /api/v1/me 动态拉,挨个真调。
历史上这里写死过一组 (slot, model),问题:slot/模型名/双协议地址都是管理台
**动态配置**,改配置后探针要么静默漏验新上游、要么拿着旧模型名误报。1.0.16 起
`/api/v1/me` 一处下发全能力目录,探针跟着目录走(共享逻辑在 me_catalog.py)。
判定是三态(口径同 me_catalog.py 文件头):
✅ 2xx 真通;❌ 网关侧故障(发版要抓,退出码 1);
⚠️ 上游/厂商侧问题(欠费 402、不可达 502/504、厂商鉴权/限流、探针参数被拒)
—— **如实反馈但不染红发版**,网关工作正常、账号没钱不是镜像的锅。
用法:
python3 test_models.py <api_key> # 目录全展开,both + 并发
python3 test_models.py <api_key> --api openai # 只调 OpenAI 格式
python3 test_models.py <api_key> --model deepseek-flash,GLM-5.3-Flash
python3 test_models.py <api_key> --list-models # 只验各 slot 的 /v1/models
python3 test_models.py <api_key> --timeout 60 --sequential
退出码:有 ❌ → 1;只有 ✅/⚠️ → 0;拉不到目录 → 2(验证本身失效,别假绿)。
"""
import json
import os
import sys
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import me_catalog as mc # noqa: E402
PROMPT = "用一句话介绍你自己。"
API_KINDS = ("anthropic", "openai", "both")
def mask_key(k: str) -> str:
if len(k) <= 12:
return "*" * len(k)
return k[:6] + "..." + k[-4:]
def build_payload(api_kind: str, model: str) -> bytes:
if api_kind == "anthropic":
body = {"model": model, "max_tokens": 256,
"messages": [{"role": "user", "content": PROMPT}]}
else:
body = {"model": model, "max_tokens": 256,
"messages": [{"role": "user", "content": PROMPT}]}
return json.dumps(body).encode("utf-8")
def extra_headers(api_kind: str, key: str) -> dict:
# anthropic 走 x-api-key + 版本头;me_catalog.http 默认发 Bearer,这里补差异
if api_kind == "anthropic":
return {"x-api-key": key, "anthropic-version": "2023-06-01"}
return {}
def call_one(api_key: str, case: dict, api_kind: str, timeout: int) -> dict:
url = case["anthropic"] if api_kind == "anthropic" else case["openai"]
headers = {"User-Agent": mc.DEFAULT_UA, "Content-Type": "application/json"}
headers.update(extra_headers(api_kind, api_key))
if api_kind != "anthropic":
headers["Authorization"] = f"Bearer {api_key}"
req = urllib.request.Request(url, data=build_payload(api_kind, case["model"]),
headers=headers, method="POST")
start = time.time()
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
code = resp.status
raw = resp.read().decode("utf-8", "replace")
elapsed = time.time() - start
except urllib.error.HTTPError as e:
code = e.code
raw = e.read().decode("utf-8", "replace") if e.fp else ""
elapsed = time.time() - start
except Exception as e: # 超时 / DNS / TLS → classify_llm 会归为网关侧
code, raw, elapsed = 0, str(e), time.time() - start
status, why = mc.classify_llm(code, raw)
if status == mc.PASS:
# HTTP 200 ≠ 成功(实测:智谱把 NPE 软包成 200 + body 业务 500)——必须查 body
soft = mc.llm_soft_error(api_kind, raw)
if soft:
status, why = mc.UPSTREAM, f"厂商 200 软错误(链路通但没干活):{soft[:90]}"
text = ""
if status == mc.PASS:
try:
d = json.loads(raw)
if api_kind == "anthropic":
text = "".join(b.get("text", "") for b in d.get("content") or []
if isinstance(b, dict))
else:
ch = d.get("choices") or []
text = (ch[0].get("message") or {}).get("content", "") if ch else ""
except Exception:
pass
return {"name": f"{api_kind} {case['slot']}/{case['model'] or '(无模型名)'}",
"slot": case["slot"], "model": case["model"], "api": api_kind,
"code": code, "status": status, "why": why,
"elapsed": elapsed, "text": text.strip(), "raw": raw[:160].replace("\n", " ")}
def check_models_list(api_key: str, cases: list, timeout: int) -> list:
"""验每个 slot 的 /v1/models(模型发现走网关本地应答,不碰厂商)。"""
seen, results = set(), []
for c in cases:
if c["slot"] in seen:
continue
seen.add(c["slot"])
code, raw, elapsed = mc.http("GET", c["models_list"], api_key, timeout)
status, why = mc.classify_llm(code, raw)
ids = []
if status == mc.PASS:
try:
ids = [it.get("id") for it in (json.loads(raw).get("data") or [])]
except Exception:
why = "响应不是 OpenAI 列表形状"
status = mc.GW_FAIL
results.append({"name": f"models {c['slot']}", "slot": c["slot"],
"code": code, "status": status, "why": why, "elapsed": elapsed,
"ids": ids})
return results
def print_line(r: dict):
icon = mc.ICONS[r["status"]]
msg = f"{icon} {r['name']:<44} {r['elapsed']:6.2f}s HTTP {r['code']}"
if r["status"] == mc.PASS and r.get("text"):
msg += f" | {r['text'].replace(chr(10), ' ')[:40]}"
if r["why"]:
msg += f" | {r['why'][:80]}"
print(msg, flush=True)
def parse_args(argv):
api_key, timeout, only = None, 60, None
api_kind, concurrent, list_models = "both", True, False
i = 1
while i < len(argv):
a = argv[i]
if a == "--concurrent":
concurrent = True
elif a == "--sequential":
concurrent = False
elif a == "--list-models":
list_models = True
elif a.startswith("--api"):
v = a.split("=", 1)[1] if "=" in a else argv[i + 1]
if v not in API_KINDS:
print(f"--api 必须是 {API_KINDS} 之一"); sys.exit(2)
api_kind = v
i += 0 if "=" in a else 1
elif a.startswith("--timeout"):
timeout = int(a.split("=", 1)[1]) if "=" in a else int(argv[i + 1])
i += 0 if "=" in a else 1
elif a.startswith("--model"):
raw = a.split("=", 1)[1] if "=" in a else argv[i + 1]
i += 0 if "=" in a else 1
only = set(raw.split(","))
elif api_key is None:
api_key = a
i += 1
if api_key is None:
print(__doc__)
sys.exit(1)
return api_key, api_kind, timeout, only, concurrent, list_models
def main():
api_key, api_kind, timeout, only, concurrent, list_models = parse_args(sys.argv)
# 1) 拉能力目录(拉不到就退 2 —— 探针失效不能假绿,也别拿写死数据凑)
me, err = mc.fetch_me(api_key, timeout)
if me is None:
print(f"❌ 拉 /api/v1/me 失败:{err}")
print(" 目录拉不到 = 探针不知道要验什么,退出(不是上游问题)")
sys.exit(2)
cases = mc.llm_cases(me)
if only:
cases = [c for c in cases if c["model"] in only]
skipped = [u.get("slot") for u in me.get("upstreams") or [] if not u.get("enabled")]
print(f"🔑 key={mask_key(api_key)} 📦 目录={len(cases)} 个用例(上游 {len(me.get('upstreams') or [])} 个"
+ (f",停用跳过 {','.join(map(str, skipped))}" if skipped else "")
+ f") ⏱ timeout={timeout}s 🔌 {api_kind} ⚙ {'并发' if concurrent else '串行'}")
print("-" * 66)
if not cases:
print("· 目录里没有可验的上游/模型(全部停用或 models 为空)—— 请核对管理台配置")
sys.exit(0)
# 2) /v1/models 模型发现(--list-models 时只跑这个;成本低、不碰厂商)
results = []
if list_models:
ml_results = check_models_list(api_key, cases, timeout)
for r in ml_results:
print_line(r)
if r.get("ids"):
print(f" models: {', '.join(str(x) for x in r['ids'])}")
sys.exit(mc.summarize(ml_results, "模型发现"))
kinds = (["openai", "anthropic"] if api_kind == "both" else [api_kind])
combos = [(k, c) for k in kinds for c in cases]
if concurrent:
with ThreadPoolExecutor(max_workers=min(8, len(combos))) as ex:
futs = {ex.submit(call_one, api_key, c, k, timeout): (k, c) for k, c in combos}
for fut in as_completed(futs):
r = fut.result()
results.append(r)
print_line(r)
else:
for k, c in combos:
r = call_one(api_key, c, k, timeout)
results.append(r)
print_line(r)
sys.exit(mc.summarize(results, "LLM 真实调用"))
if __name__ == "__main__":
try:
main()
except SystemExit:
raise
except Exception as e: # noqa: BLE001
# 脚本自身/环境异常(拼 URL 崩、缺依赖…)= **验证手段不可用(2)**。
# 未捕获栈会退 1,被 live_check 判成网关侧故障触发自动回滚——1.1.1 发版
# 实测踩过(me_catalog 拿裸主机名当 URL 崩,差点把健康版本回滚掉)。
print(f"✗ 探针脚本自身异常,按「验证不可用」处理(不触发回滚):{type(e).__name__}: {e}", file=sys.stderr)
sys.exit(2)