#!/usr/bin/env python3 """LLM 真实流量探针 —— 用例目录从 /api/v1/me 动态拉,挨个真调。 历史上这里写死过一组 (slot, model),问题:slot/模型名/双协议地址都是管理台 **动态配置**,改配置后探针要么静默漏验新上游、要么拿着旧模型名误报。1.0.16 起 `/api/v1/me` 一处下发全能力目录,探针跟着目录走(共享逻辑在 me_catalog.py)。 判定是三态(口径同 me_catalog.py 文件头): ✅ 2xx 真通;❌ 网关侧故障(发版要抓,退出码 1); ⚠️ 上游/厂商侧问题(欠费 402、不可达 502/504、厂商鉴权/限流、探针参数被拒) —— **如实反馈但不染红发版**,网关工作正常、账号没钱不是镜像的锅。 用法: python3 test_models.py # 目录全展开,both + 并发 python3 test_models.py --api openai # 只调 OpenAI 格式 python3 test_models.py --model deepseek-flash,GLM-5.3-Flash python3 test_models.py --list-models # 只验各 slot 的 /v1/models python3 test_models.py --timeout 60 --sequential 退出码:有 ❌ → 1;只有 ✅/⚠️ → 0;拉不到目录 → 2(验证本身失效,别假绿)。 """ import json import os import sys import time import urllib.error import urllib.request from concurrent.futures import ThreadPoolExecutor, as_completed sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import me_catalog as mc # noqa: E402 PROMPT = "用一句话介绍你自己。" API_KINDS = ("anthropic", "openai", "both") def mask_key(k: str) -> str: if len(k) <= 12: return "*" * len(k) return k[:6] + "..." + k[-4:] def build_payload(api_kind: str, model: str) -> bytes: if api_kind == "anthropic": body = {"model": model, "max_tokens": 256, "messages": [{"role": "user", "content": PROMPT}]} else: body = {"model": model, "max_tokens": 256, "messages": [{"role": "user", "content": PROMPT}]} return json.dumps(body).encode("utf-8") def extra_headers(api_kind: str, key: str) -> dict: # anthropic 走 x-api-key + 版本头;me_catalog.http 默认发 Bearer,这里补差异 if api_kind == "anthropic": return {"x-api-key": key, "anthropic-version": "2023-06-01"} return {} def call_one(api_key: str, case: dict, api_kind: str, timeout: int) -> dict: url = case["anthropic"] if api_kind == "anthropic" else case["openai"] headers = {"User-Agent": mc.DEFAULT_UA, "Content-Type": "application/json"} headers.update(extra_headers(api_kind, api_key)) if api_kind != "anthropic": headers["Authorization"] = f"Bearer {api_key}" req = urllib.request.Request(url, data=build_payload(api_kind, case["model"]), headers=headers, method="POST") start = time.time() try: with urllib.request.urlopen(req, timeout=timeout) as resp: code = resp.status raw = resp.read().decode("utf-8", "replace") elapsed = time.time() - start except urllib.error.HTTPError as e: code = e.code raw = e.read().decode("utf-8", "replace") if e.fp else "" elapsed = time.time() - start except Exception as e: # 超时 / DNS / TLS → classify_llm 会归为网关侧 code, raw, elapsed = 0, str(e), time.time() - start status, why = mc.classify_llm(code, raw) if status == mc.PASS: # HTTP 200 ≠ 成功(实测:智谱把 NPE 软包成 200 + body 业务 500)——必须查 body soft = mc.llm_soft_error(api_kind, raw) if soft: status, why = mc.UPSTREAM, f"厂商 200 软错误(链路通但没干活):{soft[:90]}" text = "" if status == mc.PASS: try: d = json.loads(raw) if api_kind == "anthropic": text = "".join(b.get("text", "") for b in d.get("content") or [] if isinstance(b, dict)) else: ch = d.get("choices") or [] text = (ch[0].get("message") or {}).get("content", "") if ch else "" except Exception: pass return {"name": f"{api_kind} {case['slot']}/{case['model'] or '(无模型名)'}", "slot": case["slot"], "model": case["model"], "api": api_kind, "code": code, "status": status, "why": why, "elapsed": elapsed, "text": text.strip(), "raw": raw[:160].replace("\n", " ")} def check_models_list(api_key: str, cases: list, timeout: int) -> list: """验每个 slot 的 /v1/models(模型发现走网关本地应答,不碰厂商)。""" seen, results = set(), [] for c in cases: if c["slot"] in seen: continue seen.add(c["slot"]) code, raw, elapsed = mc.http("GET", c["models_list"], api_key, timeout) status, why = mc.classify_llm(code, raw) ids = [] if status == mc.PASS: try: ids = [it.get("id") for it in (json.loads(raw).get("data") or [])] except Exception: why = "响应不是 OpenAI 列表形状" status = mc.GW_FAIL results.append({"name": f"models {c['slot']}", "slot": c["slot"], "code": code, "status": status, "why": why, "elapsed": elapsed, "ids": ids}) return results def print_line(r: dict): icon = mc.ICONS[r["status"]] msg = f"{icon} {r['name']:<44} {r['elapsed']:6.2f}s HTTP {r['code']}" if r["status"] == mc.PASS and r.get("text"): msg += f" | {r['text'].replace(chr(10), ' ')[:40]}" if r["why"]: msg += f" | {r['why'][:80]}" print(msg, flush=True) def parse_args(argv): api_key, timeout, only = None, 60, None api_kind, concurrent, list_models = "both", True, False i = 1 while i < len(argv): a = argv[i] if a == "--concurrent": concurrent = True elif a == "--sequential": concurrent = False elif a == "--list-models": list_models = True elif a.startswith("--api"): v = a.split("=", 1)[1] if "=" in a else argv[i + 1] if v not in API_KINDS: print(f"--api 必须是 {API_KINDS} 之一"); sys.exit(2) api_kind = v i += 0 if "=" in a else 1 elif a.startswith("--timeout"): timeout = int(a.split("=", 1)[1]) if "=" in a else int(argv[i + 1]) i += 0 if "=" in a else 1 elif a.startswith("--model"): raw = a.split("=", 1)[1] if "=" in a else argv[i + 1] i += 0 if "=" in a else 1 only = set(raw.split(",")) elif api_key is None: api_key = a i += 1 if api_key is None: print(__doc__) sys.exit(1) return api_key, api_kind, timeout, only, concurrent, list_models def main(): api_key, api_kind, timeout, only, concurrent, list_models = parse_args(sys.argv) # 1) 拉能力目录(拉不到就退 2 —— 探针失效不能假绿,也别拿写死数据凑) me, err = mc.fetch_me(api_key, timeout) if me is None: print(f"❌ 拉 /api/v1/me 失败:{err}") print(" 目录拉不到 = 探针不知道要验什么,退出(不是上游问题)") sys.exit(2) cases = mc.llm_cases(me) if only: cases = [c for c in cases if c["model"] in only] skipped = [u.get("slot") for u in me.get("upstreams") or [] if not u.get("enabled")] print(f"🔑 key={mask_key(api_key)} 📦 目录={len(cases)} 个用例(上游 {len(me.get('upstreams') or [])} 个" + (f",停用跳过 {','.join(map(str, skipped))}" if skipped else "") + f") ⏱ timeout={timeout}s 🔌 {api_kind} ⚙ {'并发' if concurrent else '串行'}") print("-" * 66) if not cases: print("· 目录里没有可验的上游/模型(全部停用或 models 为空)—— 请核对管理台配置") sys.exit(0) # 2) /v1/models 模型发现(--list-models 时只跑这个;成本低、不碰厂商) results = [] if list_models: ml_results = check_models_list(api_key, cases, timeout) for r in ml_results: print_line(r) if r.get("ids"): print(f" models: {', '.join(str(x) for x in r['ids'])}") sys.exit(mc.summarize(ml_results, "模型发现")) kinds = (["openai", "anthropic"] if api_kind == "both" else [api_kind]) combos = [(k, c) for k in kinds for c in cases] if concurrent: with ThreadPoolExecutor(max_workers=min(8, len(combos))) as ex: futs = {ex.submit(call_one, api_key, c, k, timeout): (k, c) for k, c in combos} for fut in as_completed(futs): r = fut.result() results.append(r) print_line(r) else: for k, c in combos: r = call_one(api_key, c, k, timeout) results.append(r) print_line(r) sys.exit(mc.summarize(results, "LLM 真实调用")) if __name__ == "__main__": try: main() except SystemExit: raise except Exception as e: # noqa: BLE001 # 脚本自身/环境异常(拼 URL 崩、缺依赖…)= **验证手段不可用(2)**。 # 未捕获栈会退 1,被 live_check 判成网关侧故障触发自动回滚——1.1.1 发版 # 实测踩过(me_catalog 拿裸主机名当 URL 崩,差点把健康版本回滚掉)。 print(f"✗ 探针脚本自身异常,按「验证不可用」处理(不触发回滚):{type(e).__name__}: {e}", file=sys.stderr) sys.exit(2)