Files
ipass__ipass/tests/test_models.py
T
admin 81f83f6f50 test: 加 tests/live_check.sh —— 上线后验证(真实流量 + 环境验收)
回归(tests/run.sh)打的是假上游,deploy/agent.sh verify 又只验管理台不发真实
LLM 请求,两半都跑不到「新版在线上真能干活」。新脚本把两者合成一条命令:

- 真实流量:test_models.py 4 slot × OpenAI/Anthropic + /v1/models;
- 环境验收:deploy/agent.sh verify(登录 / 页面标记 / subdomain_rule / 趋势 / 账号 load)。
- --models-only / --verify-only 可只跑一半;key 文件默认 ~/ssw.txt。

顺带:
- test_models.py 补退出码(原来失败也退 0,包装脚本判不出失败);
- key 文件解析支持 `sk=<key>` 与 # 注释行(取整行会拿到 sk=sk_xxx → 全量 401);
- tests/README.md 断言数同步到 108 + 262 = 370,补两个脚本说明。

实测:8/8 通过 + 验收全绿;错 key / 缺 key 文件均退出 1。
2026-09-17 07:37:08 +08:00

317 lines
11 KiB
Python

#!/usr/bin/env python3
"""
测试一组兼容接口下的模型是否可用(同时支持 Anthropic + OpenAI 两种格式)。
URL 结构:
Anthropic: https://{BASE}/{provider}/anthropic/v1/messages
OpenAI: https://{BASE}/{provider}/v1/chat/completions
provider = deepseek/qwen/glm/minimax(网关按厂商路由)
model = body 里的 model 字段
用法:
python test_models.py <api_key>
python test_models.py <api_key> --api openai
python test_models.py <api_key> --api both
python test_models.py <api_key> --concurrent
python test_models.py <api_key> --model deepseek-flash,MiniMax-M3
python test_models.py <api_key> --timeout 60
python test_models.py <api_key> --list-models
python test_models.py <api_key> --list-models --api openai
"""
import json
import time
import sys
import urllib.request
import urllib.error
from concurrent.futures import ThreadPoolExecutor, as_completed
BASE_URL = "https://lm.mayishangshu.cn:88"
PROMPT = "用一句话介绍你自己。"
MODELS = [
("deepseek", "deepseek-flash"),
("minimax", "MiniMax-M3"),
("qwen", "qwen3.8-flash"),
("glm", "GLM-5.3-Flash"),
]
PROBE_MODELS = [
("deepseek", "deepseek-chat"),
("qwen", "qwen-plus"),
("glm", "glm-4-flash"),
]
API_KINDS = ("anthropic", "openai", "both")
def call_anthropic(api_key: str, provider: str, model: str, timeout: int) -> dict:
url = f"{BASE_URL}/{provider}/anthropic/v1/messages"
headers = {
"x-api-key": api_key,
"anthropic-version": "2023-06-01",
"content-type": "application/json",
}
payload = {
"model": model,
"max_tokens": 256,
"messages": [{"role": "user", "content": PROMPT}],
}
body = json.dumps(payload).encode("utf-8")
req = urllib.request.Request(url, data=body, headers=headers, method="POST")
start = time.time()
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
raw = resp.read().decode("utf-8")
elapsed = time.time() - start
try:
parsed = json.loads(raw)
except json.JSONDecodeError:
return {"ok": False, "elapsed": elapsed, "error": f"invalid JSON: {raw[:120]}"}
text = ""
for block in parsed.get("content", []) or []:
if isinstance(block, dict) and block.get("type") == "text":
text += block.get("text", "")
return {
"ok": True,
"elapsed": elapsed,
"status": resp.status,
"text": text.strip(),
"usage": parsed.get("usage", {}),
}
except urllib.error.HTTPError as e:
elapsed = time.time() - start
err_body = e.read().decode("utf-8", errors="replace") if e.fp else ""
return {"ok": False, "elapsed": elapsed, "status": e.code, "error": err_body[:200]}
except Exception as e:
elapsed = time.time() - start
return {"ok": False, "elapsed": elapsed, "error": str(e)}
def call_openai(api_key: str, provider: str, model: str, timeout: int) -> dict:
url = f"{BASE_URL}/{provider}/v1/chat/completions"
headers = {
"Authorization": f"Bearer {api_key}",
"content-type": "application/json",
}
payload = {
"model": model,
"max_tokens": 256,
"messages": [{"role": "user", "content": PROMPT}],
}
body = json.dumps(payload).encode("utf-8")
req = urllib.request.Request(url, data=body, headers=headers, method="POST")
start = time.time()
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
raw = resp.read().decode("utf-8")
elapsed = time.time() - start
try:
parsed = json.loads(raw)
except json.JSONDecodeError:
return {"ok": False, "elapsed": elapsed, "error": f"invalid JSON: {raw[:120]}"}
text = ""
choices = parsed.get("choices") or []
if choices:
msg = choices[0].get("message") or {}
text = msg.get("content", "")
return {
"ok": True,
"elapsed": elapsed,
"status": resp.status,
"text": (text or "").strip(),
"usage": parsed.get("usage", {}),
}
except urllib.error.HTTPError as e:
elapsed = time.time() - start
err_body = e.read().decode("utf-8", errors="replace") if e.fp else ""
return {"ok": False, "elapsed": elapsed, "status": e.code, "error": err_body[:200]}
except Exception as e:
elapsed = time.time() - start
return {"ok": False, "elapsed": elapsed, "error": str(e)}
def call(api_kind: str, api_key: str, provider: str, model: str, timeout: int) -> dict:
if api_kind == "openai":
return call_openai(api_key, provider, model, timeout)
return call_anthropic(api_key, provider, model, timeout)
def mask_key(k: str) -> str:
if len(k) <= 12:
return "*" * len(k)
return k[:6] + "..." + k[-4:]
def parse_args(argv):
api_key = None
concurrent = False
timeout = 30
only = None
probe = False
list_models = False
api_kind = "anthropic"
i = 1
while i < len(argv):
a = argv[i]
if a == "--concurrent":
concurrent = True
elif a == "--probe":
probe = True
elif a == "--list-models":
list_models = True
elif a.startswith("--api"):
v = a.split("=", 1)[1] if "=" in a else argv[i + 1]
if "=" not in a:
i += 1
if v not in API_KINDS:
print(f"--api 必须是 {API_KINDS} 之一"); sys.exit(2)
api_kind = v
elif a.startswith("--timeout"):
timeout = int(a.split("=", 1)[1]) if "=" in a else int(argv[i + 1]); i += 1 if "=" not in a else 0
elif a.startswith("--model"):
raw = a.split("=", 1)[1] if "=" in a else argv[i + 1]; i += 1 if "=" not in a else 0
only = set(raw.split(","))
elif api_key is None:
api_key = a
i += 1
if api_key is None:
print(__doc__)
sys.exit(1)
return api_key, concurrent, timeout, only, probe, list_models, api_kind
def list_available(api_key: str, provider: str, api_kind: str, timeout: int) -> dict:
"""试 GET /v1/models 看网关曝不曝模型列表"""
if api_kind == "openai":
paths = (
f"{BASE_URL}/{provider}/v1/models",
f"{BASE_URL}/v1/models",
)
headers = {"Authorization": f"Bearer {api_key}"}
else:
paths = (
f"{BASE_URL}/{provider}/anthropic/v1/models",
f"{BASE_URL}/{provider}/v1/models",
f"{BASE_URL}/v1/models",
)
headers = {"x-api-key": api_key, "anthropic-version": "2023-06-01"}
for path in paths:
try:
req = urllib.request.Request(path, headers=headers)
with urllib.request.urlopen(req, timeout=timeout) as resp:
return {"ok": True, "path": path, "body": resp.read().decode("utf-8")}
except urllib.error.HTTPError as e:
last_err = (e.code, e.read().decode("utf-8", errors="replace")[:120])
except Exception as e:
last_err = ("?", str(e)[:120])
return {"ok": False, "error": last_err}
def main():
api_key, concurrent, timeout, only, probe, list_models, api_kind = parse_args(sys.argv)
if list_models:
for kind in (["openai"] if api_kind == "openai" else (API_KINDS if api_kind == "both" else ["anthropic"])):
print(f"🔑 key={mask_key(api_key)} 🔌 {kind:<9} 探测模型列表...")
print("-" * 60)
for p in ("deepseek", "qwen", "glm", "minimax"):
print(f"⏳ {p:<10} ... ", end="", flush=True)
r = list_available(api_key, p, kind, timeout)
if r["ok"]:
print(f"✅ {r['path']}")
try:
parsed = json.loads(r["body"])
ids = [it["id"] for it in (parsed.get("data") or []) if isinstance(it, dict) and "id" in it]
if ids:
print(f" models: {', '.join(ids)}")
else:
print(f" raw: {r['body'][:300]}")
except Exception:
print(f" raw: {r['body'][:300]}")
else:
print(f"❌ {r['error']}")
print()
return
# 确定要跑哪些 (api_kind, provider, model) 组合
combos = []
kinds = ["openai"] if api_kind == "openai" else (API_KINDS[:-1] if api_kind == "both" else ["anthropic"])
todo = MODELS
if probe:
todo = list(MODELS) + PROBE_MODELS
if only:
todo = [(p, m) for (p, m) in todo if m in only]
if not todo:
print("没有匹配的模型"); sys.exit(1)
for kind in kinds:
for p, m in todo:
combos.append((kind, p, m))
print(f"🔑 key={mask_key(api_key)} 🌐 {BASE_URL} ⏱ timeout={timeout}s "
f"⚙ {'concurrent' if concurrent else 'sequential'} "
f"🔌 {api_kind}{' 🔎probe' if probe else ''}")
print(f"📝 prompt: {PROMPT}")
print("-" * 60)
results = []
if concurrent:
with ThreadPoolExecutor(max_workers=min(8, len(combos))) as ex:
futs = {ex.submit(call, k, api_key, p, m, timeout): (k, p, m) for (k, p, m) in combos}
for fut in as_completed(futs):
k, p, m = futs[fut]
r = fut.result()
r.update({"api": k, "provider": p, "model": m})
results.append(r)
print_line(r)
else:
for k, p, m in combos:
print(f"⏳ [{k:<9}] {p}/{m:<16} ... ", end="", flush=True)
r = call(k, api_key, p, m, timeout)
r.update({"api": k, "provider": p, "model": m})
results.append(r)
print_line(r, newline=True)
print("-" * 60)
ok = sum(1 for r in results if r["ok"])
print(f"📊 {ok}/{len(results)} OK")
# 汇总:按 api 分组
by_api = {}
for r in results:
by_api.setdefault(r["api"], {"ok": 0, "total": 0})
by_api[r["api"]]["total"] += 1
if r["ok"]:
by_api[r["api"]]["ok"] += 1
if len(by_api) > 1:
summary = " ".join(f"{k}:{v['ok']}/{v['total']}" for k, v in by_api.items())
print(f" by api: {summary}")
# 退出码:全绿 0,有失败 1 —— 供 tests/live_check.sh 之类的包装脚本判定
sys.exit(0 if ok == len(results) else 1)
def print_line(r: dict, newline: bool = False):
tag = f"[{r.get('api','?'):<9}] {r.get('provider','?')}/{r.get('model','?')}"
if r["ok"]:
msg = f"✅ {tag:<32} {r['elapsed']:6.2f}s"
if r.get("text"):
preview = r["text"].replace("\n", " ")[:40]
msg += f" | {preview}"
else:
err = (r.get("error") or "unknown").replace("\n", " ")[:60]
msg = f"❌ {tag:<32} {r['elapsed']:6.2f}s | {err}"
if newline:
print(msg)
else:
print(msg)
if __name__ == "__main__":
main()