管理台「测试」按钮发的是空 body POST,智谱 4.x SDK 在 model 缺失时 NPE 兜底成 200 + body.error.code=500,被误读成「模型不存在」。 /_upstream_test location 加 proxy_set_body 注入最小探测体 (model=test/max_tokens=1/messages=hi),让上游真做一次最小生成, 费用与可观测性可控。tests 同步。 另收 tests/test_models.py:网关多模型 × OpenAI/Anthropic 双协议验证脚本。
314 lines
11 KiB
Python
314 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
测试一组兼容接口下的模型是否可用(同时支持 Anthropic + OpenAI 两种格式)。
|
|
|
|
URL 结构:
|
|
Anthropic: https://{BASE}/{provider}/anthropic/v1/messages
|
|
OpenAI: https://{BASE}/{provider}/v1/chat/completions
|
|
provider = deepseek/qwen/glm/minimax(网关按厂商路由)
|
|
model = body 里的 model 字段
|
|
|
|
用法:
|
|
python test_models.py <api_key>
|
|
python test_models.py <api_key> --api openai
|
|
python test_models.py <api_key> --api both
|
|
python test_models.py <api_key> --concurrent
|
|
python test_models.py <api_key> --model deepseek-flash,MiniMax-M3
|
|
python test_models.py <api_key> --timeout 60
|
|
python test_models.py <api_key> --list-models
|
|
python test_models.py <api_key> --list-models --api openai
|
|
"""
|
|
|
|
import json
|
|
import time
|
|
import sys
|
|
import urllib.request
|
|
import urllib.error
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
BASE_URL = "https://lm.mayishangshu.cn:88"
|
|
PROMPT = "用一句话介绍你自己。"
|
|
|
|
MODELS = [
|
|
("deepseek", "deepseek-flash"),
|
|
("minimax", "MiniMax-M3"),
|
|
("qwen", "qwen3.8-flash"),
|
|
("glm", "GLM-5.3-Flash"),
|
|
]
|
|
|
|
PROBE_MODELS = [
|
|
("deepseek", "deepseek-chat"),
|
|
("qwen", "qwen-plus"),
|
|
("glm", "glm-4-flash"),
|
|
]
|
|
|
|
API_KINDS = ("anthropic", "openai", "both")
|
|
|
|
|
|
def call_anthropic(api_key: str, provider: str, model: str, timeout: int) -> dict:
|
|
url = f"{BASE_URL}/{provider}/anthropic/v1/messages"
|
|
headers = {
|
|
"x-api-key": api_key,
|
|
"anthropic-version": "2023-06-01",
|
|
"content-type": "application/json",
|
|
}
|
|
payload = {
|
|
"model": model,
|
|
"max_tokens": 256,
|
|
"messages": [{"role": "user", "content": PROMPT}],
|
|
}
|
|
body = json.dumps(payload).encode("utf-8")
|
|
req = urllib.request.Request(url, data=body, headers=headers, method="POST")
|
|
|
|
start = time.time()
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
raw = resp.read().decode("utf-8")
|
|
elapsed = time.time() - start
|
|
try:
|
|
parsed = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
return {"ok": False, "elapsed": elapsed, "error": f"invalid JSON: {raw[:120]}"}
|
|
|
|
text = ""
|
|
for block in parsed.get("content", []) or []:
|
|
if isinstance(block, dict) and block.get("type") == "text":
|
|
text += block.get("text", "")
|
|
return {
|
|
"ok": True,
|
|
"elapsed": elapsed,
|
|
"status": resp.status,
|
|
"text": text.strip(),
|
|
"usage": parsed.get("usage", {}),
|
|
}
|
|
except urllib.error.HTTPError as e:
|
|
elapsed = time.time() - start
|
|
err_body = e.read().decode("utf-8", errors="replace") if e.fp else ""
|
|
return {"ok": False, "elapsed": elapsed, "status": e.code, "error": err_body[:200]}
|
|
except Exception as e:
|
|
elapsed = time.time() - start
|
|
return {"ok": False, "elapsed": elapsed, "error": str(e)}
|
|
|
|
|
|
def call_openai(api_key: str, provider: str, model: str, timeout: int) -> dict:
|
|
url = f"{BASE_URL}/{provider}/v1/chat/completions"
|
|
headers = {
|
|
"Authorization": f"Bearer {api_key}",
|
|
"content-type": "application/json",
|
|
}
|
|
payload = {
|
|
"model": model,
|
|
"max_tokens": 256,
|
|
"messages": [{"role": "user", "content": PROMPT}],
|
|
}
|
|
body = json.dumps(payload).encode("utf-8")
|
|
req = urllib.request.Request(url, data=body, headers=headers, method="POST")
|
|
|
|
start = time.time()
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
raw = resp.read().decode("utf-8")
|
|
elapsed = time.time() - start
|
|
try:
|
|
parsed = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
return {"ok": False, "elapsed": elapsed, "error": f"invalid JSON: {raw[:120]}"}
|
|
|
|
text = ""
|
|
choices = parsed.get("choices") or []
|
|
if choices:
|
|
msg = choices[0].get("message") or {}
|
|
text = msg.get("content", "")
|
|
return {
|
|
"ok": True,
|
|
"elapsed": elapsed,
|
|
"status": resp.status,
|
|
"text": (text or "").strip(),
|
|
"usage": parsed.get("usage", {}),
|
|
}
|
|
except urllib.error.HTTPError as e:
|
|
elapsed = time.time() - start
|
|
err_body = e.read().decode("utf-8", errors="replace") if e.fp else ""
|
|
return {"ok": False, "elapsed": elapsed, "status": e.code, "error": err_body[:200]}
|
|
except Exception as e:
|
|
elapsed = time.time() - start
|
|
return {"ok": False, "elapsed": elapsed, "error": str(e)}
|
|
|
|
|
|
def call(api_kind: str, api_key: str, provider: str, model: str, timeout: int) -> dict:
|
|
if api_kind == "openai":
|
|
return call_openai(api_key, provider, model, timeout)
|
|
return call_anthropic(api_key, provider, model, timeout)
|
|
|
|
|
|
def mask_key(k: str) -> str:
|
|
if len(k) <= 12:
|
|
return "*" * len(k)
|
|
return k[:6] + "..." + k[-4:]
|
|
|
|
|
|
def parse_args(argv):
|
|
api_key = None
|
|
concurrent = False
|
|
timeout = 30
|
|
only = None
|
|
probe = False
|
|
list_models = False
|
|
api_kind = "anthropic"
|
|
i = 1
|
|
while i < len(argv):
|
|
a = argv[i]
|
|
if a == "--concurrent":
|
|
concurrent = True
|
|
elif a == "--probe":
|
|
probe = True
|
|
elif a == "--list-models":
|
|
list_models = True
|
|
elif a.startswith("--api"):
|
|
v = a.split("=", 1)[1] if "=" in a else argv[i + 1]
|
|
if "=" not in a:
|
|
i += 1
|
|
if v not in API_KINDS:
|
|
print(f"--api 必须是 {API_KINDS} 之一"); sys.exit(2)
|
|
api_kind = v
|
|
elif a.startswith("--timeout"):
|
|
timeout = int(a.split("=", 1)[1]) if "=" in a else int(argv[i + 1]); i += 1 if "=" not in a else 0
|
|
elif a.startswith("--model"):
|
|
raw = a.split("=", 1)[1] if "=" in a else argv[i + 1]; i += 1 if "=" not in a else 0
|
|
only = set(raw.split(","))
|
|
elif api_key is None:
|
|
api_key = a
|
|
i += 1
|
|
if api_key is None:
|
|
print(__doc__)
|
|
sys.exit(1)
|
|
return api_key, concurrent, timeout, only, probe, list_models, api_kind
|
|
|
|
|
|
def list_available(api_key: str, provider: str, api_kind: str, timeout: int) -> dict:
|
|
"""试 GET /v1/models 看网关曝不曝模型列表"""
|
|
if api_kind == "openai":
|
|
paths = (
|
|
f"{BASE_URL}/{provider}/v1/models",
|
|
f"{BASE_URL}/v1/models",
|
|
)
|
|
headers = {"Authorization": f"Bearer {api_key}"}
|
|
else:
|
|
paths = (
|
|
f"{BASE_URL}/{provider}/anthropic/v1/models",
|
|
f"{BASE_URL}/{provider}/v1/models",
|
|
f"{BASE_URL}/v1/models",
|
|
)
|
|
headers = {"x-api-key": api_key, "anthropic-version": "2023-06-01"}
|
|
|
|
for path in paths:
|
|
try:
|
|
req = urllib.request.Request(path, headers=headers)
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
return {"ok": True, "path": path, "body": resp.read().decode("utf-8")}
|
|
except urllib.error.HTTPError as e:
|
|
last_err = (e.code, e.read().decode("utf-8", errors="replace")[:120])
|
|
except Exception as e:
|
|
last_err = ("?", str(e)[:120])
|
|
return {"ok": False, "error": last_err}
|
|
|
|
|
|
def main():
|
|
api_key, concurrent, timeout, only, probe, list_models, api_kind = parse_args(sys.argv)
|
|
|
|
if list_models:
|
|
for kind in (["openai"] if api_kind == "openai" else (API_KINDS if api_kind == "both" else ["anthropic"])):
|
|
print(f"🔑 key={mask_key(api_key)} 🔌 {kind:<9} 探测模型列表...")
|
|
print("-" * 60)
|
|
for p in ("deepseek", "qwen", "glm", "minimax"):
|
|
print(f"⏳ {p:<10} ... ", end="", flush=True)
|
|
r = list_available(api_key, p, kind, timeout)
|
|
if r["ok"]:
|
|
print(f"✅ {r['path']}")
|
|
try:
|
|
parsed = json.loads(r["body"])
|
|
ids = [it["id"] for it in (parsed.get("data") or []) if isinstance(it, dict) and "id" in it]
|
|
if ids:
|
|
print(f" models: {', '.join(ids)}")
|
|
else:
|
|
print(f" raw: {r['body'][:300]}")
|
|
except Exception:
|
|
print(f" raw: {r['body'][:300]}")
|
|
else:
|
|
print(f"❌ {r['error']}")
|
|
print()
|
|
return
|
|
|
|
# 确定要跑哪些 (api_kind, provider, model) 组合
|
|
combos = []
|
|
kinds = ["openai"] if api_kind == "openai" else (API_KINDS[:-1] if api_kind == "both" else ["anthropic"])
|
|
todo = MODELS
|
|
if probe:
|
|
todo = list(MODELS) + PROBE_MODELS
|
|
if only:
|
|
todo = [(p, m) for (p, m) in todo if m in only]
|
|
if not todo:
|
|
print("没有匹配的模型"); sys.exit(1)
|
|
for kind in kinds:
|
|
for p, m in todo:
|
|
combos.append((kind, p, m))
|
|
|
|
print(f"🔑 key={mask_key(api_key)} 🌐 {BASE_URL} ⏱ timeout={timeout}s "
|
|
f"⚙ {'concurrent' if concurrent else 'sequential'} "
|
|
f"🔌 {api_kind}{' 🔎probe' if probe else ''}")
|
|
print(f"📝 prompt: {PROMPT}")
|
|
print("-" * 60)
|
|
|
|
results = []
|
|
|
|
if concurrent:
|
|
with ThreadPoolExecutor(max_workers=min(8, len(combos))) as ex:
|
|
futs = {ex.submit(call, k, api_key, p, m, timeout): (k, p, m) for (k, p, m) in combos}
|
|
for fut in as_completed(futs):
|
|
k, p, m = futs[fut]
|
|
r = fut.result()
|
|
r.update({"api": k, "provider": p, "model": m})
|
|
results.append(r)
|
|
print_line(r)
|
|
else:
|
|
for k, p, m in combos:
|
|
print(f"⏳ [{k:<9}] {p}/{m:<16} ... ", end="", flush=True)
|
|
r = call(k, api_key, p, m, timeout)
|
|
r.update({"api": k, "provider": p, "model": m})
|
|
results.append(r)
|
|
print_line(r, newline=True)
|
|
|
|
print("-" * 60)
|
|
ok = sum(1 for r in results if r["ok"])
|
|
print(f"📊 {ok}/{len(results)} OK")
|
|
|
|
# 汇总:按 api 分组
|
|
by_api = {}
|
|
for r in results:
|
|
by_api.setdefault(r["api"], {"ok": 0, "total": 0})
|
|
by_api[r["api"]]["total"] += 1
|
|
if r["ok"]:
|
|
by_api[r["api"]]["ok"] += 1
|
|
if len(by_api) > 1:
|
|
summary = " ".join(f"{k}:{v['ok']}/{v['total']}" for k, v in by_api.items())
|
|
print(f" by api: {summary}")
|
|
|
|
|
|
def print_line(r: dict, newline: bool = False):
|
|
tag = f"[{r.get('api','?'):<9}] {r.get('provider','?')}/{r.get('model','?')}"
|
|
if r["ok"]:
|
|
msg = f"✅ {tag:<32} {r['elapsed']:6.2f}s"
|
|
if r.get("text"):
|
|
preview = r["text"].replace("\n", " ")[:40]
|
|
msg += f" | {preview}"
|
|
else:
|
|
err = (r.get("error") or "unknown").replace("\n", " ")[:60]
|
|
msg = f"❌ {tag:<32} {r['elapsed']:6.2f}s | {err}"
|
|
if newline:
|
|
print(msg)
|
|
else:
|
|
print(msg)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |