Files
ipass__ipass/tests/apix_check.py
T
admin cde14e1f1b test: apix 纳入上线后验证(tests/apix_check.py + live_check.sh 三步)
apix 发版后最容易静默失效:location 没挂 / shm 没装载都不报错,只是 404,
而回归打假上游、agent.sh verify 又不发真实请求 —— 之前完全没有覆盖。

新增 tests/apix_check.py:
- 从 /api/v1/me(mdm host + X-API-Key 头)拉能力目录,逐个探 enabled 条目;
- vendor → GET /apix/<slug>?<p>=<占位>;passthrough → GET /apixp/<slug>/health;
- **无效 Key 预检**:证明 location 挂着 + 鉴权生效(与上游是否抽风无关);
- 判定按错误体归属:网关体({"error":{...}} / {"code":..,"msg":..})→ 失败;
  上游体({"detail":...} / WAF 页)→ 只告警,不染红发版验证。

实测踩到的三个坑(已写进 README):
1. urllib 默认 UA `Python-urllib/3.x` 被供应商边缘 Cloudflare 拦(403 error code:
   1010),看着像 IP 被封,换 curl UA 立刻 200;
2. passthrough tail 是 /apixp/<slug> 之后的全部(上游 /health → /apixp/ocr/health),
   usage_hint 里的 /<tail> 要换成 slug,照抄是上游 404;
3. vendor 参数要放白名单值:accel 给 example.com 是 400 target host not allowed
   (链路通,属告警),换 raw.githubusercontent 小文件即 200。

live_check.sh 改为三步(LLM 真实流量 / apix 探针 / 环境验收),加 --apix-only。
线上实测:LLM 8/8 + apix 3/3 + 验收全绿。
2026-09-17 07:56:25 +08:00

281 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""apix 上线后探针 —— 验证 /apix/<slug>(vendor)与 /apixp/<slug>(passthrough)真能转发。
为什么要单独一个脚本:LLM 回归(tests/run.sh)打的是**假上游**,`agent.sh verify`
只验管理台,两边都不碰 apix 链路。发版后 apix 配不生效(location 没挂 / shm 没装载)
在这里才会暴露。
判定的关键在 **404 是谁给的**(实测):
* 网关风格 `{"error":{"code":...}}` → ❌ 失败(location 没挂或条目停用,正是要抓的)
* 上游风格 `{"detail":...}` → ⚠️ 告警(链路通,只是路径/参数不对)
同理 4xx 业务拒绝(如 accel 的 `target host not allowed`)算 ⚠️ 不算失败 ——
那说明鉴权、拼体、转发全通了,只是探针参数不合上游业务规则。
已知现象(2026-09-17 实测):**连续**探测 vendor 条目(accel / search)会触发
souyi 边缘 Cloudflare 的 1010 拦截,返回 `error code: 1010`;间隔约 60s 后自动恢复
200。所以连续跑出 ⚠️ 属预期,不是发版问题 —— 看「鉴权✓」就知道网关侧是好的。
探针怎么构造:
vendor → `GET /apix/<slug>?<p>=<占位>`,参数名取 `params[0]`,没有就从
`desc` / `usage_hint` 里正则找第一个 `?xx=`(如 accel 的 `?url=`)。
passthrough → `GET /apixp/<slug>/health`(轻量、不触发 GPU 懒加载等业务开销)。
注意 tail 是 `/apixp/<slug>` **之后**的全部,所以上游 `/health`
对应 `/apixp/ocr/health`,而 usage_hint 里的 `/<tail>` 要换成 slug
(`/ocr/base64` → `/apixp/ocr/ocr/base64`),别照抄。
用法:
python3 tests/apix_check.py <key> # 探针全部 enabled 条目
python3 tests/apix_check.py <key> --slug ocr # 只探某几个(逗号分隔)
python3 tests/apix_check.py <key> --timeout 40
IPASS_MDM_HOST=... IPASS_LM_HOST=... python3 tests/apix_check.py <key>
退出码:有 ❌ 失败 → 1;只有 ✅/⚠️ → 0。
"""
import json
import os
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
MDM = os.environ.get("IPASS_MDM_HOST", "https://mdm.mayishangshu.cn:88")
LM = os.environ.get("IPASS_LM_HOST", "https://lm.mayishangshu.cn:88")
# 参数占位值:让请求「格式合法」即可,不追求业务成功
PLACEHOLDER = {
"q": "ping",
"query": "ping",
"keyword": "ping",
# accel 这类加速服务只放行白名单 host(GitHub/Gitee/GitLab):
# 用最小的 raw 文件(~485B),既能拿到 200 又不吃掉额度
"url": "https://raw.githubusercontent.com/octocat/Hello-World/master/README",
"text": "ping",
}
PASS, WARN, FAIL = "pass", "warn", "fail"
# 见 http() 里的说明:供应商边缘会拦 `Python-urllib/3.x`,必须伪装成常见客户端
DEFAULT_UA = "curl/8.4.0"
# 无效 Key 预检用:形态合法(sk_ 开头)但查不到,网关应返 401
BAD_KEY = "sk_" + "0" * 48
# ⚠️ 重试前的等待(兜底用;注意 1010 的真实诱因是 UA 不是频率,见 DEFAULT_UA 注释)
RETRY_WAIT_SEC = 15
# 只有「疑似限流/网络抖动」值得重试;404(路径不存在)重试没意义
RETRYABLE = (403, 429, 0)
def http(method: str, url: str, key: str, timeout: int, body: bytes | None = None,
ctype: str | None = None, auth: str = "Bearer"):
"""发请求,返回 (code, body_text, elapsed)。网络层异常返回 (0, err, elapsed)。
`auth` 决定鉴权头形态:网关统一收 `Authorization: Bearer <sk_>`,
但 **`/api/v1/me` 只认 `X-API-Key`**(2026-09-17 实测:带 Bearer 会 401)。
"""
if auth == "X-API-Key":
headers = {"X-API-Key": key}
else:
headers = {"Authorization": f"Bearer {key}"}
# ⚠️ 必须带一个「正常」UA:供应商边缘(souyi 走 Cloudflare)会拦
# `Python-urllib/3.x` → 返回 403 `error code: 1010`(2026-09-17 实测,
# 同一请求换 curl UA 立刻 200)。可被 IPASS_UA 覆盖。
headers["User-Agent"] = os.environ.get("IPASS_UA", DEFAULT_UA)
if ctype:
headers["Content-Type"] = ctype
req = urllib.request.Request(url, data=body, headers=headers, method=method)
start = time.time()
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
return resp.status, resp.read().decode("utf-8", "replace"), time.time() - start
except urllib.error.HTTPError as e:
raw = e.read().decode("utf-8", "replace") if e.fp else ""
return e.code, raw, time.time() - start
except Exception as e: # 超时 / DNS / TLS
return 0, str(e), time.time() - start
def fetch_catalog(key: str, timeout: int) -> list | None:
"""拉 /api/v1/me 的能力目录(mdm host + X-API-Key 头)。失败返回 None。"""
code, raw, _ = http("GET", f"{MDM}/api/v1/me", key, timeout, auth="X-API-Key")
if code != 200:
print(f"✗ 拉能力目录失败:HTTP {code} {raw[:200]}")
return None
try:
return json.loads(raw).get("data", {}).get("apix") or []
except json.JSONDecodeError:
print(f"✗ 能力目录不是 JSON:{raw[:200]}")
return None
def is_gateway_error(raw: str) -> bool:
"""是网关自己返的错误体,还是转发之后上游返的(实测两种网关形态):
* apix / 管理接口:`{"code":401,"msg":"API Key 无效或已失效"}`(扁平)
* LLM 侧:`{"error":{"code":"invalid_api_key",...}}`(嵌套)
上游与边缘都不是这两种:FastAPI 是 `{"detail":...}`,Cloudflare 拦截页是
HTML/纯文本(`error code: 1010`),所以能可靠区分「网关没生效」与「上游拒单」。
"""
try:
d = json.loads(raw)
except json.JSONDecodeError:
return False
if not isinstance(d, dict):
return False
if isinstance(d.get("error"), dict):
return True
return "code" in d and "msg" in d
def guess_param(item: dict) -> str | None:
"""vendor 条目要带哪个 query 参数名。"""
params = item.get("params") or []
if params:
return str(params[0])
hint = f"{item.get('desc') or ''} {item.get('usage_hint') or ''}"
m = re.search(r"[?&](\w+)=", hint)
return m.group(1) if m else None
def classify(code: int, raw: str) -> tuple[str, str]:
"""把响应分成 pass / warn / fail。
核心原则:**「网关侧没生效」才是发版要抓的失败**,上游业务拒绝(含供应商 WAF)
只能算告警 —— 否则供应商一次抽风就把发版验证染红(实测:souyi 边缘
Cloudflare 1010 拦 IP,同一请求 20 分钟前还是 200)。
"""
if code == 0:
return FAIL, "网络错误/超时"
if 200 <= code < 300:
return PASS, ""
if code in (401, 403):
if is_gateway_error(raw):
return FAIL, "网关拒绝:Key 无效 / 不在白名单(location 是活的)"
# 非网关体 = 转发之后被上游或边缘(WAF / Cloudflare)拒的
return WARN, f"上游/边缘拒绝 HTTP {code}(非网关体,多为 WAF,需人工确认)"
if code == 404:
if is_gateway_error(raw):
return FAIL, "网关 404:条目未启用或 location 未生效"
return WARN, "上游 404:链路通但路径不存在"
if 400 <= code < 500:
return WARN, "上游 4xx:链路通,业务参数被拒"
if is_gateway_error(raw):
return FAIL, f"网关 {code}"
return WARN, f"上游 {code}(多为探针参数或上游故障,需人工确认)"
def probe(item: dict, key: str, timeout: int) -> dict:
slug = item.get("slug") or "?"
mode = item.get("mode") or "vendor"
if mode == "passthrough":
url = f"{LM}/apixp/{slug}/health"
method, body, ctype = "GET", None, None
else:
p = guess_param(item)
query = f"?{urllib.parse.urlencode({p: PLACEHOLDER.get(p, 'test')})}" if p else ""
url = f"{LM}/apix/{slug}{query}"
method, body, ctype = "GET", None, None
code, raw, elapsed = http(method, url, key, timeout, body, ctype)
status, why = classify(code, raw)
# 无效 Key 预检:证明「location 挂着 + 鉴权生效」(与上游/WAF 是否抽风无关)。
# 期望 401 且是网关错误体 —— 若连无效 Key 都没被网关拦下,说明 location 压根没生效。
# 401 不计用量(网关只对实际转发记账),所以这一步没有副作用。
acode, araw, _ = http(method, url, BAD_KEY, timeout, body, ctype)
auth_ok = (acode == 401 and is_gateway_error(araw))
if not auth_ok:
status = FAIL
why = f"网关鉴权未生效(无效 Key 返回 HTTP {acode},期望 401 网关体)"
return {
"slug": slug, "mode": mode, "url": url, "code": code, "status": status,
"why": why, "elapsed": elapsed, "raw": raw[:160].replace("\n", " "),
"auth_ok": auth_ok, "auth_code": acode,
}
def parse_args(argv):
key, timeout, only, retry = None, 30, None, True
i = 1
while i < len(argv):
a = argv[i]
if a.startswith("--timeout"):
timeout = int(a.split("=", 1)[1]) if "=" in a else int(argv[i + 1])
i += 1 if "=" not in a else 0
elif a == "--no-retry":
retry = False
elif a.startswith("--slug"):
raw = a.split("=", 1)[1] if "=" in a else argv[i + 1]
i += 1 if "=" not in a else 0
only = set(raw.split(","))
elif key is None:
key = a
i += 1
if key is None:
print(__doc__)
sys.exit(1)
return key, timeout, only, retry
def main():
key, timeout, only, retry = parse_args(sys.argv)
items = fetch_catalog(key, timeout)
if items is None:
# 拉不到目录本身就是故障:不能「静默无事可做」,否则发版验证会假绿
sys.exit(1)
if not items:
print("· 能力目录里没有 apix 条目 —— 无事可做")
sys.exit(0)
todo = [it for it in items if it.get("enabled")]
if only:
todo = [it for it in todo if it.get("slug") in only]
disabled = [it.get("slug") for it in items if not it.get("enabled")]
print(f"🌐 {LM} ⏱ timeout={timeout}s 📦 apix 条目 {len(todo)} 个待探"
+ (f"(停用跳过:{', '.join(disabled)})" if disabled else ""))
print("-" * 66)
results = []
for it in todo:
print(f"⏳ {it.get('slug'):<10} ... ", end="", flush=True)
r = probe(it, key, timeout)
# ⚠️ 多半是边缘限流(见文件头的 1010 实测):隔一会儿重试一次,
# 把「临时抽风」和「真的有问题」分开,避免发版验证被噪音污染。
if retry and r["status"] == WARN and r["code"] in RETRYABLE:
time.sleep(RETRY_WAIT_SEC)
r2 = probe(it, key, timeout)
if r2["status"] != WARN:
r2["retried"] = True
r = r2
results.append(r)
icon = {PASS: "✅", WARN: "⚠️ ", FAIL: "❌"}[r["status"]]
mark = "鉴权✓" if r["auth_ok"] else f"鉴权✗(HTTP {r['auth_code']})"
msg = f"{icon} [{r['mode']:<11}] {r['slug']:<10} {r['elapsed']:6.2f}s HTTP {r['code']} {mark}"
if r.get("retried"):
msg += " (重试后)"
if r["why"]:
msg += f" | {r['why']}"
if r["code"] not in (0,) and r["raw"]:
msg += f"\n └ {r['raw']}"
print(msg)
print("-" * 66)
n_fail = sum(1 for r in results if r["status"] == FAIL)
n_warn = sum(1 for r in results if r["status"] == WARN)
n_pass = sum(1 for r in results if r["status"] == PASS)
print(f"📊 ✅{n_pass} ⚠️ {n_warn} ❌{n_fail} (共 {len(results)} 项)")
if n_warn:
print(" ⚠️ = 链路通但上游拒单/路径不存在,不是发版问题;按需人工确认")
sys.exit(1 if n_fail else 0)
if __name__ == "__main__":
main()