From 762d308745350dcdacd7b48b92db1ed9f7202306 Mon Sep 17 00:00:00 2001 From: admin Date: Thu, 17 Sep 2026 08:41:48 +0800 Subject: [PATCH] =?UTF-8?q?test+deploy:=20=E6=8E=A2=E9=92=88=E6=94=B9?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E9=A9=B1=E5=8A=A8=20+=20=E4=B8=89=E6=80=81?= =?UTF-8?q?=E5=88=A4=E5=AE=9A=20+=20=E5=8F=91=E7=89=88=E5=A4=B1=E8=B4=A5?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E5=9B=9E=E6=BB=9A=EF=BC=88=E9=95=9C=E5=83=8F?= =?UTF-8?q?+=E6=95=B0=E6=8D=AE=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 两件事: 1) 验证脚本统一从 /api/v1/me 拉能力目录(新增 tests/me_catalog.py 共享模块)。 之前 slot×模型、apix 条目写死在脚本里——配置是管理台动态的,改配置后探针 要么漏验新上游、要么拿旧模型名误报。现在 test_models 按 upstreams×models 全展开(both 双格式并发),apix_check 按目录逐条探;拉不到目录退出码 2 (验证失效 ≠ 上游没问题,不许假绿)。 三态口径:❌ 网关侧故障(错误体归属 + HTTP 码 + error.code 值三重判据—— 智谱 401 体也是 {"error":{code,message}} 形状,光看形态会把欠费误判成发版 故障);⚠️ 上游/厂商侧(欠费 402、不可达 502/504、WAF、200 软错误) **如实汇总反馈但不染红发版**;网关 429 限流按 GW_WARN 处理。 live_check.sh 退出码语义固定:0 过 / 1 网关侧故障 / 2 验证不可用。 分类器 15 个边界场景离线单测全对(撞形状/码集合/软错误/不误伤正常 200)。 2) agent.sh release 改 4/4 步:验证失败不再停在原地—— 抓 PRE_IMG + update job 的备份路径 → live_check → rc=1 时自动 POST /rollback {tag, restore_backup} 回滚镜像并恢复 data/audit,回滚后 再跑 verify 自检;rc=2(验证不可用)**不回滚**(盲回滚比带病观察危险)。 ipass-agent 1.0.2:rollback 本地无镜像自动 pull(自动路径不许卡「先手动拉」); 新增 restore_data——先 stop 容器(防周期落盘把污染写回)、备份 tar 解包顶替 data/audit、现目录改名 .broken- 留证、只认 ~/ipass/backup/ 下的 tar。 restore_data 本地单测过(恢复/留证/staging 清理/越界拒绝)。 release 开头加控制面版本闸门:<1.0.2 会静默忽略 restore_backup → 中止 (IPASS_FORCE_RELEASE=1 可跳过),⚠️ 线上控制面还是 1.0.1,发版前先上 .11 跑 §12.5 自更新。 附带修 bash 老坑:`$VAR` 紧跟全角字符被吞进变量名(unbound variable, live_check A 路径实测炸出),统一 ${VAR}。 线上实测:live_check 三步全绿(LLM 10/10 目录全展开 + apix 3/3 + 验收)。 只动 tests/deploy,不进镜像;ipass-agent 属控制面(跑宿主),需单独自更新。 --- deploy/agent.sh | 96 ++++++++-- deploy/ipass-agent.py | 94 +++++++++- readme.md | 10 +- tests/apix_check.py | 255 +++++++------------------- tests/live_check.sh | 92 ++++++---- tests/me_catalog.py | 260 ++++++++++++++++++++++++++ tests/test_models.py | 413 ++++++++++++++++-------------------------- 7 files changed, 724 insertions(+), 496 deletions(-) create mode 100644 tests/me_catalog.py diff --git a/deploy/agent.sh b/deploy/agent.sh index b057126..d820465 100755 --- a/deploy/agent.sh +++ b/deploy/agent.sh @@ -19,7 +19,8 @@ # bash deploy/agent.sh update [tag] # 完整更新(缺省用 .env 的 IPASS_IMAGE_TAG) # bash deploy/agent.sh wait # 轮询任务直到结束并打印日志 # bash deploy/agent.sh verify # 正式环境端到端验收(登录+拉页面+查规则) -# bash deploy/agent.sh release # 构建推送镜像并更新、验收(一条龙) +# bash deploy/agent.sh release # 一条龙:构建推送 → 更新 → live_check 验证 +# # → 网关侧故障自动回滚镜像 + 恢复数据备份 # ...(其余子命令见 help) # # 依赖:仓库根 `.env`(已 gitignore)。缺文件或变量为空时直接报错退出。 @@ -133,7 +134,7 @@ cmd_verify() { if [ -z "$DEFAULT_TAG" ]; then echo " ! .env 未设 IPASS_IMAGE_TAG —— 裸 update 无缺省 tag(须显式传)" elif [ "$img" = "${REGISTRY}:${DEFAULT_TAG}" ] || [ "$img" = "$DEFAULT_TAG" ]; then - echo " ✓ .env 默认 tag 与线上一致($DEFAULT_TAG)" + echo " ✓ .env 默认 tag 与线上一致(${DEFAULT_TAG})" else echo " ! .env 的 IPASS_IMAGE_TAG=$DEFAULT_TAG 落后于线上(${img##*:})" echo " 裸 \`bash deploy/agent.sh update\` 会退回 $DEFAULT_TAG —— 请把 .env 同步成 ${img##*:}" @@ -402,8 +403,14 @@ case "$CMD" in wait) [ -n "${1:-}" ] || { echo "用法: bash deploy/agent.sh wait " >&2; exit 2; } cmd_wait "$1" ;; verify) cmd_verify ;; - rollback) if [ -n "${1:-}" ]; then post "/rollback" "{\"tag\":\"$1\"}" | pretty - else post "/rollback" '{}' | pretty; fi ;; + rollback) # 第二参给备份 tar 路径(.11 上 ~/ipass/backup/ 下)→ 回滚镜像同时恢复数据 + if [ -n "${2:-}" ]; then + post "/rollback" "{\"tag\":\"$1\",\"restore_backup\":\"$2\"}" | pretty + elif [ -n "${1:-}" ]; then + post "/rollback" "{\"tag\":\"$1\"}" | pretty + else + post "/rollback" '{}' | pretty + fi ;; restart) post "/restart" '{}' | pretty ;; raw) [ -n "${1:-}" ] || { echo "用法: bash deploy/agent.sh raw GET /check" >&2; exit 2; } get "$2" | pretty ;; @@ -411,17 +418,84 @@ case "$CMD" in TAG="${1:-}" [ -n "$TAG" ] || { echo "用法: bash deploy/agent.sh release 1.0.7" >&2; exit 2; } IMG="$(resolve_image "$TAG")" - echo "== 1/3 构建并推送 $IMG ==" + + # ---- 控制面版本闸门:自动回滚依赖 restore_backup + rollback 自动拉镜像, + # 这两条是 ipass-agent 1.0.2 加的。旧控制面会**静默忽略** restore_backup —— + # 真出故障时就回不了数据,必须开跑前说清楚。 + AGENT_VER="$(get /health | jget version)" + if [ -z "$AGENT_VER" ] || [ "$(printf '%s\n1.0.2\n' "$AGENT_VER" | sort -V | head -1)" != "1.0.2" ]; then + echo "! 控制面版本 ${AGENT_VER:-未知} < 1.0.2:验证失败时将无法自动恢复数据。" + echo " 先在 .11 上跑一次自更新(readme §12.5)再继续发版。" + [ "${IPASS_FORCE_RELEASE:-}" = "1" ] || { echo "✗ 中止(确认要跳过闸门:IPASS_FORCE_RELEASE=1 重跑)" >&2; exit 3; } + fi + + # ---- 回滚目标的两个要素,**动手前先抓**: + # 1) 当前线上镜像(release 同版本重发时 PRE_IMG==IMG,回滚无意义要提醒) + # 2) update job 结果里的数据备份路径(验证失败时连数据一起恢复—— + # 故障版本可能已把污染写穿 shm→data,只回镜像不回数据,旧版会读坏数据) + PRE_IMG="$(get /status | jget image)" + echo "· 回滚目标:镜像 ${PRE_IMG:-(未知)}" + + echo "== 1/4 构建并推送 $IMG ==" bash "$ROOT/deploy/release.sh" "${TAG#ipass-}" echo - echo "== 2/3 更新正式环境 ==" + echo "== 2/4 更新正式环境 ==" JOB="$(post "/update" "{\"image\":\"$IMG\"}" | jget job)" [ -n "$JOB" ] || { echo "✗ 未拿到 job id" >&2; exit 1; } echo "· job = $JOB" - cmd_wait "$JOB" || { echo "✗ 更新任务未成功" >&2; exit 1; } + if ! cmd_wait "$JOB"; then + echo "✗ 更新任务未成功(容器还没换成新镜像,无需回滚)" >&2 + exit 1 + fi + BACKUP="$(get "/jobs/$JOB" | "$PY" -c 'import json,sys;print((json.load(sys.stdin).get("result") or {}).get("backup") or "")' 2>/dev/null || true)" + echo "· 数据备份 = ${BACKUP:-(未记录!回滚时无法恢复数据)}" + echo - echo "== 3/3 验收 ==" - cmd_verify ;; + echo "== 3/4 发版后验证(live_check:LLM 真实流量 + apix 探针 + 环境验收) ==" + # 退出码约定(与 tests/live_check.sh 一致): + # 0=通过;1=网关侧故障(要回滚);2=验证手段本身不可用(缺 key/拉不到目录, + # **不能**当成新版本坏了去回滚——盲目回滚比带病观察更危险) + LIVE_RC=0 + bash "$ROOT/tests/live_check.sh" || LIVE_RC=$? + if [ "$LIVE_RC" = "2" ]; then + echo "⚠ 验证不可用(退出码 2)——**不回滚**,但发版未确认成功。" + echo " 请修好验证入口(key 文件 / 网络)后手动:bash tests/live_check.sh" + exit 2 + fi + + if [ "$LIVE_RC" != "0" ]; then + echo + echo "== 4/4 ✗ 验证失败 → 自动回滚 ==" + if [ -z "$PRE_IMG" ]; then + echo "✗ 拿不到回滚目标镜像(release 前 /status 就没返回 image),不敢自动回滚。" + echo " 人工处理:bash deploy/agent.sh rollback <上一版完整镜像名>,数据备份 ${BACKUP:-无}" + exit 1 + fi + if [ "$PRE_IMG" = "$IMG" ]; then + echo "✗ 线上镜像与本次目标相同(同版本重发),回滚无意义——问题不在这次发布动作," + echo " 按上面失败项人工排查(配置/数据/外部依赖)。" + exit 1 + fi + RB_ARGS="{\"tag\":\"$PRE_IMG\"" + [ -n "$BACKUP" ] && RB_ARGS="$RB_ARGS,\"restore_backup\":\"$BACKUP\"" + [ -n "$BACKUP" ] || echo " ! 无备份路径:只回滚镜像、不恢复数据(污染可能残留)" + RB_JOB="$(post "/rollback" "$RB_ARGS" | jget job)" + [ -n "$RB_JOB" ] || { echo "✗ 回滚任务未启动" >&2; exit 1; } + echo "· 回滚 job = ${RB_JOB}(镜像 -> ${PRE_IMG}${BACKUP:+,数据 <- $BACKUP})" + cmd_wait "$RB_JOB" || { echo "✗✗ 回滚任务本身失败!立刻人工上 .11 处理" >&2; exit 1; } + echo + echo "· 回滚后自检(环境验收):" + if cmd_verify; then + echo "✔ 已回滚到 ${PRE_IMG},验收通过。发版判定:失败(新版本未上线,线上恢复旧版)。" + else + echo "✗✗ 回滚后验收仍不过!立刻人工上 .11 处理(数据备份:${BACKUP:-无})" + fi + exit 1 + fi + echo + echo "== 4/4 验证通过,无需回滚 ==" + echo "✔ 发版成功:${IMG}(回滚快照保留在 $PRE_IMG / ${BACKUP:-无备份})" + ;; *) cat <<'USAGE' iPass 控制面包装(凭据只从 .env 读,命令行里不出现任何 key) @@ -440,11 +514,11 @@ iPass 控制面包装(凭据只从 .env 读,命令行里不出现任何 key 变更 bash deploy/agent.sh update [tag] # 拉镜像 + 重建容器(缺省用 .env 的 IPASS_IMAGE_TAG) bash deploy/agent.sh wait # 轮询任务直到结束并打印日志 - bash deploy/agent.sh rollback [tag] # 回滚 + bash deploy/agent.sh rollback [tag] [备份tar] # 回滚;带 .11 上 ~/ipass/backup/*.tar.gz 时同时恢复数据 bash deploy/agent.sh restart # 只重启 bash deploy/agent.sh pull [tag] # 只拉镜像 - 发版一条龙(构建推送 → 更新 → 验收) + 发版一条龙(构建推送 → 更新 → 发版后验证 → 失败自动回滚镜像+恢复数据) bash deploy/agent.sh release 1.0.7 正式环境验收 diff --git a/deploy/ipass-agent.py b/deploy/ipass-agent.py index c80e6a5..6f2e027 100644 --- a/deploy/ipass-agent.py +++ b/deploy/ipass-agent.py @@ -51,7 +51,9 @@ update / restart / rollback 若带 `name` 且**不等于**受管容器 → 直 GET /jobs/?offset=N 任务详情 + 增量日志 POST /pull {"image":...} 拉镜像(异步任务) POST /update {"image":...,"skip_pull":...} 完整更新(异步任务) - POST /rollback {"tag":...} 回滚到 rollback 标签(异步任务) + POST /rollback {"tag":...,"restore_backup":...} 回滚到指定 tag(异步任务); + restore_backup=update 返回的备份 tar 路径时,回滚前把 + 数据目录/审计目录恢复成备份内容(发版验证失败的自动回滚用) POST /restart 只重启容器(同步) `name` 只接受受管容器(见上);传其它容器名 → 403。 @@ -81,7 +83,7 @@ from urllib.parse import urlparse, parse_qs # ------------------------------------------------------------------ 常量 SERVICE = "ipass-agent" -VERSION = "1.0.1" +VERSION = "1.0.2" DEFAULT_PORT = 18099 DEFAULT_IMAGE = ("crpi-yscq976uwza88qap.cn-shenzhen.personal.cr.aliyuncs.com/" @@ -497,6 +499,63 @@ def backup_data(data_dir, job): return None +def restore_data(data_src, backup, job): + """把数据目录(含 audit)恢复成备份 tar 时刻的内容。 + + 用于**发版验证失败后的自动回滚**:新(故障)版本可能已把污染写穿进 shm→data + (1.0.44 这类改冷却/路由逻辑的版本尤其可能),只回滚镜像不回数据,旧版本启动 + 会读到新形状的数据。调用方必须**先停容器**再进来(worker 周期落盘会在恢复后 + 又把旧内存态写回来)。 + + 安全:备份路径限定在 ~/ipass/backup/ 下(防被拿来读任意 tar);原地数据不删, + 改名为 `.broken-` 留在旁边,出问题还能人工对比。 + """ + if not backup: + raise RuntimeError("restore_backup 为空") + backup = os.path.abspath(os.path.expanduser(str(backup))) + allow_root = os.path.join(os.path.expanduser("~"), "ipass", "backup") + if not backup.startswith(allow_root + os.sep): + raise RuntimeError("拒绝恢复非备份目录下的文件:%s" % backup) + if not os.path.exists(backup): + raise RuntimeError("备份文件不存在:%s(数据未恢复!)" % backup) + + parent = os.path.dirname(data_src.rstrip("/")) + data_src = data_src.rstrip("/") + ts = ts_tag() + staging = os.path.join(parent, ".ipass-restore-%s" % ts) + os.makedirs(staging) + try: + with tarfile.open(backup, "r:gz") as tf: + members = [m for m in tf.getmembers() + if m.name.split("/")[0] in ("data", "audit")] + tf.extractall(staging, members=members) + + # data:现目录改名留证,staging/data 顶上 + cur = data_src + if os.path.isdir(cur): + broken = "%s.broken-%s" % (cur.rstrip("/"), ts) + os.rename(cur, broken) + jlog(job, " 现数据目录留证:%s" % broken) + os.renames(os.path.join(staging, "data"), cur) + os.chmod(cur, 0o777) # 与常规一致:容器内 worker 是 nobody,宿主属主不可靠 + jlog(job, " ✓ 数据目录已恢复:%s(来自 %s)" % (cur, os.path.basename(backup))) + + # audit:备份里有才恢复(1.0.7 前的备份没有该成员) + staged_audit = os.path.join(staging, "audit") + live_audit = os.path.join(parent, "audit") + if os.path.isdir(staged_audit): + if os.path.isdir(live_audit): + os.rename(live_audit, "%s.broken-%s" % (live_audit, ts)) + os.renames(staged_audit, live_audit) + os.chmod(live_audit, 0o777) + jlog(job, " ✓ 审计目录已恢复:%s" % live_audit) + summ = data_summary(cur) + jlog(job, " 恢复后摘要:账号 %s / 域名 %s / 上游 %s" + % (summ.get("users"), summ.get("domains"), summ.get("upstreams"))) + finally: + shutil.rmtree(staging, ignore_errors=True) + + # ------------------------------------------------------------------ 任务(job) JOBS = {} @@ -740,7 +799,34 @@ def op_rollback(job, p): "`bash deploy/agent.sh rollback ipass-1.0.15`)" ) if not image_exists(tag): - raise RuntimeError("回滚镜像不存在:%s(要先 docker pull?)" % tag) + # 自动补齐:私仓永远保留所有 tag。发版失败触发的**自动**回滚必须能自愈, + # 不能停在「本地恰好没拉过旧镜像」上(人工场景报错让人决定)。 + jlog(job, "本地没有 %s,先拉取(回滚不该卡在缺镜像上)" % tag) + op_pull(job, {"image": tag, "registry_user": p.get("registry_user"), + "registry_pass": p.get("registry_pass")}) + if not image_exists(tag): + raise RuntimeError("拉取后仍找不到镜像:%s" % tag) + + # 数据回滚:发版验证失败时,data/audit 里可能已被新(故障)版本写穿污染。 + # restore_backup = 本次 update 返回的备份 tar 路径 → 恢复到备份时刻。 + backup = p.get("restore_backup") + if backup: + name = ensure_managed(p.get("name")) + cfg = inspect_one(name) + if cfg is None: + raise RuntimeError("容器 %s 不存在,无法定位数据目录来恢复" % name) + data_src, _ = extract_mounts(cfg) + # 与 op_update 同口径:显式 data_dir 优先于容器挂载 + if p.get("data_dir"): + data_src = os.path.expanduser(str(p["data_dir"])) + if not data_src: + raise RuntimeError("受管容器没挂 %s,无处恢复" % CONTAINER_DATA) + # 先 stop 不 rm:worker 周期落盘会在恢复后又把(污染)统计写回 data; + # 不删容器是留给 op_update 里的 inspect 复刻参数用(stop 后 rm 照常成功)。 + jlog(job, "→ 停止容器(恢复数据前冻结写入)") + docker(["stop", "-t", "10", name], timeout=60) + restore_data(data_src, backup, job) + jlog(job, "回滚到 %s\n" % tag) params = {"image": tag, "skip_pull": True} for k in ("name", "data_dir", "host_port"): @@ -902,7 +988,7 @@ class Handler(BaseHTTPRequestHandler): "GET /jobs?limit=5", "GET /jobs/?offset=N", "POST /pull {image}", "POST /update {image,skip_pull,name,data_dir,host_port}", - "POST /rollback {tag?}", "POST /restart", + "POST /rollback {tag,restore_backup?}", "POST /restart", ], }) if path == "/check": diff --git a/readme.md b/readme.md index 77be989..215b4a3 100755 --- a/readme.md +++ b/readme.md @@ -702,7 +702,8 @@ token 在 `~/.ipass-agent-token`,重启沿用,远端 `agent.sh` 无需改配 自定义 env 会保留,Docker 自带的(PATH / NGINX_VERSION…)不会照抄。 - 变更类接口是**异步任务**:立刻返回 `{"job":""}`,用 `GET /jobs/?offset=N` 增量拉日志。 **同一时刻只允许一个变更任务**(第二个返回 409),避免并发删容器互相踩。 -- 动手前自动 `tar` 备份数据目录、给旧镜像打 `ipass-gateway:rollback-<时间>` 标签。 +- 动手前自动 `tar` 备份数据目录(含 audit)到 `~/ipass/backup/ipass-data-<时间>.tar.gz` + (1.0.17 起不再给旧镜像打本地 `rollback-*` 标签——私仓永远保留所有 tag,回滚须显式传 tag)。 | 方法 | 路径 | 说明 | | --- | --- | --- | @@ -716,7 +717,7 @@ token 在 `~/.ipass-agent-token`,重启沿用,远端 `agent.sh` 无需改配 | GET | `/jobs?limit=5`、`/jobs/?offset=N` | 任务列表 / 任务详情 + 增量日志 | | POST | `/pull` | `{image, registry_user?, registry_pass?}` → 只拉镜像 | | POST | `/update` | `{image?, skip_pull?, name?, data_dir?, host_port?}` → 完整更新(`name` 只接受受管容器) | -| POST | `/rollback` | `{tag?}` → 回滚到最近(或指定)的 `rollback-*` 镜像 | +| POST | `/rollback` | `{tag, restore_backup?}` → 回滚到指定 tag(1.0.17 起**必须显式传 tag**,本地没有会自动 pull);`restore_backup` 给 update 返回的备份 tar 路径时,先停容器把 data/audit 恢复成备份时刻再回滚(发版验证失败的自动回滚用,控制面 ≥1.0.2) | | POST | `/restart` | 只重启容器(同步返回;`{name?}` 只接受受管容器) | 典型远程调用: @@ -748,8 +749,9 @@ bash deploy/agent.sh status # 容器状态 + 镜像 digest + bash deploy/agent.sh update [tag] # 拉镜像 + 重建容器(缺省用 .env 的 IPASS_IMAGE_TAG) bash deploy/agent.sh wait # 轮询任务直到结束并打印 7 步日志 bash deploy/agent.sh verify # 正式环境验收:登录 → 页面标记 → 后端 subdomain_rule -bash deploy/agent.sh release 1.0.7 # 一条龙:构建推送 → 更新 → 验收 -bash deploy/agent.sh rollback [tag] # 出事回滚 +bash deploy/agent.sh release 1.0.7 # 一条龙:构建推送 → 更新 → live_check 发版后验证 + # → 网关侧故障自动回滚镜像 + 恢复数据备份 +bash deploy/agent.sh rollback [备份tar] # 出事回滚(带 tar 同时恢复数据) ``` `verify` 是「镜像真的换新了没有」的落地检查:它登录线上管理台,核对页面里的新标记 diff --git a/tests/apix_check.py b/tests/apix_check.py index 47271e9..3d2223e 100644 --- a/tests/apix_check.py +++ b/tests/apix_check.py @@ -1,202 +1,91 @@ #!/usr/bin/env python3 -"""apix 上线后探针 —— 验证 /apix/(vendor)与 /apixp/(passthrough)真能转发。 +"""apix 上线后探针 —— 条目从 /api/v1/me 动态拉,逐个验 vendor / passthrough 转发。 -为什么要单独一个脚本:LLM 回归(tests/run.sh)打的是**假上游**,`agent.sh verify` -只验管理台,两边都不碰 apix 链路。发版后 apix 配不生效(location 没挂 / shm 没装载) -在这里才会暴露。 +与 test_models.py 共用 me_catalog.py(目录拉取 + 三态判定),口径完全一致: + ❌ 网关侧故障(location 没挂 / 鉴权没生效 / 连不上)→ 退出码 1,发版要抓; + ⚠️ 上游/供应商侧问题(欠费 402、服务不可达 502、WAF 拦截、探针参数被拒) + → **如实打印并汇总列出**,但不染红发版验证。 -判定的关键在 **404 是谁给的**(实测): - * 网关风格 `{"error":{"code":...}}` → ❌ 失败(location 没挂或条目停用,正是要抓的) - * 上游风格 `{"detail":...}` → ⚠️ 告警(链路通,只是路径/参数不对) -同理 4xx 业务拒绝(如 accel 的 `target host not allowed`)算 ⚠️ 不算失败 —— -那说明鉴权、拼体、转发全通了,只是探针参数不合上游业务规则。 - -已知现象(2026-09-17 实测):**连续**探测 vendor 条目(accel / search)会触发 -souyi 边缘 Cloudflare 的 1010 拦截,返回 `error code: 1010`;间隔约 60s 后自动恢复 -200。所以连续跑出 ⚠️ 属预期,不是发版问题 —— 看「鉴权✓」就知道网关侧是好的。 - -探针怎么构造: +探针构造(参数名/路径都跟着目录走,不写死): vendor → `GET /apix/?

=<占位>`,参数名取 `params[0]`,没有就从 - `desc` / `usage_hint` 里正则找第一个 `?xx=`(如 accel 的 `?url=`)。 - passthrough → `GET /apixp//health`(轻量、不触发 GPU 懒加载等业务开销)。 - 注意 tail 是 `/apixp/` **之后**的全部,所以上游 `/health` - 对应 `/apixp/ocr/health`,而 usage_hint 里的 `/` 要换成 slug - (`/ocr/base64` → `/apixp/ocr/ocr/base64`),别照抄。 + `desc`/`usage_hint` 正则找第一个 `?xx=`(如 accel 的 `?url=`)。 + passthrough → `GET /apixp//health`(轻量,不触发 GPU 懒加载)。 + tail 是 `/apixp/` **之后**的全部:上游 `/health` 对应 + `/apixp/ocr/health`;usage_hint 里的 `/` 要换成 slug,别照抄。 + +每条探针前还做一次**无效 Key 预检**(期望 401 + 网关体):证明 location 挂着、 +鉴权生效——与上游/供应商是否抽风无关。若连无效 Key 都没被网关拦下,判 ❌。 用法: python3 tests/apix_check.py # 探针全部 enabled 条目 python3 tests/apix_check.py --slug ocr # 只探某几个(逗号分隔) - python3 tests/apix_check.py --timeout 40 - IPASS_MDM_HOST=... IPASS_LM_HOST=... python3 tests/apix_check.py + python3 tests/apix_check.py --timeout 40 --no-retry -退出码:有 ❌ 失败 → 1;只有 ✅/⚠️ → 0。 +退出码:有 ❌ → 1;只有 ✅/⚠️ → 0;拉不到目录 → 2(探针失效,不是上游没问题)。 """ -import json import os import re import sys import time -import urllib.error import urllib.parse -import urllib.request -MDM = os.environ.get("IPASS_MDM_HOST", "https://mdm.mayishangshu.cn:88") -LM = os.environ.get("IPASS_LM_HOST", "https://lm.mayishangshu.cn:88") +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import me_catalog as mc # noqa: E402 -# 参数占位值:让请求「格式合法」即可,不追求业务成功 +# vendor 参数占位值:让请求「格式合法」即可。accel 这类只放行白名单 host +# (GitHub/Gitee/GitLab)——用最小 raw 文件(~485B),拿 200 又不吃额度。 PLACEHOLDER = { - "q": "ping", - "query": "ping", - "keyword": "ping", - # accel 这类加速服务只放行白名单 host(GitHub/Gitee/GitLab): - # 用最小的 raw 文件(~485B),既能拿到 200 又不吃掉额度 + "q": "ping", "query": "ping", "keyword": "ping", "text": "ping", "url": "https://raw.githubusercontent.com/octocat/Hello-World/master/README", - "text": "ping", } -PASS, WARN, FAIL = "pass", "warn", "fail" - -# 见 http() 里的说明:供应商边缘会拦 `Python-urllib/3.x`,必须伪装成常见客户端 -DEFAULT_UA = "curl/8.4.0" - -# 无效 Key 预检用:形态合法(sk_ 开头)但查不到,网关应返 401 +# 无效 Key 预检:形态合法(sk_ 开头)但查不到,网关应返 401 BAD_KEY = "sk_" + "0" * 48 -# ⚠️ 重试前的等待(兜底用;注意 1010 的真实诱因是 UA 不是频率,见 DEFAULT_UA 注释) -RETRY_WAIT_SEC = 15 -# 只有「疑似限流/网络抖动」值得重试;404(路径不存在)重试没意义 +# 只有「疑似限流/抖动」值得重试(souyi 边缘偶发拦截);404 重试没意义 RETRYABLE = (403, 429, 0) +RETRY_WAIT_SEC = 15 -def http(method: str, url: str, key: str, timeout: int, body: bytes | None = None, - ctype: str | None = None, auth: str = "Bearer"): - """发请求,返回 (code, body_text, elapsed)。网络层异常返回 (0, err, elapsed)。 - - `auth` 决定鉴权头形态:网关统一收 `Authorization: Bearer `, - 但 **`/api/v1/me` 只认 `X-API-Key`**(2026-09-17 实测:带 Bearer 会 401)。 - """ - if auth == "X-API-Key": - headers = {"X-API-Key": key} - else: - headers = {"Authorization": f"Bearer {key}"} - # ⚠️ 必须带一个「正常」UA:供应商边缘(souyi 走 Cloudflare)会拦 - # `Python-urllib/3.x` → 返回 403 `error code: 1010`(2026-09-17 实测, - # 同一请求换 curl UA 立刻 200)。可被 IPASS_UA 覆盖。 - headers["User-Agent"] = os.environ.get("IPASS_UA", DEFAULT_UA) - if ctype: - headers["Content-Type"] = ctype - req = urllib.request.Request(url, data=body, headers=headers, method=method) - start = time.time() - try: - with urllib.request.urlopen(req, timeout=timeout) as resp: - return resp.status, resp.read().decode("utf-8", "replace"), time.time() - start - except urllib.error.HTTPError as e: - raw = e.read().decode("utf-8", "replace") if e.fp else "" - return e.code, raw, time.time() - start - except Exception as e: # 超时 / DNS / TLS - return 0, str(e), time.time() - start - - -def fetch_catalog(key: str, timeout: int) -> list | None: - """拉 /api/v1/me 的能力目录(mdm host + X-API-Key 头)。失败返回 None。""" - code, raw, _ = http("GET", f"{MDM}/api/v1/me", key, timeout, auth="X-API-Key") - if code != 200: - print(f"✗ 拉能力目录失败:HTTP {code} {raw[:200]}") - return None - try: - return json.loads(raw).get("data", {}).get("apix") or [] - except json.JSONDecodeError: - print(f"✗ 能力目录不是 JSON:{raw[:200]}") - return None - - -def is_gateway_error(raw: str) -> bool: - """是网关自己返的错误体,还是转发之后上游返的(实测两种网关形态): - - * apix / 管理接口:`{"code":401,"msg":"API Key 无效或已失效"}`(扁平) - * LLM 侧:`{"error":{"code":"invalid_api_key",...}}`(嵌套) - - 上游与边缘都不是这两种:FastAPI 是 `{"detail":...}`,Cloudflare 拦截页是 - HTML/纯文本(`error code: 1010`),所以能可靠区分「网关没生效」与「上游拒单」。 - """ - try: - d = json.loads(raw) - except json.JSONDecodeError: - return False - if not isinstance(d, dict): - return False - if isinstance(d.get("error"), dict): - return True - return "code" in d and "msg" in d - - -def guess_param(item: dict) -> str | None: - """vendor 条目要带哪个 query 参数名。""" +def build_url(item: dict): + """按 mode 构造探针 URL / 方法。返回 (method, url, body, ctype) 或 None。""" + slug = item.get("slug") + mode = item.get("mode") or "vendor" + if mode == "passthrough": + return "GET", f"{mc.LM}/apixp/{slug}/health", None, None params = item.get("params") or [] - if params: - return str(params[0]) - hint = f"{item.get('desc') or ''} {item.get('usage_hint') or ''}" - m = re.search(r"[?&](\w+)=", hint) - return m.group(1) if m else None - - -def classify(code: int, raw: str) -> tuple[str, str]: - """把响应分成 pass / warn / fail。 - - 核心原则:**「网关侧没生效」才是发版要抓的失败**,上游业务拒绝(含供应商 WAF) - 只能算告警 —— 否则供应商一次抽风就把发版验证染红(实测:souyi 边缘 - Cloudflare 1010 拦 IP,同一请求 20 分钟前还是 200)。 - """ - if code == 0: - return FAIL, "网络错误/超时" - if 200 <= code < 300: - return PASS, "" - if code in (401, 403): - if is_gateway_error(raw): - return FAIL, "网关拒绝:Key 无效 / 不在白名单(location 是活的)" - # 非网关体 = 转发之后被上游或边缘(WAF / Cloudflare)拒的 - return WARN, f"上游/边缘拒绝 HTTP {code}(非网关体,多为 WAF,需人工确认)" - if code == 404: - if is_gateway_error(raw): - return FAIL, "网关 404:条目未启用或 location 未生效" - return WARN, "上游 404:链路通但路径不存在" - if 400 <= code < 500: - return WARN, "上游 4xx:链路通,业务参数被拒" - if is_gateway_error(raw): - return FAIL, f"网关 {code}" - return WARN, f"上游 {code}(多为探针参数或上游故障,需人工确认)" + p = str(params[0]) if params else None + if not p: + hint = f"{item.get('desc') or ''} {item.get('usage_hint') or ''}" + m = re.search(r"[?&](\w+)=", hint) + p = m.group(1) if m else None + query = f"?{urllib.parse.urlencode({p: PLACEHOLDER.get(p, 'test')})}" if p else "" + return "GET", f"{mc.LM}/apix/{slug}{query}", None, None def probe(item: dict, key: str, timeout: int) -> dict: + built = build_url(item) slug = item.get("slug") or "?" mode = item.get("mode") or "vendor" - if mode == "passthrough": - url = f"{LM}/apixp/{slug}/health" - method, body, ctype = "GET", None, None - else: - p = guess_param(item) - query = f"?{urllib.parse.urlencode({p: PLACEHOLDER.get(p, 'test')})}" if p else "" - url = f"{LM}/apix/{slug}{query}" - method, body, ctype = "GET", None, None + if not built: + return {"slug": slug, "mode": mode, "url": "-", "code": 0, + "status": mc.GW_FAIL, "why": "探针 URL 构造失败", "elapsed": 0, + "raw": "", "auth_ok": False, "auth_code": 0} + method, url, body, ctype = built + code, raw, elapsed = mc.http(method, url, key, timeout, body, ctype) + status, why = mc.classify_apix(code, raw) - code, raw, elapsed = http(method, url, key, timeout, body, ctype) - status, why = classify(code, raw) - - # 无效 Key 预检:证明「location 挂着 + 鉴权生效」(与上游/WAF 是否抽风无关)。 - # 期望 401 且是网关错误体 —— 若连无效 Key 都没被网关拦下,说明 location 压根没生效。 - # 401 不计用量(网关只对实际转发记账),所以这一步没有副作用。 - acode, araw, _ = http(method, url, BAD_KEY, timeout, body, ctype) - auth_ok = (acode == 401 and is_gateway_error(araw)) + # 无效 Key 预检(期望 401 且归属是网关,双判据见 me_catalog.from_gateway) + acode, araw, _ = mc.http(method, url, BAD_KEY, timeout, body, ctype) + auth_ok = (acode == 401 and mc.from_gateway(acode, araw)[0]) if not auth_ok: - status = FAIL + status = mc.GW_FAIL why = f"网关鉴权未生效(无效 Key 返回 HTTP {acode},期望 401 网关体)" - return { - "slug": slug, "mode": mode, "url": url, "code": code, "status": status, - "why": why, "elapsed": elapsed, "raw": raw[:160].replace("\n", " "), - "auth_ok": auth_ok, "auth_code": acode, - } + return {"slug": slug, "mode": mode, "url": url, "code": code, "status": status, + "why": why, "elapsed": elapsed, "raw": raw[:160].replace("\n", " "), + "auth_ok": auth_ok, "auth_code": acode} def parse_args(argv): @@ -224,56 +113,48 @@ def parse_args(argv): def main(): key, timeout, only, retry = parse_args(sys.argv) - items = fetch_catalog(key, timeout) - if items is None: - # 拉不到目录本身就是故障:不能「静默无事可做」,否则发版验证会假绿 - sys.exit(1) - if not items: - print("· 能力目录里没有 apix 条目 —— 无事可做") - sys.exit(0) + me, err = mc.fetch_me(key, timeout) + if me is None: + print(f"❌ 拉 /api/v1/me 失败:{err}") + print(" 拉不到目录 = 探针不知道要验什么(不是上游没问题),退出") + sys.exit(2) + items = me.get("apix") or [] todo = [it for it in items if it.get("enabled")] if only: todo = [it for it in todo if it.get("slug") in only] disabled = [it.get("slug") for it in items if not it.get("enabled")] - print(f"🌐 {LM} ⏱ timeout={timeout}s 📦 apix 条目 {len(todo)} 个待探" - + (f"(停用跳过:{', '.join(disabled)})" if disabled else "")) + print(f"🌐 {mc.LM} ⏱ timeout={timeout}s 📦 apix 条目 {len(todo)} 个待探" + + (f"(停用跳过:{', '.join(map(str, disabled))})" if disabled else "")) print("-" * 66) results = [] for it in todo: print(f"⏳ {it.get('slug'):<10} ... ", end="", flush=True) r = probe(it, key, timeout) - # ⚠️ 多半是边缘限流(见文件头的 1010 实测):隔一会儿重试一次, - # 把「临时抽风」和「真的有问题」分开,避免发版验证被噪音污染。 - if retry and r["status"] == WARN and r["code"] in RETRYABLE: + if retry and r["status"] in (mc.UPSTREAM, mc.GW_WARN) and r["code"] in RETRYABLE: time.sleep(RETRY_WAIT_SEC) r2 = probe(it, key, timeout) - if r2["status"] != WARN: + if r2["status"] not in (mc.UPSTREAM, mc.GW_WARN): r2["retried"] = True r = r2 results.append(r) - icon = {PASS: "✅", WARN: "⚠️ ", FAIL: "❌"}[r["status"]] + icon = mc.ICONS[r["status"]] mark = "鉴权✓" if r["auth_ok"] else f"鉴权✗(HTTP {r['auth_code']})" msg = f"{icon} [{r['mode']:<11}] {r['slug']:<10} {r['elapsed']:6.2f}s HTTP {r['code']} {mark}" if r.get("retried"): msg += " (重试后)" if r["why"]: msg += f" | {r['why']}" - if r["code"] not in (0,) and r["raw"]: - msg += f"\n └ {r['raw']}" print(msg) + if r["code"] != 0 and r["status"] != mc.PASS and r["raw"]: + print(f" └ {r['raw']}") - print("-" * 66) - n_fail = sum(1 for r in results if r["status"] == FAIL) - n_warn = sum(1 for r in results if r["status"] == WARN) - n_pass = sum(1 for r in results if r["status"] == PASS) - print(f"📊 ✅{n_pass} ⚠️ {n_warn} ❌{n_fail} (共 {len(results)} 项)") - if n_warn: - print(" ⚠️ = 链路通但上游拒单/路径不存在,不是发版问题;按需人工确认") - - sys.exit(1 if n_fail else 0) + if not results: + print("· 目录里没有 enabled 的 apix 条目 —— 无事可做") + sys.exit(0) + sys.exit(mc.summarize(results, "apix 探针")) if __name__ == "__main__": diff --git a/tests/live_check.sh b/tests/live_check.sh index 3d410bd..d5bb7ab 100755 --- a/tests/live_check.sh +++ b/tests/live_check.sh @@ -1,32 +1,39 @@ #!/usr/bin/env bash # --------------------------------------------------------------------------- -# 上线后验证(live smoke)—— 发版流程的最后一步 +# 上线后验证(live smoke)—— 发版流程的最后一步(agent.sh release 第 3/4 步调它) # # 本机回归(tests/run.sh)跑的是**假上游**,验不到真实链路;`deploy/agent.sh verify` # 又只验管理台页面与后端常量,**一个真实请求都不发**。真正确认「新版在线上能干活」 # 需要三步合起来: # -# 1. LLM 真实流量:tests/test_models.py —— 4 slot × OpenAI/Anthropic 真实调用 -# + `/v1/models`(下游 Key 鉴权、slot 路由、Key 替换、出站头都在这条链上) -# 2. apix 探针:tests/apix_check.py —— `/apix/`(vendor) 与 `/apixp/` -# (passthrough) 是否真的转发(**发版后 apix 最容易静默失效**:location 没挂、 -# shm 没装载都不会报错,只是 404) -# 3. 环境验收:deploy/agent.sh verify —— 登录、页面新标记、subdomain_rule、趋势、 +# 1. LLM 真实流量:tests/test_models.py —— 用例目录从 /api/v1/me **动态拉** +# (slot × 配置模型 × OpenAI/Anthropic 全展开),挨个真实调用 +# 2. apix 探针:tests/apix_check.py —— 同样从 /api/v1/me 拉条目,验 +# `/apix/`(vendor) 与 `/apixp/`(passthrough) 真转发。apix 发版后 +# 最容易**静默失效**:location 没挂 / shm 没装载都不报错,只是 404 +# 3. 环境验收:deploy/agent.sh verify —— 登录、页面标记、subdomain_rule、趋势、 # 账号级 load(确认跑的确实是新镜像,而不是旧容器没换) # +# 三态口径(LLM/apix 共享 me_catalog.py): +# ❌ = 网关侧故障(鉴权没生效/location 没挂/连不上)→ 影响退出码,发版要抓 +# ⚠️ = 上游/厂商侧问题(欠费 402、不可达 502、WAF、探针参数被拒)→ **如实汇总 +# 反馈但不染红**——网关正常、一方账号没钱了不是新镜像的锅 +# +# 退出码(release 自动化按这个决定是否回滚,别改语义): +# 0 全部通过(⚠️ 不阻断,但会列出来) +# 1 有确认的网关侧故障 → release 会回滚镜像+恢复数据 +# 2 验证手段不可用(缺 key 文件 / 拉不到 /api/v1/me / 子脚本缺失) +# → **不能**当成新版本坏了去回滚,release 只告警不回滚 +# # 用法: # bash tests/live_check.sh # 三步都跑(key 默认 ~/ssw.txt) # bash tests/live_check.sh ~/.keys/ipass.txt # 指定 key 文件 -# bash tests/live_check.sh --models-only # 只跑 LLM 真实流量 -# bash tests/live_check.sh --apix-only # 只跑 apix 探针 -# bash tests/live_check.sh --verify-only # 只跑环境验收 +# bash tests/live_check.sh --models-only / --apix-only / --verify-only # IPASS_LIVE_KEY_FILE=~/ssw.txt bash tests/live_check.sh # # Key 文件解析(踩过的坑):文件里常写成 `sk=` 或带 `#` 注释行, -# **取整行会拿到 `sk=sk_xxx` → 全量 401 invalid_api_key**。这里统一: -# 跳过空行/注释行 → 有 `=` 取第一个 `=` 之后 → 去引号与空白 → 只认 `sk_` 开头。 -# -# ⚠️ 全程不打印 Key 原文(子脚本自带 mask,只显示 `sk_2df...2dae`)。 +# **取整行会拿到 `sk=sk_xxx` → 全量 401**。这里统一:跳空行/注释 → 有 `=` 取 +# 第一个 `=` 之后 → 去引号空白 → 只认 `sk_` 开头。⚠️ 全程不打印 Key 原文。 # --------------------------------------------------------------------------- set -uo pipefail @@ -43,7 +50,7 @@ while [ $# -gt 0 ]; do --models-only) RUN_APIX=0; RUN_VERIFY=0; shift ;; --apix-only) RUN_MODELS=0; RUN_VERIFY=0; shift ;; --verify-only) RUN_MODELS=0; RUN_APIX=0; shift ;; - -h|--help) sed -n '2,32p' "${BASH_SOURCE[0]}"; exit 0 ;; + -h|--help) sed -n '2,42p' "${BASH_SOURCE[0]}"; exit 0 ;; -*) echo "✗ 未知参数:$1(用 -h 看用法)" >&2; exit 2 ;; *) KEY_FILE="$1"; shift ;; esac @@ -56,7 +63,7 @@ PY="" for c in /Users/xs/.workbuddy/binaries/python/versions/3.13.12/bin/python3 python3; do if command -v "$c" >/dev/null 2>&1; then PY="$c"; break; fi done -[ -n "$PY" ] || { echo "✗ 找不到 python3" >&2; exit 2; } +[ -n "$PY" ] || { echo "✗ 找不到 python3(验证不可用)" >&2; exit 2; } # 从 key 文件里挑出真正的 Key(见文件头注释:支持 `k=v` / 注释行 / 裸 key) pick_key() { @@ -73,43 +80,52 @@ pick_key() { }' "$1" } -rc=0 +# 聚合退出码:任一 1 → 1(回滚);否则任一 2 → 2(验证不可用,不回滚);否则 0 +CODE=0 +bump() { # bump <本次rc> + case "$1" in + 1) CODE=1 ;; + 2) [ "$CODE" != "1" ] && CODE=2 ;; + esac +} step=0 KEY="" -# 要发真实请求的两步都得先有 Key +# 要发真实请求的两步都得先有 Key —— 没有就是「验证不可用」(2),不是新版故障(1) if [ "$RUN_MODELS" -eq 1 ] || [ "$RUN_APIX" -eq 1 ]; then if [ ! -f "$KEY_FILE" ]; then - echo "✗ 找不到 Key 文件:$KEY_FILE" - echo " 传路径或用 IPASS_LIVE_KEY_FILE 指定;文件内容可以是裸 key 或 sk=" - exit 1 + echo "✗ 找不到 Key 文件:${KEY_FILE}(验证不可用)" + echo " 传路径或用 IPASS_LIVE_KEY_FILE 指定;内容可以是裸 key 或 sk=" + exit 2 fi KEY="$(pick_key "$KEY_FILE")" if [ -z "$KEY" ]; then echo "✗ $KEY_FILE 里没解析到 Key(期望一行 sk_ 开头,或 sk= 形式)" - exit 1 + exit 2 fi fi # --------------------------------------------------------------------------- -# 1/3 LLM 真实流量 +# 1/3 LLM 真实流量(目录驱动:/api/v1/me upstreams × models) # --------------------------------------------------------------------------- if [ "$RUN_MODELS" -eq 1 ]; then step=$((step + 1)) - echo "== $step/$TOTAL LLM 真实流量(test_models.py:4 slot × OpenAI/Anthropic) ==" - "$PY" "$HERE/test_models.py" "$KEY" --api both --concurrent --timeout 60 || rc=1 + echo "== $step/$TOTAL LLM 真实流量(test_models.py:目录全展开 × OpenAI/Anthropic) ==" + rc=0; "$PY" "$HERE/test_models.py" "$KEY" --timeout 60 || rc=$? + bump "$rc" + [ "$rc" = "2" ] && echo " ! test_models 无法建立验证基线(目录拉不到)——不回滚,查 Key/网络" echo fi # --------------------------------------------------------------------------- -# 2/3 apix 探针 +# 2/3 apix 探针(目录驱动:/api/v1/me apix 条目) # --------------------------------------------------------------------------- if [ "$RUN_APIX" -eq 1 ]; then step=$((step + 1)) echo "== $step/$TOTAL apix 探针(apix_check.py:vendor + passthrough) ==" - # ⚠️ 判定口径见脚本头:只有「网关侧没生效」才算失败, - # 上游业务拒绝 / 供应商 WAF(如 Cloudflare 1010)只告警,不染红发版验证。 - "$PY" "$HERE/apix_check.py" "$KEY" --timeout 40 || rc=1 + rc=0; "$PY" "$HERE/apix_check.py" "$KEY" --timeout 40 || rc=$? + bump "$rc" + [ "$rc" = "2" ] && echo " ! apix_check 无法建立验证基线(目录拉不到)——不回滚,查 Key/网络" echo fi @@ -120,16 +136,18 @@ if [ "$RUN_VERIFY" -eq 1 ]; then step=$((step + 1)) echo "== $step/$TOTAL 正式环境验收(deploy/agent.sh verify) ==" if [ ! -f "$REPO/.env" ]; then - echo "· 跳过:仓库根没有 .env(控制面凭据只从这里读,没有就跑不了验收)" + echo "· 跳过:仓库根没有 .env(控制面凭据只从这里读)——验收不可用" + bump 2 else - bash "$REPO/deploy/agent.sh" verify || rc=1 + rc=0; bash "$REPO/deploy/agent.sh" verify || rc=1 + bump "$rc" fi echo fi -if [ "$rc" -eq 0 ]; then - echo "✔ 上线后验证通过" -else - echo "✗ 上线后验证有失败项(见上面输出)" -fi -exit $rc +case "$CODE" in + 0) echo "✔ 上线后验证通过(如有 ⚠️ 为上游/厂商侧问题,已如实列出,不阻断)" ;; + 1) echo "✗ 上线后验证发现网关侧故障(见上面 ❌)——release 将回滚镜像+恢复数据" ;; + 2) echo "✗ 上线后验证不可用(缺 key/拉不到目录/无 .env)——**不能据此判新版故障**,release 不回滚" ;; +esac +exit $CODE diff --git a/tests/me_catalog.py b/tests/me_catalog.py new file mode 100644 index 0000000..4c13df0 --- /dev/null +++ b/tests/me_catalog.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +"""/api/v1/me 能力目录 + 三态判定 —— test_models.py / apix_check.py 共享模块。 + +为什么要统一从这里拉:slot、模型名、apix 条目都是**管理台动态配置**,写死在探针 +脚本里等于「发版验证验的是半年前的配置」——加个上游、改个模型名,脚本就静默漏验 +或误报。1.0.16 起 `/api/v1/me` 把 upstreams + apix 一处下发全(见 +docs/plugin/ME_CAPABILITY_CATALOG.md),探针跟着目录走,配置变了验证自动跟上。 + +三态判定(发版验证的核心口径): + ✅ pass —— 2xx,链路真的通 + ❌ gw_fail —— **网关侧**问题(错误体是网关形态 / 连不上网关 / 网关 4xx) + → 发版要抓的失败,退出码 1 + ⚠️ upstream —— 请求已穿过网关、被供应商/上游拒了(欠费 402/429、不可达 502/504、 + 鉴权过期 401…)→ **如实反馈但不判发版失败**:那是一方账号的问题, + 不是新镜像的问题。告警会汇总打印,不静默。 + +区分「谁给的错误」靠错误体形态(实测归纳): + 网关 LLM 体 {"error":{"type":..,"code":..,"message":..}} + 网关管理/apix {"code":401,"msg":"..."}(扁平) + 上游 FastAPI {"detail":...};厂商各家自有;Cloudflare 拦截页是 HTML。 +""" + +import json +import os +import time +import urllib.error +import urllib.request + +MDM = os.environ.get("IPASS_MDM_HOST", "https://mdm.mayishangshu.cn:88") +LM = os.environ.get("IPASS_LM_HOST", "https://lm.mayishangshu.cn:88") + +# 供应商边缘(souyi 走 Cloudflare)会拦默认 `Python-urllib/3.x` → 403 error code:1010 +# (2026-09-17 实测,同请求换 curl UA 立刻 200)。可被 IPASS_UA 覆盖。 +DEFAULT_UA = "curl/8.4.0" + +# 三态 + 网关告警(429 限流是「探针自己打太快」,不该染红发版) +PASS, UPSTREAM, GW_WARN, GW_FAIL = "pass", "upstream", "gw_warn", "gw_fail" +ICONS = {PASS: "✅", UPSTREAM: "⚠️ ", GW_WARN: "⚠️ ", GW_FAIL: "❌"} + + +def http(method: str, url: str, key: str, timeout: int, body: bytes | None = None, + ctype: str | None = None, auth: str = "Bearer"): + """发请求,返回 (code, body_text, elapsed)。网络层异常 code=0。 + + `auth`:网关统一收 `Authorization: Bearer `,但 **`/api/v1/me` 只认 + `X-API-Key`**(2026-09-17 实测:带 Bearer 会 401)。 + """ + headers = {"X-API-Key": key} if auth == "X-API-Key" else {"Authorization": f"Bearer {key}"} + headers["User-Agent"] = os.environ.get("IPASS_UA", DEFAULT_UA) + if ctype: + headers["Content-Type"] = ctype + req = urllib.request.Request(url, data=body, headers=headers, method=method) + start = time.time() + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.status, resp.read().decode("utf-8", "replace"), time.time() - start + except urllib.error.HTTPError as e: + raw = e.read().decode("utf-8", "replace") if e.fp else "" + return e.code, raw, time.time() - start + except Exception as e: # 超时 / DNS / TLS + return 0, str(e), time.time() - start + + +def fetch_me(key: str, timeout: int = 20): + """拉 /api/v1/me 的 data(含 upstreams + apix)。失败返回 (None, 原因)。""" + code, raw, _ = http("GET", f"{MDM}/api/v1/me", key, timeout, auth="X-API-Key") + if code != 200: + return None, f"HTTP {code} {raw[:160]}" + try: + return json.loads(raw).get("data") or {}, "" + except json.JSONDecodeError: + return None, f"响应不是 JSON:{raw[:160]}" + + +def gateway_error(raw: str): + """是网关自己返的错误体则返回 (code, msg),否则 (False, None)。 + + 两种网关形态(实测):LLM 侧嵌套 `{"error":{...}}`;管理/apix 侧扁平 + `{"code":N,"msg":"..."}`。上游 FastAPI(detail)与 Cloudflare(HTML)都不匹配。 + """ + try: + d = json.loads(raw) + except json.JSONDecodeError: + return False, None + if not isinstance(d, dict): + return False, None + err = d.get("error") + if isinstance(err, dict): + return True, f"{err.get('code') or err.get('type') or ''} {err.get('message') or ''}".strip() + if "code" in d and "msg" in d: + return True, f"{d.get('code')} {d.get('msg')}".strip() + return False, None + + +def llm_cases(me: dict): + """把目录展开成真实验证用例:每个 enabled 上游 × 每个配置模型(挨个跑)。 + + 返回 [{slot, model, openai, anthropic}];`allowed_models` 白名单非空时按其过滤 + (否则会拿 403 permission_denied,那是账号策略不是链路故障)。 + """ + allowed = set(me.get("allowed_models") or []) + cases = [] + for u in me.get("upstreams") or []: + if not u.get("enabled"): + continue + slot = u.get("slot") or "?" + base_o = (u.get("base_url") or f"{LM}/{slot}").rstrip("/") + base_a = (u.get("anthropic_base_url") or u.get("base_anthropic") + or f"{LM}/{slot}/anthropic").rstrip("/") + models = u.get("models") or [] + if allowed and slot not in allowed: + continue + if not models: + # 没配模型名:用占位探一次,让「配置缺失」如实暴露(网关会返回模型拒绝) + models = [""] + for m in models: + cases.append({ + "slot": slot, "model": m, + "openai": f"{base_o}/v1/chat/completions", + "anthropic": f"{base_a}/v1/messages", + "models_list": f"{base_o}/v1/models", + }) + return cases + + +def apix_items(me: dict): + """enabled 的 apix 条目(探针构造留给 apix_check.py)。""" + return [it for it in (me.get("apix") or []) if it.get("enabled")] + + +# 网关(Rust 侧 auth::fail)只可能发这些状态码;402/5xx 等出现即说明是上游透传。 +GW_CODES = {400, 401, 403, 404, 429} +# 网关 LLM 体 error.code 的**全集**(grep auth::fail 得来)——厂商(智谱等)的 401/404 +# 体也是 {"error":{code,message}} 形状,光看形态+码集合会误判,error.code 必须也在 +# 这个集合里才算网关(HTTP 码与 error.code 双对得上才是网关手笔)。 +GW_ERR_CODES = {"invalid_api_key", "model_not_found", "permission_denied", + "rate_limit_exceeded", "insufficient_quota", "invalid_request_error"} + + +def from_gateway(code: int, raw: str): + """(是否网关所返, 网关 msg)。形态 + HTTP 码 + error.code 值三重判据。""" + if code not in GW_CODES: + return False, None + try: + d = json.loads(raw) + except json.JSONDecodeError: + return False, None + if not isinstance(d, dict): + return False, None + err = d.get("error") + if isinstance(err, dict): + c = err.get("code") or "" + if c in GW_ERR_CODES: + return True, f"{c} {err.get('message') or ''}".strip() + return False, None + if "code" in d and "msg" in d: + # 扁平管理体(/api/v1/me、apix、admin):网关独有形态,无厂商撞车(实测) + return True, f"{d.get('code')} {d.get('msg')}".strip() + return False, None + + +def llm_soft_error(api_kind: str, raw: str): + """HTTP 200 但 body 无正文 = 厂商软错误(实测:智谱把 NPE 包成 200+业务 500)。 + 返回错误摘要或 None。""" + try: + d = json.loads(raw) + except json.JSONDecodeError: + return "响应不是 JSON" + if not isinstance(d, dict): + return None + if api_kind == "anthropic": + if d.get("content"): + return None + else: + if d.get("choices"): + return None + if d.get("type") == "error" or "error" in d or ("code" in d and d.get("code") not in (0, "0", 200, "200")): + return json.dumps(d, ensure_ascii=False)[:120] + # 没有正文字段也没有明显错误键:形状不认识也算软错误 + return f"200 但无正文(形状不认识): {json.dumps(d, ensure_ascii=False)[:100]}" + + +def classify_llm(code: int, raw: str) -> tuple[str, str]: + """LLM 调用响应 → 三态。见文件头口径。""" + if code == 0: + # 连的是网关 host(lm/mdm),连不上 = 网关/网络问题,不是上游 + return GW_FAIL, "连不上网关(超时/DNS/TLS)" + if 200 <= code < 300: + return PASS, "" + is_gw, msg = from_gateway(code, raw) + if is_gw: + if code == 429: + # 限流/每日配额是账号策略(探针打太快或 Key 超额),不是发版故障 + return GW_WARN, f"网关限流/配额:{msg}(非发版故障,如实反馈)" + return GW_FAIL, f"网关拒绝 HTTP {code}:{msg}" + # —— 以下都是「请求已穿过网关、上游/厂商拒的」→ 如实反馈、不染红 + if code in (502, 504): + return UPSTREAM, f"上游不可达(nginx {code},网关已转发但厂商连不上)" + if code == 402: + return UPSTREAM, f"厂商 402:疑似额度耗尽/欠费(全账号冷却也不救=没额度了){body_hint(raw)}" + if code == 429: + return UPSTREAM, f"厂商 429:限流{body_hint(raw)}" + if code in (401, 403): + return UPSTREAM, f"厂商鉴权 {code}:上游 Key 可能过期/失效(管理台换 Key){body_hint(raw)}" + if code == 404: + return UPSTREAM, f"厂商 404:模型名/路径不存在(核对管理台 models){body_hint(raw)}" + if 400 <= code < 500: + return UPSTREAM, f"厂商 {code}:探针参数被拒(链路通){body_hint(raw)}" + return UPSTREAM, f"厂商 {code}{body_hint(raw)}(上游故障——HTTP 200≠成功,body 是原话)" + + +def body_hint(raw: str) -> str: + s = (raw or "").replace("\n", " ").strip() + return f"|{s[:80]}" if s else "" + + +def classify_apix(code: int, raw: str) -> tuple[str, str]: + """apix 探针响应 → 三态。归属判定与 classify_llm 同一套双判据。 + + apix 的 401/403 有两种来源:网关(Key 无效)或供应商/内部服务(上游 Key 失效、 + WAF)。**网关码集合 + 网关体形态**同时命中才判 ❌;上游 FastAPI(detail)与 + WAF 页(HTML)都不匹配 → ⚠️ 如实反馈。 + """ + if code == 0: + return GW_FAIL, "连不上网关(超时/DNS/TLS)" + if 200 <= code < 300: + return PASS, "" + is_gw, msg = from_gateway(code, raw) + if is_gw: + if code == 429: + return GW_WARN, f"网关限流/配额:{msg}(探针打太快,非发版故障)" + return GW_FAIL, f"网关拒绝 HTTP {code}:{msg}" + if code in (502, 504): + return UPSTREAM, f"上游不可达(nginx {code},网关已转发但服务连不上){body_hint(raw)}" + if code == 402: + return UPSTREAM, f"上游 402:疑似额度耗尽/欠费{body_hint(raw)}" + if code in (401, 403): + return UPSTREAM, f"上游/边缘拒绝 HTTP {code}(供应商 Key 失效或 WAF,非网关问题){body_hint(raw)}" + if code == 404: + return UPSTREAM, f"上游 404:链路通但路径不存在(核对管理台 base 与路径拼接){body_hint(raw)}" + if 400 <= code < 500: + return UPSTREAM, f"上游 {code}:探针参数被拒(链路通){body_hint(raw)}" + return UPSTREAM, f"上游 {code}({body_hint(raw).strip() or '无响应体'})" + + +def summarize(results: list, label: str) -> int: + """打印三态汇总,返回建议退出码(gw_fail>0 → 1)。""" + n_fail = sum(1 for r in results if r["status"] == GW_FAIL) + n_warn = sum(1 for r in results if r["status"] in (UPSTREAM, GW_WARN)) + n_pass = sum(1 for r in results if r["status"] == PASS) + print("-" * 66) + print(f"📊 {label}:✅{n_pass} ⚠️{n_warn} ❌{n_fail} (共 {len(results)} 项)") + if n_fail: + print(" ❌ = 网关侧故障(发版要抓的),逐条见上面输出") + if n_warn: + print(" ⚠️ = 网关已放行、上游/厂商侧问题(欠费/不可达/参数),不染红发版,但要有人跟进:") + for r in results: + if r["status"] in (UPSTREAM, GW_WARN): + print(f" ⚠️ {r.get('name', r.get('slug', '?'))}:{r['why']}") + return 1 if n_fail else 0 diff --git a/tests/test_models.py b/tests/test_models.py index 1584dde..db04d4c 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,180 +1,163 @@ #!/usr/bin/env python3 -""" -测试一组兼容接口下的模型是否可用(同时支持 Anthropic + OpenAI 两种格式)。 +"""LLM 真实流量探针 —— 用例目录从 /api/v1/me 动态拉,挨个真调。 -URL 结构: - Anthropic: https://{BASE}/{provider}/anthropic/v1/messages - OpenAI: https://{BASE}/{provider}/v1/chat/completions - provider = deepseek/qwen/glm/minimax(网关按厂商路由) - model = body 里的 model 字段 +历史上这里写死过一组 (slot, model),问题:slot/模型名/双协议地址都是管理台 +**动态配置**,改配置后探针要么静默漏验新上游、要么拿着旧模型名误报。1.0.16 起 +`/api/v1/me` 一处下发全能力目录,探针跟着目录走(共享逻辑在 me_catalog.py)。 + +判定是三态(口径同 me_catalog.py 文件头): + ✅ 2xx 真通;❌ 网关侧故障(发版要抓,退出码 1); + ⚠️ 上游/厂商侧问题(欠费 402、不可达 502/504、厂商鉴权/限流、探针参数被拒) + —— **如实反馈但不染红发版**,网关工作正常、账号没钱不是镜像的锅。 用法: - python test_models.py - python test_models.py --api openai - python test_models.py --api both - python test_models.py --concurrent - python test_models.py --model deepseek-flash,MiniMax-M3 - python test_models.py --timeout 60 - python test_models.py --list-models - python test_models.py --list-models --api openai + python3 test_models.py # 目录全展开,both + 并发 + python3 test_models.py --api openai # 只调 OpenAI 格式 + python3 test_models.py --model deepseek-flash,GLM-5.3-Flash + python3 test_models.py --list-models # 只验各 slot 的 /v1/models + python3 test_models.py --timeout 60 --sequential + +退出码:有 ❌ → 1;只有 ✅/⚠️ → 0;拉不到目录 → 2(验证本身失效,别假绿)。 """ import json -import time +import os import sys -import urllib.request +import time import urllib.error +import urllib.request from concurrent.futures import ThreadPoolExecutor, as_completed -BASE_URL = "https://lm.mayishangshu.cn:88" +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import me_catalog as mc # noqa: E402 + PROMPT = "用一句话介绍你自己。" - -MODELS = [ - ("deepseek", "deepseek-flash"), - ("minimax", "MiniMax-M3"), - ("qwen", "qwen3.8-flash"), - ("glm", "GLM-5.3-Flash"), -] - -PROBE_MODELS = [ - ("deepseek", "deepseek-chat"), - ("qwen", "qwen-plus"), - ("glm", "glm-4-flash"), -] - API_KINDS = ("anthropic", "openai", "both") -def call_anthropic(api_key: str, provider: str, model: str, timeout: int) -> dict: - url = f"{BASE_URL}/{provider}/anthropic/v1/messages" - headers = { - "x-api-key": api_key, - "anthropic-version": "2023-06-01", - "content-type": "application/json", - } - payload = { - "model": model, - "max_tokens": 256, - "messages": [{"role": "user", "content": PROMPT}], - } - body = json.dumps(payload).encode("utf-8") - req = urllib.request.Request(url, data=body, headers=headers, method="POST") - - start = time.time() - try: - with urllib.request.urlopen(req, timeout=timeout) as resp: - raw = resp.read().decode("utf-8") - elapsed = time.time() - start - try: - parsed = json.loads(raw) - except json.JSONDecodeError: - return {"ok": False, "elapsed": elapsed, "error": f"invalid JSON: {raw[:120]}"} - - text = "" - for block in parsed.get("content", []) or []: - if isinstance(block, dict) and block.get("type") == "text": - text += block.get("text", "") - return { - "ok": True, - "elapsed": elapsed, - "status": resp.status, - "text": text.strip(), - "usage": parsed.get("usage", {}), - } - except urllib.error.HTTPError as e: - elapsed = time.time() - start - err_body = e.read().decode("utf-8", errors="replace") if e.fp else "" - return {"ok": False, "elapsed": elapsed, "status": e.code, "error": err_body[:200]} - except Exception as e: - elapsed = time.time() - start - return {"ok": False, "elapsed": elapsed, "error": str(e)} - - -def call_openai(api_key: str, provider: str, model: str, timeout: int) -> dict: - url = f"{BASE_URL}/{provider}/v1/chat/completions" - headers = { - "Authorization": f"Bearer {api_key}", - "content-type": "application/json", - } - payload = { - "model": model, - "max_tokens": 256, - "messages": [{"role": "user", "content": PROMPT}], - } - body = json.dumps(payload).encode("utf-8") - req = urllib.request.Request(url, data=body, headers=headers, method="POST") - - start = time.time() - try: - with urllib.request.urlopen(req, timeout=timeout) as resp: - raw = resp.read().decode("utf-8") - elapsed = time.time() - start - try: - parsed = json.loads(raw) - except json.JSONDecodeError: - return {"ok": False, "elapsed": elapsed, "error": f"invalid JSON: {raw[:120]}"} - - text = "" - choices = parsed.get("choices") or [] - if choices: - msg = choices[0].get("message") or {} - text = msg.get("content", "") - return { - "ok": True, - "elapsed": elapsed, - "status": resp.status, - "text": (text or "").strip(), - "usage": parsed.get("usage", {}), - } - except urllib.error.HTTPError as e: - elapsed = time.time() - start - err_body = e.read().decode("utf-8", errors="replace") if e.fp else "" - return {"ok": False, "elapsed": elapsed, "status": e.code, "error": err_body[:200]} - except Exception as e: - elapsed = time.time() - start - return {"ok": False, "elapsed": elapsed, "error": str(e)} - - -def call(api_kind: str, api_key: str, provider: str, model: str, timeout: int) -> dict: - if api_kind == "openai": - return call_openai(api_key, provider, model, timeout) - return call_anthropic(api_key, provider, model, timeout) - - def mask_key(k: str) -> str: if len(k) <= 12: return "*" * len(k) return k[:6] + "..." + k[-4:] +def build_payload(api_kind: str, model: str) -> bytes: + if api_kind == "anthropic": + body = {"model": model, "max_tokens": 256, + "messages": [{"role": "user", "content": PROMPT}]} + else: + body = {"model": model, "max_tokens": 256, + "messages": [{"role": "user", "content": PROMPT}]} + return json.dumps(body).encode("utf-8") + + +def extra_headers(api_kind: str, key: str) -> dict: + # anthropic 走 x-api-key + 版本头;me_catalog.http 默认发 Bearer,这里补差异 + if api_kind == "anthropic": + return {"x-api-key": key, "anthropic-version": "2023-06-01"} + return {} + + +def call_one(api_key: str, case: dict, api_kind: str, timeout: int) -> dict: + url = case["anthropic"] if api_kind == "anthropic" else case["openai"] + headers = {"User-Agent": mc.DEFAULT_UA, "Content-Type": "application/json"} + headers.update(extra_headers(api_kind, api_key)) + if api_kind != "anthropic": + headers["Authorization"] = f"Bearer {api_key}" + req = urllib.request.Request(url, data=build_payload(api_kind, case["model"]), + headers=headers, method="POST") + start = time.time() + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + code = resp.status + raw = resp.read().decode("utf-8", "replace") + elapsed = time.time() - start + except urllib.error.HTTPError as e: + code = e.code + raw = e.read().decode("utf-8", "replace") if e.fp else "" + elapsed = time.time() - start + except Exception as e: # 超时 / DNS / TLS → classify_llm 会归为网关侧 + code, raw, elapsed = 0, str(e), time.time() - start + status, why = mc.classify_llm(code, raw) + if status == mc.PASS: + # HTTP 200 ≠ 成功(实测:智谱把 NPE 软包成 200 + body 业务 500)——必须查 body + soft = mc.llm_soft_error(api_kind, raw) + if soft: + status, why = mc.UPSTREAM, f"厂商 200 软错误(链路通但没干活):{soft[:90]}" + text = "" + if status == mc.PASS: + try: + d = json.loads(raw) + if api_kind == "anthropic": + text = "".join(b.get("text", "") for b in d.get("content") or [] + if isinstance(b, dict)) + else: + ch = d.get("choices") or [] + text = (ch[0].get("message") or {}).get("content", "") if ch else "" + except Exception: + pass + return {"name": f"{api_kind} {case['slot']}/{case['model'] or '(无模型名)'}", + "slot": case["slot"], "model": case["model"], "api": api_kind, + "code": code, "status": status, "why": why, + "elapsed": elapsed, "text": text.strip(), "raw": raw[:160].replace("\n", " ")} + + +def check_models_list(api_key: str, cases: list, timeout: int) -> list: + """验每个 slot 的 /v1/models(模型发现走网关本地应答,不碰厂商)。""" + seen, results = set(), [] + for c in cases: + if c["slot"] in seen: + continue + seen.add(c["slot"]) + code, raw, elapsed = mc.http("GET", c["models_list"], api_key, timeout) + status, why = mc.classify_llm(code, raw) + ids = [] + if status == mc.PASS: + try: + ids = [it.get("id") for it in (json.loads(raw).get("data") or [])] + except Exception: + why = "响应不是 OpenAI 列表形状" + status = mc.GW_FAIL + results.append({"name": f"models {c['slot']}", "slot": c["slot"], + "code": code, "status": status, "why": why, "elapsed": elapsed, + "ids": ids}) + return results + + +def print_line(r: dict): + icon = mc.ICONS[r["status"]] + msg = f"{icon} {r['name']:<44} {r['elapsed']:6.2f}s HTTP {r['code']}" + if r["status"] == mc.PASS and r.get("text"): + msg += f" | {r['text'].replace(chr(10), ' ')[:40]}" + if r["why"]: + msg += f" | {r['why'][:80]}" + print(msg, flush=True) + + def parse_args(argv): - api_key = None - concurrent = False - timeout = 30 - only = None - probe = False - list_models = False - api_kind = "anthropic" + api_key, timeout, only = None, 60, None + api_kind, concurrent, list_models = "both", True, False i = 1 while i < len(argv): a = argv[i] if a == "--concurrent": concurrent = True - elif a == "--probe": - probe = True + elif a == "--sequential": + concurrent = False elif a == "--list-models": list_models = True elif a.startswith("--api"): v = a.split("=", 1)[1] if "=" in a else argv[i + 1] - if "=" not in a: - i += 1 if v not in API_KINDS: print(f"--api 必须是 {API_KINDS} 之一"); sys.exit(2) api_kind = v + i += 0 if "=" in a else 1 elif a.startswith("--timeout"): - timeout = int(a.split("=", 1)[1]) if "=" in a else int(argv[i + 1]); i += 1 if "=" not in a else 0 + timeout = int(a.split("=", 1)[1]) if "=" in a else int(argv[i + 1]) + i += 0 if "=" in a else 1 elif a.startswith("--model"): - raw = a.split("=", 1)[1] if "=" in a else argv[i + 1]; i += 1 if "=" not in a else 0 + raw = a.split("=", 1)[1] if "=" in a else argv[i + 1] + i += 0 if "=" in a else 1 only = set(raw.split(",")) elif api_key is None: api_key = a @@ -182,136 +165,60 @@ def parse_args(argv): if api_key is None: print(__doc__) sys.exit(1) - return api_key, concurrent, timeout, only, probe, list_models, api_kind - - -def list_available(api_key: str, provider: str, api_kind: str, timeout: int) -> dict: - """试 GET /v1/models 看网关曝不曝模型列表""" - if api_kind == "openai": - paths = ( - f"{BASE_URL}/{provider}/v1/models", - f"{BASE_URL}/v1/models", - ) - headers = {"Authorization": f"Bearer {api_key}"} - else: - paths = ( - f"{BASE_URL}/{provider}/anthropic/v1/models", - f"{BASE_URL}/{provider}/v1/models", - f"{BASE_URL}/v1/models", - ) - headers = {"x-api-key": api_key, "anthropic-version": "2023-06-01"} - - for path in paths: - try: - req = urllib.request.Request(path, headers=headers) - with urllib.request.urlopen(req, timeout=timeout) as resp: - return {"ok": True, "path": path, "body": resp.read().decode("utf-8")} - except urllib.error.HTTPError as e: - last_err = (e.code, e.read().decode("utf-8", errors="replace")[:120]) - except Exception as e: - last_err = ("?", str(e)[:120]) - return {"ok": False, "error": last_err} + return api_key, api_kind, timeout, only, concurrent, list_models def main(): - api_key, concurrent, timeout, only, probe, list_models, api_kind = parse_args(sys.argv) + api_key, api_kind, timeout, only, concurrent, list_models = parse_args(sys.argv) - if list_models: - for kind in (["openai"] if api_kind == "openai" else (API_KINDS if api_kind == "both" else ["anthropic"])): - print(f"🔑 key={mask_key(api_key)} 🔌 {kind:<9} 探测模型列表...") - print("-" * 60) - for p in ("deepseek", "qwen", "glm", "minimax"): - print(f"⏳ {p:<10} ... ", end="", flush=True) - r = list_available(api_key, p, kind, timeout) - if r["ok"]: - print(f"✅ {r['path']}") - try: - parsed = json.loads(r["body"]) - ids = [it["id"] for it in (parsed.get("data") or []) if isinstance(it, dict) and "id" in it] - if ids: - print(f" models: {', '.join(ids)}") - else: - print(f" raw: {r['body'][:300]}") - except Exception: - print(f" raw: {r['body'][:300]}") - else: - print(f"❌ {r['error']}") - print() - return - - # 确定要跑哪些 (api_kind, provider, model) 组合 - combos = [] - kinds = ["openai"] if api_kind == "openai" else (API_KINDS[:-1] if api_kind == "both" else ["anthropic"]) - todo = MODELS - if probe: - todo = list(MODELS) + PROBE_MODELS + # 1) 拉能力目录(拉不到就退 2 —— 探针失效不能假绿,也别拿写死数据凑) + me, err = mc.fetch_me(api_key, timeout) + if me is None: + print(f"❌ 拉 /api/v1/me 失败:{err}") + print(" 目录拉不到 = 探针不知道要验什么,退出(不是上游问题)") + sys.exit(2) + cases = mc.llm_cases(me) if only: - todo = [(p, m) for (p, m) in todo if m in only] - if not todo: - print("没有匹配的模型"); sys.exit(1) - for kind in kinds: - for p, m in todo: - combos.append((kind, p, m)) + cases = [c for c in cases if c["model"] in only] + skipped = [u.get("slot") for u in me.get("upstreams") or [] if not u.get("enabled")] - print(f"🔑 key={mask_key(api_key)} 🌐 {BASE_URL} ⏱ timeout={timeout}s " - f"⚙ {'concurrent' if concurrent else 'sequential'} " - f"🔌 {api_kind}{' 🔎probe' if probe else ''}") - print(f"📝 prompt: {PROMPT}") - print("-" * 60) + print(f"🔑 key={mask_key(api_key)} 📦 目录={len(cases)} 个用例(上游 {len(me.get('upstreams') or [])} 个" + + (f",停用跳过 {','.join(map(str, skipped))}" if skipped else "") + + f") ⏱ timeout={timeout}s 🔌 {api_kind} ⚙ {'并发' if concurrent else '串行'}") + print("-" * 66) + if not cases: + print("· 目录里没有可验的上游/模型(全部停用或 models 为空)—— 请核对管理台配置") + sys.exit(0) + + # 2) /v1/models 模型发现(--list-models 时只跑这个;成本低、不碰厂商) results = [] + if list_models: + ml_results = check_models_list(api_key, cases, timeout) + for r in ml_results: + print_line(r) + if r.get("ids"): + print(f" models: {', '.join(str(x) for x in r['ids'])}") + sys.exit(mc.summarize(ml_results, "模型发现")) + + kinds = (["openai", "anthropic"] if api_kind == "both" else [api_kind]) + combos = [(k, c) for k in kinds for c in cases] if concurrent: with ThreadPoolExecutor(max_workers=min(8, len(combos))) as ex: - futs = {ex.submit(call, k, api_key, p, m, timeout): (k, p, m) for (k, p, m) in combos} + futs = {ex.submit(call_one, api_key, c, k, timeout): (k, c) for k, c in combos} for fut in as_completed(futs): - k, p, m = futs[fut] r = fut.result() - r.update({"api": k, "provider": p, "model": m}) results.append(r) print_line(r) else: - for k, p, m in combos: - print(f"⏳ [{k:<9}] {p}/{m:<16} ... ", end="", flush=True) - r = call(k, api_key, p, m, timeout) - r.update({"api": k, "provider": p, "model": m}) + for k, c in combos: + r = call_one(api_key, c, k, timeout) results.append(r) - print_line(r, newline=True) + print_line(r) - print("-" * 60) - ok = sum(1 for r in results if r["ok"]) - print(f"📊 {ok}/{len(results)} OK") - - # 汇总:按 api 分组 - by_api = {} - for r in results: - by_api.setdefault(r["api"], {"ok": 0, "total": 0}) - by_api[r["api"]]["total"] += 1 - if r["ok"]: - by_api[r["api"]]["ok"] += 1 - if len(by_api) > 1: - summary = " ".join(f"{k}:{v['ok']}/{v['total']}" for k, v in by_api.items()) - print(f" by api: {summary}") - - # 退出码:全绿 0,有失败 1 —— 供 tests/live_check.sh 之类的包装脚本判定 - sys.exit(0 if ok == len(results) else 1) - - -def print_line(r: dict, newline: bool = False): - tag = f"[{r.get('api','?'):<9}] {r.get('provider','?')}/{r.get('model','?')}" - if r["ok"]: - msg = f"✅ {tag:<32} {r['elapsed']:6.2f}s" - if r.get("text"): - preview = r["text"].replace("\n", " ")[:40] - msg += f" | {preview}" - else: - err = (r.get("error") or "unknown").replace("\n", " ")[:60] - msg = f"❌ {tag:<32} {r['elapsed']:6.2f}s | {err}" - if newline: - print(msg) - else: - print(msg) + sys.exit(mc.summarize(results, "LLM 真实调用")) if __name__ == "__main__": - main() \ No newline at end of file + main()