1. me_catalog.py 把 .env 约定的**裸主机名**当完整 URL 用:release 里
agent.sh source .env 后子进程继承,live_check 两步探针 ValueError 崩退 1,
被三态口径判成「网关侧故障」触发自动回滚(1.1.1 实测,健康版本差点被回)。
现在补 scheme + IPASS_PUBLIC_PORT;探针脚本未捕获异常统一退 2(验证不可用),
崩溃永远不该触发回滚。
2. rollback 参数键位漂移:远端控制面对 release 发的 {"tag":...} 报「需要显式传
tag」→ 自动回滚 job 失败。agent.sh 的 rollback/回滚体改 python 拼 JSON 并
tag/image 双写;ipass-agent 1.0.4 的 op_rollback 两种键都认、短 tag/裸版本
用受管容器仓库段补全;release 版本闸门抬到 1.0.4(自动回滚必须先证明可用)。
3. 线上默认 tag 迁出 .env → deploy/image.conf(非秘密、入库):release 成功
自动回写 + commit,不再需要人肉同步;.env 的 IPASS_IMAGE_TAG 仅临时覆盖。
另修 agent.sh upstream-test:旧契约(slot 塞 body、无探测体)换成面板同款
(query slot + 第一个登记模型),实测 minimax 200。
278 lines
13 KiB
Python
278 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""/api/v1/me 能力目录 + 三态判定 —— test_models.py / apix_check.py 共享模块。
|
||
|
||
为什么要统一从这里拉:slot、模型名、apix 条目都是**管理台动态配置**,写死在探针
|
||
脚本里等于「发版验证验的是半年前的配置」——加个上游、改个模型名,脚本就静默漏验
|
||
或误报。1.0.16 起 `/api/v1/me` 把 upstreams + apix 一处下发全(见
|
||
docs/plugin/ME_CAPABILITY_CATALOG.md),探针跟着目录走,配置变了验证自动跟上。
|
||
|
||
三态判定(发版验证的核心口径):
|
||
✅ pass —— 2xx,链路真的通
|
||
❌ gw_fail —— **网关侧**问题(错误体是网关形态 / 连不上网关 / 网关 4xx)
|
||
→ 发版要抓的失败,退出码 1
|
||
⚠️ upstream —— 请求已穿过网关、被供应商/上游拒了(欠费 402/429、不可达 502/504、
|
||
鉴权过期 401…)→ **如实反馈但不判发版失败**:那是一方账号的问题,
|
||
不是新镜像的问题。告警会汇总打印,不静默。
|
||
|
||
区分「谁给的错误」靠错误体形态(实测归纳):
|
||
网关 LLM 体 {"error":{"type":..,"code":..,"message":..}}
|
||
网关管理/apix {"code":401,"msg":"..."}(扁平)
|
||
上游 FastAPI {"detail":...};厂商各家自有;Cloudflare 拦截页是 HTML。
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
|
||
def _base_url(env_name: str, default_host: str) -> str:
|
||
""".env 约定 IPASS_MDM_HOST / IPASS_LM_HOST 存**裸主机名**(与 agent.sh 的
|
||
LIVE_HOST + LIVE_PORT 拆分同一口径,宿主端口在 IPASS_PUBLIC_PORT,三域名共用)。
|
||
|
||
直接当完整 URL 用会 `ValueError: unknown url type`——1.1.1 发版实测:release
|
||
里 agent.sh 已 source .env,子进程继承了裸主机名,live_check 两步探针全崩,
|
||
还因「未捕获异常退 1」被误判成网关侧故障触发了自动回滚。这里统一补 scheme 与端口。
|
||
"""
|
||
v = (os.environ.get(env_name) or default_host).strip().rstrip("/")
|
||
if "://" not in v:
|
||
v = "https://" + v
|
||
hostport = v.split("://", 1)[1].split("/", 1)[0]
|
||
if ":" not in hostport:
|
||
v = "%s:%s" % (v, (os.environ.get("IPASS_PUBLIC_PORT") or "88").strip())
|
||
return v
|
||
|
||
|
||
MDM = _base_url("IPASS_MDM_HOST", "mdm.mayishangshu.cn")
|
||
LM = _base_url("IPASS_LM_HOST", "lm.mayishangshu.cn")
|
||
|
||
# 供应商边缘(souyi 走 Cloudflare)会拦默认 `Python-urllib/3.x` → 403 error code:1010
|
||
# (2026-09-17 实测,同请求换 curl UA 立刻 200)。可被 IPASS_UA 覆盖。
|
||
DEFAULT_UA = "curl/8.4.0"
|
||
|
||
# 三态 + 网关告警(429 限流是「探针自己打太快」,不该染红发版)
|
||
PASS, UPSTREAM, GW_WARN, GW_FAIL = "pass", "upstream", "gw_warn", "gw_fail"
|
||
ICONS = {PASS: "✅", UPSTREAM: "⚠️ ", GW_WARN: "⚠️ ", GW_FAIL: "❌"}
|
||
|
||
|
||
def http(method: str, url: str, key: str, timeout: int, body: bytes | None = None,
|
||
ctype: str | None = None, auth: str = "Bearer"):
|
||
"""发请求,返回 (code, body_text, elapsed)。网络层异常 code=0。
|
||
|
||
`auth`:网关统一收 `Authorization: Bearer <sk_>`,但 **`/api/v1/me` 只认
|
||
`X-API-Key`**(2026-09-17 实测:带 Bearer 会 401)。
|
||
"""
|
||
headers = {"X-API-Key": key} if auth == "X-API-Key" else {"Authorization": f"Bearer {key}"}
|
||
headers["User-Agent"] = os.environ.get("IPASS_UA", DEFAULT_UA)
|
||
if ctype:
|
||
headers["Content-Type"] = ctype
|
||
req = urllib.request.Request(url, data=body, headers=headers, method=method)
|
||
start = time.time()
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||
return resp.status, resp.read().decode("utf-8", "replace"), time.time() - start
|
||
except urllib.error.HTTPError as e:
|
||
raw = e.read().decode("utf-8", "replace") if e.fp else ""
|
||
return e.code, raw, time.time() - start
|
||
except Exception as e: # 超时 / DNS / TLS
|
||
return 0, str(e), time.time() - start
|
||
|
||
|
||
def fetch_me(key: str, timeout: int = 20):
|
||
"""拉 /api/v1/me 的 data(含 upstreams + apix)。失败返回 (None, 原因)。"""
|
||
code, raw, _ = http("GET", f"{MDM}/api/v1/me", key, timeout, auth="X-API-Key")
|
||
if code != 200:
|
||
return None, f"HTTP {code} {raw[:160]}"
|
||
try:
|
||
return json.loads(raw).get("data") or {}, ""
|
||
except json.JSONDecodeError:
|
||
return None, f"响应不是 JSON:{raw[:160]}"
|
||
|
||
|
||
def gateway_error(raw: str):
|
||
"""是网关自己返的错误体则返回 (code, msg),否则 (False, None)。
|
||
|
||
两种网关形态(实测):LLM 侧嵌套 `{"error":{...}}`;管理/apix 侧扁平
|
||
`{"code":N,"msg":"..."}`。上游 FastAPI(detail)与 Cloudflare(HTML)都不匹配。
|
||
"""
|
||
try:
|
||
d = json.loads(raw)
|
||
except json.JSONDecodeError:
|
||
return False, None
|
||
if not isinstance(d, dict):
|
||
return False, None
|
||
err = d.get("error")
|
||
if isinstance(err, dict):
|
||
return True, f"{err.get('code') or err.get('type') or ''} {err.get('message') or ''}".strip()
|
||
if "code" in d and "msg" in d:
|
||
return True, f"{d.get('code')} {d.get('msg')}".strip()
|
||
return False, None
|
||
|
||
|
||
def llm_cases(me: dict):
|
||
"""把目录展开成真实验证用例:每个 enabled 上游 × 每个配置模型(挨个跑)。
|
||
|
||
返回 [{slot, model, openai, anthropic}];`allowed_models` 白名单非空时按其过滤
|
||
(否则会拿 403 permission_denied,那是账号策略不是链路故障)。
|
||
"""
|
||
allowed = set(me.get("allowed_models") or [])
|
||
cases = []
|
||
for u in me.get("upstreams") or []:
|
||
if not u.get("enabled"):
|
||
continue
|
||
slot = u.get("slot") or "?"
|
||
base_o = (u.get("base_url") or f"{LM}/{slot}").rstrip("/")
|
||
base_a = (u.get("anthropic_base_url") or u.get("base_anthropic")
|
||
or f"{LM}/{slot}/anthropic").rstrip("/")
|
||
models = u.get("models") or []
|
||
if allowed and slot not in allowed:
|
||
continue
|
||
if not models:
|
||
# 没配模型名:用占位探一次,让「配置缺失」如实暴露(网关会返回模型拒绝)
|
||
models = [""]
|
||
for m in models:
|
||
cases.append({
|
||
"slot": slot, "model": m,
|
||
"openai": f"{base_o}/v1/chat/completions",
|
||
"anthropic": f"{base_a}/v1/messages",
|
||
"models_list": f"{base_o}/v1/models",
|
||
})
|
||
return cases
|
||
|
||
|
||
def apix_items(me: dict):
|
||
"""enabled 的 apix 条目(探针构造留给 apix_check.py)。"""
|
||
return [it for it in (me.get("apix") or []) if it.get("enabled")]
|
||
|
||
|
||
# 网关(Rust 侧 auth::fail)只可能发这些状态码;402/5xx 等出现即说明是上游透传。
|
||
GW_CODES = {400, 401, 403, 404, 429}
|
||
# 网关 LLM 体 error.code 的**全集**(grep auth::fail 得来)——厂商(智谱等)的 401/404
|
||
# 体也是 {"error":{code,message}} 形状,光看形态+码集合会误判,error.code 必须也在
|
||
# 这个集合里才算网关(HTTP 码与 error.code 双对得上才是网关手笔)。
|
||
GW_ERR_CODES = {"invalid_api_key", "model_not_found", "permission_denied",
|
||
"rate_limit_exceeded", "insufficient_quota", "invalid_request_error"}
|
||
|
||
|
||
def from_gateway(code: int, raw: str):
|
||
"""(是否网关所返, 网关 msg)。形态 + HTTP 码 + error.code 值三重判据。"""
|
||
if code not in GW_CODES:
|
||
return False, None
|
||
try:
|
||
d = json.loads(raw)
|
||
except json.JSONDecodeError:
|
||
return False, None
|
||
if not isinstance(d, dict):
|
||
return False, None
|
||
err = d.get("error")
|
||
if isinstance(err, dict):
|
||
c = err.get("code") or ""
|
||
if c in GW_ERR_CODES:
|
||
return True, f"{c} {err.get('message') or ''}".strip()
|
||
return False, None
|
||
if "code" in d and "msg" in d:
|
||
# 扁平管理体(/api/v1/me、apix、admin):网关独有形态,无厂商撞车(实测)
|
||
return True, f"{d.get('code')} {d.get('msg')}".strip()
|
||
return False, None
|
||
|
||
|
||
def llm_soft_error(api_kind: str, raw: str):
|
||
"""HTTP 200 但 body 无正文 = 厂商软错误(实测:智谱把 NPE 包成 200+业务 500)。
|
||
返回错误摘要或 None。"""
|
||
try:
|
||
d = json.loads(raw)
|
||
except json.JSONDecodeError:
|
||
return "响应不是 JSON"
|
||
if not isinstance(d, dict):
|
||
return None
|
||
if api_kind == "anthropic":
|
||
if d.get("content"):
|
||
return None
|
||
else:
|
||
if d.get("choices"):
|
||
return None
|
||
if d.get("type") == "error" or "error" in d or ("code" in d and d.get("code") not in (0, "0", 200, "200")):
|
||
return json.dumps(d, ensure_ascii=False)[:120]
|
||
# 没有正文字段也没有明显错误键:形状不认识也算软错误
|
||
return f"200 但无正文(形状不认识): {json.dumps(d, ensure_ascii=False)[:100]}"
|
||
|
||
|
||
def classify_llm(code: int, raw: str) -> tuple[str, str]:
|
||
"""LLM 调用响应 → 三态。见文件头口径。"""
|
||
if code == 0:
|
||
# 连的是网关 host(lm/mdm),连不上 = 网关/网络问题,不是上游
|
||
return GW_FAIL, "连不上网关(超时/DNS/TLS)"
|
||
if 200 <= code < 300:
|
||
return PASS, ""
|
||
is_gw, msg = from_gateway(code, raw)
|
||
if is_gw:
|
||
if code == 429:
|
||
# 限流/每日配额是账号策略(探针打太快或 Key 超额),不是发版故障
|
||
return GW_WARN, f"网关限流/配额:{msg}(非发版故障,如实反馈)"
|
||
return GW_FAIL, f"网关拒绝 HTTP {code}:{msg}"
|
||
# —— 以下都是「请求已穿过网关、上游/厂商拒的」→ 如实反馈、不染红
|
||
if code in (502, 504):
|
||
return UPSTREAM, f"上游不可达(nginx {code},网关已转发但厂商连不上)"
|
||
if code == 402:
|
||
return UPSTREAM, f"厂商 402:疑似额度耗尽/欠费(全账号冷却也不救=没额度了){body_hint(raw)}"
|
||
if code == 429:
|
||
return UPSTREAM, f"厂商 429:限流{body_hint(raw)}"
|
||
if code in (401, 403):
|
||
return UPSTREAM, f"厂商鉴权 {code}:上游 Key 可能过期/失效(管理台换 Key){body_hint(raw)}"
|
||
if code == 404:
|
||
return UPSTREAM, f"厂商 404:模型名/路径不存在(核对管理台 models){body_hint(raw)}"
|
||
if 400 <= code < 500:
|
||
return UPSTREAM, f"厂商 {code}:探针参数被拒(链路通){body_hint(raw)}"
|
||
return UPSTREAM, f"厂商 {code}{body_hint(raw)}(上游故障——HTTP 200≠成功,body 是原话)"
|
||
|
||
|
||
def body_hint(raw: str) -> str:
|
||
s = (raw or "").replace("\n", " ").strip()
|
||
return f"|{s[:80]}" if s else ""
|
||
|
||
|
||
def classify_apix(code: int, raw: str) -> tuple[str, str]:
|
||
"""apix 探针响应 → 三态。归属判定与 classify_llm 同一套双判据。
|
||
|
||
apix 的 401/403 有两种来源:网关(Key 无效)或供应商/内部服务(上游 Key 失效、
|
||
WAF)。**网关码集合 + 网关体形态**同时命中才判 ❌;上游 FastAPI(detail)与
|
||
WAF 页(HTML)都不匹配 → ⚠️ 如实反馈。
|
||
"""
|
||
if code == 0:
|
||
return GW_FAIL, "连不上网关(超时/DNS/TLS)"
|
||
if 200 <= code < 300:
|
||
return PASS, ""
|
||
is_gw, msg = from_gateway(code, raw)
|
||
if is_gw:
|
||
if code == 429:
|
||
return GW_WARN, f"网关限流/配额:{msg}(探针打太快,非发版故障)"
|
||
return GW_FAIL, f"网关拒绝 HTTP {code}:{msg}"
|
||
if code in (502, 504):
|
||
return UPSTREAM, f"上游不可达(nginx {code},网关已转发但服务连不上){body_hint(raw)}"
|
||
if code == 402:
|
||
return UPSTREAM, f"上游 402:疑似额度耗尽/欠费{body_hint(raw)}"
|
||
if code in (401, 403):
|
||
return UPSTREAM, f"上游/边缘拒绝 HTTP {code}(供应商 Key 失效或 WAF,非网关问题){body_hint(raw)}"
|
||
if code == 404:
|
||
return UPSTREAM, f"上游 404:链路通但路径不存在(核对管理台 base 与路径拼接){body_hint(raw)}"
|
||
if 400 <= code < 500:
|
||
return UPSTREAM, f"上游 {code}:探针参数被拒(链路通){body_hint(raw)}"
|
||
return UPSTREAM, f"上游 {code}({body_hint(raw).strip() or '无响应体'})"
|
||
|
||
|
||
def summarize(results: list, label: str) -> int:
|
||
"""打印三态汇总,返回建议退出码(gw_fail>0 → 1)。"""
|
||
n_fail = sum(1 for r in results if r["status"] == GW_FAIL)
|
||
n_warn = sum(1 for r in results if r["status"] in (UPSTREAM, GW_WARN))
|
||
n_pass = sum(1 for r in results if r["status"] == PASS)
|
||
print("-" * 66)
|
||
print(f"📊 {label}:✅{n_pass} ⚠️{n_warn} ❌{n_fail} (共 {len(results)} 项)")
|
||
if n_fail:
|
||
print(" ❌ = 网关侧故障(发版要抓的),逐条见上面输出")
|
||
if n_warn:
|
||
print(" ⚠️ = 网关已放行、上游/厂商侧问题(欠费/不可达/参数),不染红发版,但要有人跟进:")
|
||
for r in results:
|
||
if r["status"] in (UPSTREAM, GW_WARN):
|
||
print(f" ⚠️ {r.get('name', r.get('slug', '?'))}:{r['why']}")
|
||
return 1 if n_fail else 0
|