1. me_catalog.py 把 .env 约定的**裸主机名**当完整 URL 用:release 里
agent.sh source .env 后子进程继承,live_check 两步探针 ValueError 崩退 1,
被三态口径判成「网关侧故障」触发自动回滚(1.1.1 实测,健康版本差点被回)。
现在补 scheme + IPASS_PUBLIC_PORT;探针脚本未捕获异常统一退 2(验证不可用),
崩溃永远不该触发回滚。
2. rollback 参数键位漂移:远端控制面对 release 发的 {"tag":...} 报「需要显式传
tag」→ 自动回滚 job 失败。agent.sh 的 rollback/回滚体改 python 拼 JSON 并
tag/image 双写;ipass-agent 1.0.4 的 op_rollback 两种键都认、短 tag/裸版本
用受管容器仓库段补全;release 版本闸门抬到 1.0.4(自动回滚必须先证明可用)。
3. 线上默认 tag 迁出 .env → deploy/image.conf(非秘密、入库):release 成功
自动回写 + commit,不再需要人肉同步;.env 的 IPASS_IMAGE_TAG 仅临时覆盖。
另修 agent.sh upstream-test:旧契约(slot 塞 body、无探测体)换成面板同款
(query slot + 第一个登记模型),实测 minimax 200。
169 lines
7.0 KiB
Python
169 lines
7.0 KiB
Python
#!/usr/bin/env python3
|
||
"""apix 上线后探针 —— 条目从 /api/v1/me 动态拉,逐个验 vendor / passthrough 转发。
|
||
|
||
与 test_models.py 共用 me_catalog.py(目录拉取 + 三态判定),口径完全一致:
|
||
❌ 网关侧故障(location 没挂 / 鉴权没生效 / 连不上)→ 退出码 1,发版要抓;
|
||
⚠️ 上游/供应商侧问题(欠费 402、服务不可达 502、WAF 拦截、探针参数被拒)
|
||
→ **如实打印并汇总列出**,但不染红发版验证。
|
||
|
||
探针构造(参数名/路径都跟着目录走,不写死):
|
||
vendor → `GET /apix/<slug>?<p>=<占位>`,参数名取 `params[0]`,没有就从
|
||
`desc`/`usage_hint` 正则找第一个 `?xx=`(如 accel 的 `?url=`)。
|
||
passthrough → `GET /apixp/<slug>/health`(轻量,不触发 GPU 懒加载)。
|
||
tail 是 `/apixp/<slug>` **之后**的全部:上游 `/health` 对应
|
||
`/apixp/ocr/health`;usage_hint 里的 `/<tail>` 要换成 slug,别照抄。
|
||
|
||
每条探针前还做一次**无效 Key 预检**(期望 401 + 网关体):证明 location 挂着、
|
||
鉴权生效——与上游/供应商是否抽风无关。若连无效 Key 都没被网关拦下,判 ❌。
|
||
|
||
用法:
|
||
python3 tests/apix_check.py <key> # 探针全部 enabled 条目
|
||
python3 tests/apix_check.py <key> --slug ocr # 只探某几个(逗号分隔)
|
||
python3 tests/apix_check.py <key> --timeout 40 --no-retry
|
||
|
||
退出码:有 ❌ → 1;只有 ✅/⚠️ → 0;拉不到目录 → 2(探针失效,不是上游没问题)。
|
||
"""
|
||
|
||
import os
|
||
import re
|
||
import sys
|
||
import time
|
||
import urllib.parse
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
import me_catalog as mc # noqa: E402
|
||
|
||
# vendor 参数占位值:让请求「格式合法」即可。accel 这类只放行白名单 host
|
||
# (GitHub/Gitee/GitLab)——用最小 raw 文件(~485B),拿 200 又不吃额度。
|
||
PLACEHOLDER = {
|
||
"q": "ping", "query": "ping", "keyword": "ping", "text": "ping",
|
||
"url": "https://raw.githubusercontent.com/octocat/Hello-World/master/README",
|
||
}
|
||
|
||
# 无效 Key 预检:形态合法(sk_ 开头)但查不到,网关应返 401
|
||
BAD_KEY = "sk_" + "0" * 48
|
||
|
||
# 只有「疑似限流/抖动」值得重试(souyi 边缘偶发拦截);404 重试没意义
|
||
RETRYABLE = (403, 429, 0)
|
||
RETRY_WAIT_SEC = 15
|
||
|
||
|
||
def build_url(item: dict):
|
||
"""按 mode 构造探针 URL / 方法。返回 (method, url, body, ctype) 或 None。"""
|
||
slug = item.get("slug")
|
||
mode = item.get("mode") or "vendor"
|
||
if mode == "passthrough":
|
||
return "GET", f"{mc.LM}/apixp/{slug}/health", None, None
|
||
params = item.get("params") or []
|
||
p = str(params[0]) if params else None
|
||
if not p:
|
||
hint = f"{item.get('desc') or ''} {item.get('usage_hint') or ''}"
|
||
m = re.search(r"[?&](\w+)=", hint)
|
||
p = m.group(1) if m else None
|
||
query = f"?{urllib.parse.urlencode({p: PLACEHOLDER.get(p, 'test')})}" if p else ""
|
||
return "GET", f"{mc.LM}/apix/{slug}{query}", None, None
|
||
|
||
|
||
def probe(item: dict, key: str, timeout: int) -> dict:
|
||
built = build_url(item)
|
||
slug = item.get("slug") or "?"
|
||
mode = item.get("mode") or "vendor"
|
||
if not built:
|
||
return {"slug": slug, "mode": mode, "url": "-", "code": 0,
|
||
"status": mc.GW_FAIL, "why": "探针 URL 构造失败", "elapsed": 0,
|
||
"raw": "", "auth_ok": False, "auth_code": 0}
|
||
method, url, body, ctype = built
|
||
code, raw, elapsed = mc.http(method, url, key, timeout, body, ctype)
|
||
status, why = mc.classify_apix(code, raw)
|
||
|
||
# 无效 Key 预检(期望 401 且归属是网关,双判据见 me_catalog.from_gateway)
|
||
acode, araw, _ = mc.http(method, url, BAD_KEY, timeout, body, ctype)
|
||
auth_ok = (acode == 401 and mc.from_gateway(acode, araw)[0])
|
||
if not auth_ok:
|
||
status = mc.GW_FAIL
|
||
why = f"网关鉴权未生效(无效 Key 返回 HTTP {acode},期望 401 网关体)"
|
||
|
||
return {"slug": slug, "mode": mode, "url": url, "code": code, "status": status,
|
||
"why": why, "elapsed": elapsed, "raw": raw[:160].replace("\n", " "),
|
||
"auth_ok": auth_ok, "auth_code": acode}
|
||
|
||
|
||
def parse_args(argv):
|
||
key, timeout, only, retry = None, 30, None, True
|
||
i = 1
|
||
while i < len(argv):
|
||
a = argv[i]
|
||
if a.startswith("--timeout"):
|
||
timeout = int(a.split("=", 1)[1]) if "=" in a else int(argv[i + 1])
|
||
i += 1 if "=" not in a else 0
|
||
elif a == "--no-retry":
|
||
retry = False
|
||
elif a.startswith("--slug"):
|
||
raw = a.split("=", 1)[1] if "=" in a else argv[i + 1]
|
||
i += 1 if "=" not in a else 0
|
||
only = set(raw.split(","))
|
||
elif key is None:
|
||
key = a
|
||
i += 1
|
||
if key is None:
|
||
print(__doc__)
|
||
sys.exit(1)
|
||
return key, timeout, only, retry
|
||
|
||
|
||
def main():
|
||
key, timeout, only, retry = parse_args(sys.argv)
|
||
me, err = mc.fetch_me(key, timeout)
|
||
if me is None:
|
||
print(f"❌ 拉 /api/v1/me 失败:{err}")
|
||
print(" 拉不到目录 = 探针不知道要验什么(不是上游没问题),退出")
|
||
sys.exit(2)
|
||
|
||
items = me.get("apix") or []
|
||
todo = [it for it in items if it.get("enabled")]
|
||
if only:
|
||
todo = [it for it in todo if it.get("slug") in only]
|
||
disabled = [it.get("slug") for it in items if not it.get("enabled")]
|
||
|
||
print(f"🌐 {mc.LM} ⏱ timeout={timeout}s 📦 apix 条目 {len(todo)} 个待探"
|
||
+ (f"(停用跳过:{', '.join(map(str, disabled))})" if disabled else ""))
|
||
print("-" * 66)
|
||
|
||
results = []
|
||
for it in todo:
|
||
print(f"⏳ {it.get('slug'):<10} ... ", end="", flush=True)
|
||
r = probe(it, key, timeout)
|
||
if retry and r["status"] in (mc.UPSTREAM, mc.GW_WARN) and r["code"] in RETRYABLE:
|
||
time.sleep(RETRY_WAIT_SEC)
|
||
r2 = probe(it, key, timeout)
|
||
if r2["status"] not in (mc.UPSTREAM, mc.GW_WARN):
|
||
r2["retried"] = True
|
||
r = r2
|
||
results.append(r)
|
||
icon = mc.ICONS[r["status"]]
|
||
mark = "鉴权✓" if r["auth_ok"] else f"鉴权✗(HTTP {r['auth_code']})"
|
||
msg = f"{icon} [{r['mode']:<11}] {r['slug']:<10} {r['elapsed']:6.2f}s HTTP {r['code']} {mark}"
|
||
if r.get("retried"):
|
||
msg += " (重试后)"
|
||
if r["why"]:
|
||
msg += f" | {r['why']}"
|
||
print(msg)
|
||
if r["code"] != 0 and r["status"] != mc.PASS and r["raw"]:
|
||
print(f" └ {r['raw']}")
|
||
|
||
if not results:
|
||
print("· 目录里没有 enabled 的 apix 条目 —— 无事可做")
|
||
sys.exit(0)
|
||
sys.exit(mc.summarize(results, "apix 探针"))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
main()
|
||
except SystemExit:
|
||
raise
|
||
except Exception as e: # noqa: BLE001
|
||
# 同 test_models.py:脚本崩溃 = 验证手段不可用(2),绝不能退 1 触发回滚。
|
||
print(f"✗ 探针脚本自身异常,按「验证不可用」处理(不触发回滚):{type(e).__name__}: {e}", file=sys.stderr)
|
||
sys.exit(2)
|