Files
admin f179e6e987 fix(llm): is_probe 跳过 mark_exhausted/bump,只留 mark_health (1.1.2)
线上事故回归:面板「测试」按钮走 /_upstream_test 真实转发,账号由 pick 随机
选;上游一旦回耗尽形态(状态码命中 upe 或 body 命中 marker),旧逻辑会把
「管理员手抖点测试」当成「账号额度耗尽」冷却一把好钥匙,恢复要走完整退避/
探测。13:50 一次面板测试轮把 minimax 三把好 key 全部点进冷却,整池不可用。

* handlers.rs LlmHealthHandler:is_probe URI 前缀请求**不写冷却/耗尽**;只
  跑 mark_health(健康展示仍需)。2xx clear_cool 保留——测试成功 = 该账号此刻
  健康,是正向证据。bump_acct_load 保留(测试也是真转发,统计有用)。
* tests/llm/e2e.sh:新增「管理台测试按钮不得写路由状态」剧本,6 条断言;
  fixture probetest 单账号 Bearer probe-dead + exhaust_status=[401] +
  markers 一并命中——status 通道 + body 通道都能触发,真实分流修复前 cool
  必须为 true,修复后为 false;真实流量同时仍正常冷却(不能为避 bug 把真标记
  也关掉)。
* tests/llm/fixtures/llm-upstreams.json:加 probetest 槽位(同形 401+marker)。

回归:bash tests/run.sh llm → 270/0 全绿(原 262 + 8)。
2026-09-18 12:10:33 +08:00

1119 lines
65 KiB
Bash
Executable File
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/bash
# ============================================================================
# LLM 代理 · 本机回归
#
# 一般不用直接跑,走统一入口: bash tests/run.sh llm
# 直接跑需自行保证:假上游(tests/llm/upstream.py)已在 18999 监听,
# nginx 已按 tests/llm/nginx.conf.in 渲染并启动,且 data/ 用的是 fixtures 里
# 那两个固定文件(alice 白名单=openai、bob 白名单为空=不限)。
#
# ★ 核心:不只验网关自己的响应码,而是**读靶机实际收到的请求**——
# 「下游 Key 是否真被换成上游真 Key」只有看靶机才能证明。
#
# 可覆盖:GW / UPSTREAM_LOG / PY
# ============================================================================
set -u
HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
RUN=${RUN:-$(cd "$HERE/.." && pwd)/.run/llm}
GW=${GW:-http://127.0.0.1:18090}
# 管理端口(同一份 nginx / 同一块共享内存)——用量与模型清单从这边读
API=${API:-http://127.0.0.1:18091}
UP_LOG=${UPSTREAM_LOG:-$RUN/upstream.log}
PY=${PY:-python3}
# fixtures 里的固定 Key(见 fixtures/users.json)
ALICE_KEY=sk_alice_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
BOB_KEY=sk_bob_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
# 多账号路由验证用 admin:白名单为空=不限、无配额限制、且本套件中途会删除 bob 故不复用
ADMIN_KEY=sk_admin_0000000000000000000000000000000000000000000000ab
pass=0; fail=0
ck() {
if [ "$2" = "$3" ]; then
printf " \033[32m✓\033[0m %s\n" "$1"; pass=$((pass+1))
else
printf " \033[31m✗\033[0m %s 期望=[%s] 实际=[%s]\n" "$1" "$2" "$3"; fail=$((fail+1))
fi
}
code() { curl -s -o /dev/null -w '%{http_code}' "$@"; }
jbody() { local out='{' sep=''; while [ $# -gt 1 ]; do out="$out$sep\"$1\":\"$2\""; sep=','; shift 2; done; printf '%s}' "$out"; }
# 靶机日志:取最后一条记录的某个字段
up_field() {
"$PY" -c "
import json, sys
last = None
try:
for l in open(sys.argv[1], encoding='utf-8'):
l = l.strip()
if l: last = l
except FileNotFoundError:
pass
print(json.loads(last).get(sys.argv[2], '') if last else '')
" "$UP_LOG" "$1" 2>/dev/null
}
up_count() { if [ -f "$UP_LOG" ]; then awk 'NF{n++} END{print n+0}' "$UP_LOG"; else echo 0; fi }
# 靶机日志:取最后一条记录里**某个请求头**的值(头名大小写不敏感)。
# 专门为「出站头策略」服务:要证明某个头没被发出去 / 某个头还活着,
# 只能看上游实际收到了什么。取不到的空值 = 该头确实没发。
up_hdr() {
"$PY" -c "
import json, sys
last = None
try:
for l in open(sys.argv[1], encoding='utf-8'):
l = l.strip()
if l: last = l
except FileNotFoundError:
pass
h = (json.loads(last).get('headers') or {}) if last else {}
print(h.get(sys.argv[2].lower(), ''))
" "$UP_LOG" "$1" 2>/dev/null
}
post() { curl -s -H "Authorization: Bearer $1" -X POST "$GW$2" -H 'Content-Type: application/json' -d "$3"; }
# 从 stdin 的 JSON 里按点分路径取值
jqv() {
"$PY" -c "
import sys, json
try: d = json.load(sys.stdin)
except Exception: print(''); sys.exit()
for k in sys.argv[1].split('.'):
if d is None: break
d = d[int(k)] if k.isdigit() and isinstance(d, list) else (d.get(k) if isinstance(d, dict) else None)
print('' if d is None else d)
" "$1" 2>/dev/null
}
# 在 stdin 里找子串(命中输出 yes,否则 no)
has() { if grep -q "$1"; then echo yes; else echo no; fi; }
# 反向断言:**不**出现该子串才算过
nhas() { if grep -q "$1"; then echo no; else echo yes; fi; }
echo "【1】鉴权(OpenAI 兼容错误体)"
BODY=$(jbody model gpt-4o)
ck "无 Key → 401" 401 "$(code -X POST $GW/openai/v1/chat/completions)"
ck "错误 Key → 401" 401 "$(code -X POST $GW/openai/v1/chat/completions -H 'Authorization: Bearer sk_deadbeef')"
ck "错误体是 OpenAI 形状(含 error.type)" yes \
"$(curl -s -X POST $GW/openai/v1/chat/completions | grep -q '"error"' && echo yes || echo no)"
ck "路径缺 slot → 400" 400 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/)"
ck "未知上游 → 404" 404 "$(code -H "Authorization: Bearer $BOB_KEY" -X POST $GW/nope/v1/chat/completions)"
echo
echo "【2】上游解析与白名单"
ck "alice + openai(在白名单)→ 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "响应体来自上游(含 pong)" yes "$(post "$ALICE_KEY" /openai/v1/chat/completions "$BODY" | grep -q pong && echo yes || echo no)"
ck "alice + other(不在白名单)→ 403" 403 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/other/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "alice + downstream(已停用)→ 404" 404 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/downstream/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "bob + other(空白名单=不限)→ 200" 200 "$(code -H "Authorization: Bearer $BOB_KEY" -X POST $GW/other/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
echo
echo "【3】★ Key 替换 / 路径 / 头(读靶机实际收到的请求)"
N0=$(up_count)
post "$ALICE_KEY" "/openai/v1/chat/completions?a=1" "$BODY" >/dev/null
ck "靶机恰好收到 1 次转发" 1 "$(( $(up_count) - N0 ))"
ck "★Authorization 已换成上游真 Key" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
ck "★下游 Key 未泄漏给上游" yes "$(up_field auth | grep -q 'sk_alice_' && echo leaked || echo yes)"
ck "Host 头 = 上游主机(非下游域名)" "127.0.0.1:18999" "$(up_field host)"
ck "★路径去 slot 且保留 query" "/v1/chat/completions?a=1" "$(up_field path)"
ck "请求体原样透传" "$BODY" "$(up_field body)"
# ★ 出站头策略(对齐 LiteLLM 的两层:默认不放行 + 凭据头显式抹空)。
# 构造最坏情形:客户端把**网关自己的 Key** 塞进每一个凭据头,
# 同时带上两套协议的扩展头,然后看上游究竟收到了什么。
# 注意这里**故意不发 Authorization**,顺带验证「入站只靠 x-api-key 也能过网关」。
curl -s -o /dev/null -X POST "$GW/openai/v1/chat/completions" \
-H "x-api-key: $ALICE_KEY" \
-H "api-key: $ALICE_KEY" \
-H "x-goog-api-key: $ALICE_KEY" \
-H 'anthropic-version: 2023-06-01' \
-H 'anthropic-beta: prompt-caching-2024-07-31' \
-H 'openai-beta: assistants=v2' \
-H 'Content-Type: application/json' -d "$BODY"
ck "★x-api-key 未泄漏给上游(默认摘除清单)" "" "$(up_hdr x-api-key)"
ck "★api-key 未泄漏给上游" "" "$(up_hdr api-key)"
ck "★x-goog-api-key 未泄漏给上游" "" "$(up_hdr x-goog-api-key)"
ck "★Authorization 仍是上游真 Key(客户端塞不进凭据)" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
ck "★Content-Type 已注入(漏了所有 POST 都会废)" "application/json" "$(up_hdr content-type)"
ck " Accept 已注入(SSE 协商要靠它)" "*/*" "$(up_hdr accept)"
ck "★anthropic-version 已注入(Anthropic 协议必填)" "2023-06-01" "$(up_hdr anthropic-version)"
ck "★anthropic-beta 已注入(Claude Code 扩展依赖)" "prompt-caching-2024-07-31" "$(up_hdr anthropic-beta)"
ck " openai-beta 已注入" "assistants=v2" "$(up_hdr openai-beta)"
ck " User-Agent 已透传(厂商侧排障要靠它)" yes \
"$([ -n "$(up_hdr user-agent)" ] && echo yes || echo no)"
echo
echo "【4】上游 base 带路径时的拼接"
post "$BOB_KEY" /other/v1/chat/completions "$BODY" >/dev/null
ck "★base=/llm + 客户端带 /v1 → 剥掉(base 自带路径)" "/llm/chat/completions" "$(up_field path)"
ck "用的是该上游自己的 Key" "Bearer sk-OTHER-UPSTREAM" "$(up_field auth)"
# ★ 版本段去重(实测踩过的坑:厂商 base_url 自带 /v1,客户端也带 /v1 → /v1/v1 → 上游 404 空响应)
post "$BOB_KEY" /deepseeklike/v1/chat/completions "$BODY" >/dev/null
ck "★base 自带 /v1 + 客户端带 /v1 → 只保留一份" "/v1/chat/completions" "$(up_field path)"
post "$BOB_KEY" /deepseeklike/chat/completions "$BODY" >/dev/null
ck " 客户端不带 /v1 → 照常前缀上 base 的 /v1" "/v1/chat/completions" "$(up_field path)"
post "$BOB_KEY" /deepseeklike/v2/chat/completions "$BODY" >/dev/null
ck " 不同版本段不误判为重复" "/v1/v2/chat/completions" "$(up_field path)"
# ★ 「统一注册」= base 只写到 host,一条记录同时服务两套协议。
# 这是 LiteLLM(/anthropic/* 前缀)/ one-api(根地址 + 路径天然区分)/
# Higress(按路径后缀判协议)三个开源实现的共同做法:
# **协议由客户端路径表达,网关不做任何推断、不需要分发代码**。
# 夹具里 openai 的 base = http://127.0.0.1:18999(纯 host),正好验这条。
post "$ALICE_KEY" "/openai/anthropic/v1/messages" "$BODY" >/dev/null
ck "★Anthropic 路径原样拼在 host 粒度 base 之后(裸 host 客户端 /v1 保留)" "/anthropic/v1/messages" "$(up_field path)"
ck " 同一条记录也换上游真 Key(不是只有 OpenAI 路径才换)" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
echo
echo "【5】★ 流式(SSE 不被缓冲)"
SSE=$(GW="$GW" KEY="$ALICE_KEY" "$PY" "$HERE/sse_test.py" 2>&1)
ck "Content-Type 是 text/event-stream" "text/event-stream" "$(echo "$SSE" | sed -n 's/^CT=//p')"
ck "收到 3 个 data 块" 3 "$(echo "$SSE" | sed -n 's/^CHUNKS=//p')"
FIRST=$(echo "$SSE" | sed -n 's/^FIRST=//p')
TOTAL=$(echo "$SSE" | sed -n 's/^TOTAL=//p')
ck "★首块 < 0.5s 到达(未被攒包)" yes "$(awk -v f="${FIRST:-9}" 'BEGIN{print (f<0.5)?"yes":"no"}')"
ck "总耗时 ≥ 1.0s(确实分了 3 次推)" yes "$(awk -v t="${TOTAL:-0}" 'BEGIN{print (t>=1.0)?"yes":"no"}')"
echo
echo "【6】★ 限流与每日配额(carol rpm=2、dave daily=2,见 fixtures/users.json)"
CAROL_KEY=sk_carol_cccccccccccccccccccccccccccccccccccccccccccccc
DAVE_KEY=sk_dave_dddddddddddddddddddddddddddddddddddddddddddddddd
# 探测闭环组专用(合成数据)。probe1 的 FNV64(key)%2=0 → 冷分配必落 probe 槽 #0;
# probe2/3 只在探测分支(先于哈希)被使用,落点不依赖哈希。
PROBE1_KEY=sk_probe_one_00000000000000000000000000000000
PROBE2_KEY=sk_probe_two_00000000000000000000000000000001
PROBE3_KEY=sk_probe_three_0000000000000000000000000000000
cpost() { code -H "Authorization: Bearer $1" -X POST "$GW/other/v1/chat/completions" -H 'Content-Type: application/json' -d "$BODY"; }
ck "carol 第 1 发 → 200" 200 "$(cpost "$CAROL_KEY")"
ck "carol 第 2 发 → 200" 200 "$(cpost "$CAROL_KEY")"
ck "carol 第 3 发 → 429(rpm=2)" 429 "$(cpost "$CAROL_KEY")"
ck "429 是 rate_limit_error / rate_limit_exceeded" yes \
"$(post "$CAROL_KEY" /other/v1/chat/completions "$BODY" | grep -q 'rate_limit' && echo yes || echo no)"
ck "dave 第 1 发 → 200" 200 "$(cpost "$DAVE_KEY")"
ck "dave 第 2 发 → 200" 200 "$(cpost "$DAVE_KEY")"
ck "dave 第 3 发 → 429(daily=2)" 429 "$(cpost "$DAVE_KEY")"
ck "429 是 insufficient_quota" yes \
"$(post "$DAVE_KEY" /other/v1/chat/completions "$BODY" | grep -q 'insufficient_quota' && echo yes || echo no)"
ck "carol 被限后 alice 不受影响(按 Key 隔离)→ 200" 200 \
"$(code -H "Authorization: Bearer $ALICE_KEY" -X POST "$GW/openai/v1/chat/completions" -H 'Content-Type: application/json' -d "$BODY")"
if [ "${HAVE_TLS:-0}" = "1" ]; then
echo
echo "【7】★ HTTPS(18443 自签,模板 tests/llm/tls.conf.in)"
HTTPS_BASE=https://127.0.0.1:18443
ck "https + 错误 Key → 401(TLS 层下鉴权仍生效)" 401 \
"$(code -k -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Authorization: Bearer nope')"
ck "https + alice → 200" 200 \
"$(code -k -H "Authorization: Bearer $ALICE_KEY" -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "https 响应体来自上游" yes \
"$(curl -sk -H "Authorization: Bearer $ALICE_KEY" -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY" | grep -q pong && echo yes || echo no)"
else
echo
echo "【7】HTTPS:跳过(本机 nginx 无 http_ssl_module 或无 openssl)"
fi
echo
echo "【8】★ 模型发现(网关本地应答,不去上游)"
M0=$(up_count)
ck "无 Key → 401" 401 "$(code $GW/v1/models)"
ck "alice GET /v1/models → 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models)"
ck " 形状是 OpenAI 的 object=list" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | has '"object":"list"')"
ck " 含白名单内上游的模型 gpt-4o" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | has 'gpt-4o')"
ck " 不含白名单外上游的模型 claude-3-5-sonnet" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | nhas 'claude-3-5-sonnet')"
ck " 不含已停用上游的模型 should-not-appear" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | nhas 'should-not-appear')"
ck "bob(白名单为空=不限)能看到 other 的模型" yes \
"$(curl -s -H "Authorization: Bearer $BOB_KEY" $GW/v1/models | has 'claude-3-5-sonnet')"
ck "GET /models 同样本地应答 → 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/models)"
ck "★整个模型发现过程 0 次上游调用" 0 "$(( $(up_count) - M0 ))"
# 带 slot 的 /v1/models 是**另一条路**:照常透传给厂商,取"那家厂商的"模型清单
N0=$(up_count)
ck "★/<slot>/v1/models 仍透传上游 → 200" 200 \
"$(code -H "Authorization: Bearer $ALICE_KEY" $GW/openai/v1/models)"
ck " 上游确实收到了 1 次请求" 1 "$(( $(up_count) - N0 ))"
ck " 转发路径未剥错(/v1/models)" "/v1/models" "$(up_field path)"
ck " 响应体来自上游(不是网关本地那份)" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/openai/v1/models | has 'upstream-model')"
echo
echo "【9】★ 账号态 API:可用模型 + 自己的用量"
# base_url 跟随**调用方所用的端口**(88 部署必须带 :88;443 反代后面则不带)——
# 见 rust/src/upstreams.rs 的 base_url 注释,写死任何一种都会在另一种部署里给出错地址。
API_PORT="${API##*:}"
ck "无 Key 查模型 → 401" 401 "$(code $API/api/v1/models)"
ck "无 Key 查用量 → 401" 401 "$(code $API/api/v1/usage)"
ck "无 Key 查 me → 401" 401 "$(code $API/api/v1/me)"
ck "alice 查 me → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/me)"
# ★ /me 必须带 recent 块(1.0.29 同事反馈「一次拿全我最近调用」)
# - calls_7d ≥ 1(alice 前面已调过 openai / other,应有 audit meta.jsonl)
# - by_slot 数组非空(含调过的上游 slot)
# - hour_24h 是 24 长度的数组
ME_DATA=$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/me)
ck "★me.recent.calls_7d ≥ 1" yes \
"$("$PY" -c "
import json, sys
d = json.loads(sys.argv[1])
r = d.get('data', {}).get('recent', {})
print('yes' if r.get('calls_7d', 0) >= 1 else 'no')
" "$ME_DATA")"
ck "★me.recent.by_slot 非空" yes \
"$("$PY" -c "
import json, sys
d = json.loads(sys.argv[1])
r = d.get('data', {}).get('recent', {})
print('yes' if len(r.get('by_slot', [])) >= 1 else 'no')
" "$ME_DATA")"
ck "★me.recent.hour_24h 是 24 长度数组" yes \
"$("$PY" -c "
import json, sys
d = json.loads(sys.argv[1])
r = d.get('data', {}).get('recent', {})
h = r.get('hour_24h', [])
print('yes' if isinstance(h, list) and len(h) == 24 else 'no')
" "$ME_DATA")"
ck "★me.recent.by_slot 字段完整" yes \
"$("$PY" -c "
import json, sys
d = json.loads(sys.argv[1])
r = d.get('data', {}).get('recent', {})
slots = r.get('by_slot', [])
ok = bool(slots) and all(
set(['slot','calls_7d','calls_24h']).issubset(s.keys()) for s in slots
)
print('yes' if ok else 'no')
" "$ME_DATA")"
ck "alice 查模型 → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/models)"
ck " 模型清单里带 base_url(lm 域名 + 端口 + slot)" yes \
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has "lm.mayishangshu.cn:$API_PORT/openai")"
# ★ base_url / anthropic_base_url 是**纯网关出口地址**(只含 slot,不带厂商路径、不带 /v1):
# 厂商版本/协议路径段(/v1 / /compatible-mode/v1 / /api/coding/paas/v4 / /anthropic …)
# 原样存在上游的 base / base_anthropic 里,转发时由 join_target 拼到 SDK 余下路径前。
# 这样 OpenAI SDK 填 `…/openai`、Anthropic SDK 填 `…/openai/anthropic` 即可,网关不猜路径。
# (校验完整字段值而非整份响应子串——anthropic_base_url 同段会出现,避免误伤。)
ck "★base_url 是纯 slot(无 /v1、不带上游 base 路径)" yes \
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has "\"base_url\":\"https://lm.mayishangshu.cn:$API_PORT/openai\"")"
ck "★base_url 不含 /v1(版本段交给上游 base)" no \
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has "\"base_url\":\"https://lm.mayishangshu.cn:$API_PORT/openai/v1\"")"
ck "★base_url 纯 slot:other 也不带 /llm(路径在上游 base 里)" yes \
"$(curl -s -H "X-API-Key: $BOB_KEY" $API/api/v1/models | has "lm.mayishangshu.cn:$API_PORT/other\"")"
ck "★anthropic_base_url 是 /<slot>/anthropic(Anthropic SDK 内部拼 /v1/messages)" yes \
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has "anthropic_base_url\":\"https://lm.mayishangshu.cn:$API_PORT/openai/anthropic\"")"
# ★ 端口自适应:Host 不带端口(走 443 反代)时 base_url 也不该带端口。
# 写死 88 → 443 部署下客户端连到另一个端口的服务;裸域名 → 88 部署下被出口设备抢答
# (实测 HTTP 200 + 一个 HTML 登录页,是最难看出问题的一种失败)。
ck "★Host 无端口 → base_url 不带端口" yes \
"$(curl -s -H "X-API-Key: $ALICE_KEY" -H 'Host: lm.example' $API/api/v1/models | has 'lm.mayishangshu.cn/openai')"
ck "★Host 带端口 → base_url 带同一端口" yes \
"$(curl -s -H "X-API-Key: $ALICE_KEY" -H "Host: lm.example:$API_PORT" $API/api/v1/models | has "lm.mayishangshu.cn:$API_PORT/openai")"
echo
echo "【9.5】★ 双协议地址:OpenAI 与 Anthropic 路径各走各的(智谱/阿里形态)"
# glm 上游 OpenAI 路径 /api/coding/paas/v4、Anthropic 路径 /api/anthropic,完全不同。
# 网关按「路径里有没有 /anthropic 段」分流到 base / base_anthropic,其余路径原样拼。
# 真实客户端统一带 /v1(见 /openai/v1/chat/completions 约定),所以这里也发 /v1,
# 验证 join_target 会剥掉客户端多余的 /v1(否则 glm 会被拼成 …/v4/v1 被智谱 404)。
# - `/glm/v1/chat/completions` → 上游 /api/coding/paas/v4/chat/completions
# - `/glm/anthropic/v1/messages` → 上游 /api/anthropic/v1/messages
# 用 admin key(绕过白名单)直打,up_field path 读靶机实际收到的路径。
GLM_BODY='{"model":"glm-4","max_tokens":1,"messages":[{"role":"user","content":"hi"}]}'
ck "★glm OpenAI 协议(带 /v1)→ 200" 200 \
"$(code -X POST -H "X-API-Key: $ADMIN_KEY" -H 'Content-Type: application/json' \
-d "$GLM_BODY" $GW/glm/v1/chat/completions)"
ck " 靶机路径 = base + /chat/completions(客户端 /v1 已剥)" \
"/api/coding/paas/v4/chat/completions" "$(up_field path)"
ck "★glm Anthropic 协议 → 200" 200 \
"$(code -X POST -H "X-API-Key: $ADMIN_KEY" -H 'Content-Type: application/json' \
-d "$GLM_BODY" $GW/glm/anthropic/v1/messages)"
ck " 靶机路径 = base_anthropic + /v1/messages(/anthropic 段已剥)" \
"/api/anthropic/v1/messages" "$(up_field path)"
ALICE_USAGE=$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/usage)
ck "alice 查用量 → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/usage)"
ALICE_TOTAL=$(echo "$ALICE_USAGE" | jqv data.total)
ck "★alice 累计用量 > 0(前面真发过请求)" yes \
"$(awk -v t="${ALICE_TOTAL:-0}" 'BEGIN{print (t>0)?"yes":"no"}')"
ck " 今日 = 累计(都是刚发的)" "$ALICE_TOTAL" "$(echo "$ALICE_USAGE" | jqv data.today)"
ck " 分上游明细含 openai" yes "$(echo "$ALICE_USAGE" | has '"slot": *"openai"')"
ck "★被拒的越权/未知上游不进明细(无 other/downstream)" yes \
"$(echo "$ALICE_USAGE" | nhas '"slot": *"other"')"
ck " 回显限额供客户端自我节流" yes "$(echo "$ALICE_USAGE" | has '"limits"')"
# ★ 关键语义:统计的是"真的转发出去"的请求 —— 超限被 429 的那一发不计
CAROL_TOTAL=$(curl -s -H "X-API-Key: $CAROL_KEY" $API/api/v1/usage | jqv data.total)
ck "★carol 发了 3 次、第 3 次被限流 → 用量只记 2" 2 "${CAROL_TOTAL:-x}"
DAVE_TOTAL=$(curl -s -H "X-API-Key: $DAVE_KEY" $API/api/v1/usage | jqv data.total)
ck "★dave 同上(每日配额)→ 用量只记 2" 2 "${DAVE_TOTAL:-x}"
ck "bob 的明细含 other(他真调过)" yes \
"$(curl -s -H "X-API-Key: $BOB_KEY" $API/api/v1/usage | has '"slot": *"other"')"
echo
echo "【10】管理员用量总览 + 删账号清理用量键"
ck "无会话 → 401" 401 "$(code $API/api/admin/usage)"
ck "不是接口路径 → 404" 404 "$(code $API/api/v1/nope)"
JAR=$RUN/cookie.jar
LOGIN=$(curl -s -c "$JAR" -X POST $API/api/admin/login -H 'Content-Type: application/json' \
-d "$(jbody username admin password 123456)")
ck "管理员登录 → 200" 200 "$(echo "$LOGIN" | jqv code)"
USAGE=$(curl -s -b "$JAR" $API/api/admin/usage)
ck "用量总览含 alice" yes "$(echo "$USAGE" | has '"username": *"alice"')"
# ★ 上游视图返回 base_anthropic:双协议地址各厂商路径不同,必须随视图给出
# (智谱 /api/coding/paas/v4 ↔ /api/anthropic、阿里 /compatible-mode/v1 ↔ /apps/anthropic)。
ck "★上游视图返回 base_anthropic 字段" yes \
"$(curl -s -b "$JAR" $API/api/admin/upstreams | has '"base_anthropic"')"
ck " 总览含按上游分组(openai)" yes "$(echo "$USAGE" | has '"slot": *"openai"')"
ck " 全局合计 > 0" yes "$(echo "$USAGE" | jqv data.total.total | awk '{print ($1>0)?"yes":"no"}')"
ck "★删除前用量榜里有 bob(他真调过 other)" yes "$(echo "$USAGE" | has '"username": *"bob"')"
# ★ 删除账号必须**连带清掉用量键**(`ust:` / `ums:`)——它们是按前缀扫出来出报表的,
# 不清就会让已删账号一直挂在用量榜上(与"删账号必须摘 ukey:"同类的正确性问题)。
curl -s -b "$JAR" -X POST $API/api/admin/users/delete -H 'Content-Type: application/json' \
-d "$(jbody username bob)" >/dev/null
ck "★bob 已被删除(他的 Key 立即失效 → 401)" 401 "$(code -H "X-API-Key: $BOB_KEY" $API/api/v1/usage)"
USAGE2=$(curl -s -b "$JAR" $API/api/admin/usage)
ck "★删除后用量榜不再出现 bob" yes "$(echo "$USAGE2" | nhas 'bob')"
ck " 但 alice 仍在榜单上" yes "$(echo "$USAGE2" | has '"username": *"alice"')"
echo
echo "【11】★ 上游连通性测试(管理台「测试」按钮的后端链路)"
# 管理台把探测体原样发上来,网关只负责「定位上游 + 换真 Key + 转发 + 透传状态」。
# 判据照旧:不只看 HTTP 200,还要看靶机实际收到的 path / auth。
PING='{"model":"ds-chat","max_tokens":1,"stream":false,"messages":[{"role":"user","content":"ping"}]}'
ck "无会话测试 → 401" 401 \
"$(code -X POST "$API/api/admin/upstreams/test?slot=openai" -d '{}')"
ck "★admin 测 openai → 透传上游 200" 200 \
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=openai" -H 'Content-Type: application/json' -d "$PING")"
ck " 靶机收到的 path 不含 slot 段" "/v1/chat/completions" "$(up_field path)"
ck " 换上了 openai 上游的真 Key" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
ck " ★探测体原样到达靶机(model=ds-chat,未被网关注入体覆盖)" "yes" \
"$(up_field body | grep -q 'ds-chat' && echo yes || echo no)"
ck "★base 自带 /v1 的上游(deepseeklike)→ 去重后 200" 200 \
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=deepseeklike" -H 'Content-Type: application/json' -d "$PING")"
ck " 靶机 path 仍是 /v1/chat/completions(无 /v1/v1、无 slot 段)" "/v1/chat/completions" "$(up_field path)"
ck " 换上了 deepseeklike 的真 Key" "Bearer sk-DS-REAL" "$(up_field auth)"
ck "★已停用的上游测不了(shm 已摘除 → 404)" 404 \
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=downstream" -d '{}')"
ck "不存在的 slot → 404 信封" 404 \
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=ghost" -d '{}')"
ck "★内部 location 外部直连 → 404(不能绕过 admin 会话触达上游)" 404 \
"$(code -b "$JAR" -X POST "$API/_upstream_test/openai/v1/chat/completions" -d '{}')"
# —— 静态回归闸 ——
# 生产 conf 与测试模板是两份拷贝,历史事故(面板测 minimax 报
# `unknown model 'test' (2013)`)正是二者漂移:1.0.43 为当时的「空 body 探测」
# 兜底,在 conf/nginx.conf 的 /_upstream_test/ 注入了 proxy_set_body dummy 体,
# 而 nginx 的 proxy_set_body 是**无条件覆盖**——把面板发来的「第一个登记模型」
# 探测体整个换掉,严格校验模型名的厂商全部误报「不通」,本夹具却因模板没这行
# 而全绿。探测体必须原样转发,这道闸防注入行只在线上配置里复活。
# 只看生效行:conf 注释里「别加 proxy_set_body」的告诫会字面命中 grep
UB_BLOCK=$(awk '/location \/_upstream_test\//{f=1} f{print} f&&/^ \}$/{exit}' "$HERE/../../conf/nginx.conf" | grep -v '^[[:space:]]*#')
ck "★生产 conf 探测块无 proxy_set_body(探测体原样转发)" "no" \
"$(echo "$UB_BLOCK" | grep -q proxy_set_body && echo yes || echo no)"
echo
echo "【11.5】★ API 代理(/apix/<slug> → 外部供应商)"
# 与 LLM 代理同一把 sk_ Key,但上游形态不同:供应商只有一个收口,资产由
# **请求体里的 asset_id** 决定,所以请求体必须由网关拼——判据是看靶机
# 收到的 path / 头 / body 三件事,只看网关自己的 200 什么都证明不了。
ck "无 Key → 401" 401 "$(code "$GW/apix/search?q=hello")"
ck "坏 Key → 401" 401 "$(code -H 'X-API-Key: sk_nope' "$GW/apix/search?q=hello")"
ck "不存在的 slug → 404" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/ghost?q=1")"
ck "★已停用的 slug → 404(shm 里已摘除)" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/offline?q=1")"
ck "★正常调用 → 透传上游 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/search?q=hello")"
ck " 靶机 path = 供应商的收口" "/api/v1/ai/request" "$(up_field path)"
ck " ★GET 被改写成 POST(proxy_method + proxy_set_body)" "POST" "$(up_field method)"
ck " ★换上了供应商真 Key" "ak_test_supplier_key" "$(up_hdr x-api-key)"
ck " ★调用体由网关拼:asset_id 来自配置" yes \
"$(up_field body | grep -q '"asset_id":"searxng-search"' && echo yes || echo no)"
ck " ★query 参数进 call_data" yes \
"$(up_field body | grep -q '"q":"hello"' && echo yes || echo no)"
ck " ★下行 Key 没被带出去" yes "$(up_field body | nhas 'sk_alice')"
ck " 下游 Authorization 未透传" "" "$(up_hdr authorization)"
# x-api-key 头**会**有值——那正是换上的供应商真 Key。要证的是它不再以下游的
# sk_ 开头:客户端用自己的 Key 换来供应商凭据,而不是凭据被原样透传。
AXH=$(up_hdr x-api-key)
ck " ★下游 x-api-key 已被换掉(不再是 sk_ 开头)" no \
"$(printf '%s' "$AXH" | grep -q '^sk_' && echo yes || echo no)"
# 管理台:清单 / 凭据掩码 / 停启用写穿
ck "管理台列表含 search" yes "$(curl -s -b "$JAR" $API/api/admin/apix | has '"slug": *"search"')"
ck " ★条目 Key 只回显掩码(不回显真值)" no \
"$(curl -s -b "$JAR" $API/api/admin/apix | has 'ak_test_supplier_key')"
ck " 条目 base 回显" "http://127.0.0.1:18999" \
"$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c '
import sys, json
d = json.load(sys.stdin)["data"]
it = next((x for x in d["items"] if x["slug"] == "search"), None)
print(it.get("base") if it else "miss")')"
ck " 供应商名回显" "vendor-a" \
"$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c '
import sys, json
d = json.load(sys.stdin)["data"]
it = next((x for x in d["items"] if x["slug"] == "search"), None)
print(it.get("supplier") if it else "miss")')"
ck " 已停用条目 enabled=false" "False" "$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c '
import sys, json
d = json.load(sys.stdin)["data"]
it = next((x for x in d["items"] if x["slug"] == "offline"), None)
print(str(it.get("enabled")) if it else "miss")')"
ck "账号态接口只列已启用的(offline 不出现)" "3" \
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/apix | jqv data.count)"
ck "非法 slug → 400" 400 "$(code -b "$JAR" -X POST $API/api/admin/apix/save \
-H 'Content-Type: application/json' -d "$(jbody slug 'Bad Slug!' asset_id x)")"
ck "缺 asset_id → 400" 400 "$(code -b "$JAR" -X POST $API/api/admin/apix/save \
-H 'Content-Type: application/json' -d "$(jbody slug med asset_id '')")"
ck "★vendor 新建缺 Key → 400(vendor 必须 Key)" 400 \
"$(code -b "$JAR" -X POST $API/api/admin/apix/save \
-H 'Content-Type: application/json' -d "$(jbody slug vendorbare asset_id x base http://127.0.0.1:18999)")"
ck "★passthrough 新建可空 Key → 200(1.0.20 现场 bug 防回归)" 200 \
"$(code -b "$JAR" -X POST $API/api/admin/apix/save \
-H 'Content-Type: application/json' -d "$(jbody slug pt_nokey mode passthrough base http://127.0.0.1:18999)")"
ck "新建代理 med → 200" 200 "$(code -b "$JAR" -X POST $API/api/admin/apix/save \
-H 'Content-Type: application/json' -d "$(jbody slug med asset_id asset_26476ef1-4687-45 name 疾病百科 params kw base http://127.0.0.1:18999 key ak_test_supplier_key supplier vendor-a)")"
ck " ★新代理立即可用(写穿 shm)" 200 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/med?kw=感冒")"
ck " 靶机 body 带新 asset_id 与新参数" yes \
"$(up_field body | grep -q '"asset_id":"asset_26476ef1-4687-45"' && echo yes || echo no)"
ck " 参数名用的是 med 声明的 kw" yes "$(up_field body | grep -q '"kw":"' && echo yes || echo no)"
curl -s -b "$JAR" -X POST $API/api/admin/apix/save -H 'Content-Type: application/json' \
-d "$(jbody slug search asset_id searxng-search enabled false)" >/dev/null
ck "★停用后立即 404(写穿 shm,无回源兜底)" 404 \
"$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/search?q=1")"
curl -s -b "$JAR" -X POST $API/api/admin/apix/save -H 'Content-Type: application/json' \
-d "$(jbody slug search asset_id searxng-search enabled true)" >/dev/null
ck " 重新启用后又能调通" 200 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/search?q=1")"
curl -s -b "$JAR" -X POST $API/api/admin/apix/delete -H 'Content-Type: application/json' \
-d "$(jbody slug med)" >/dev/null
ck "删除后再调 → 404" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/med?kw=x")"
# 探测:走供应商的**免费**预估接口,不执行资产、不扣费
ck "★探测(免费预估)→ 透传 200" 200 "$(code -b "$JAR" -X POST \
"$API/api/admin/apix/test?slug=search" -H 'Content-Type: application/json' -d '{}')"
ck " 探测打的是 /api/v1/ai/estimate" "/api/v1/ai/estimate" "$(up_field path)"
ck " 探测也换了供应商真 Key" "ak_test_supplier_key" "$(up_hdr x-api-key)"
ck "★内部探测口外部直连 → 404" 404 \
"$(code -b "$JAR" -X POST "$API/_apix_probe/search" -d '{}')"
ck " 不存在的 slug 探测 → 404" 404 "$(code -b "$JAR" -X POST \
"$API/api/admin/apix/test?slug=ghost" -H 'Content-Type: application/json' -d '{}')"
ck "★用量统计按 api/<slug> 归组(与 LLM 上游天然分开)" yes \
"$(curl -s -b "$JAR" $API/api/admin/usage | has '"slot": *"api/search"')"
ck "★趋势含 api/<slug> 维度" yes "$(curl -s -b "$JAR" "$API/api/admin/trend?days=7" | has '"slot": *"api/search"')"
ck "★usage_hint 写入与回显(1.0.22 能力目录 hint)" yes \
"$(curl -s -b "$JAR" -X POST $API/api/admin/apix/save \
-H 'Content-Type: application/json' \
-d "$(jbody slug hintsrc asset_id asset_hint_26476ef1 base http://127.0.0.1:18999 key ak_test_supplier_key usage_hint 'POST /<tail> multipart fields=[file]; POST /<tail>/base64 json {image_base64:string}; Swagger: http://x/docs')" >/dev/null; \
curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
it = next((x for x in d["data"]["items"] if x["slug"] == "hintsrc"), None)
print("yes" if it and "POST /<tail>" in it.get("usage_hint","") else "no")')"
# 清理
curl -s -b "$JAR" -X POST $API/api/admin/apix/delete -H 'Content-Type: application/json' \
-d "$(jbody slug hintsrc)" >/dev/null
echo
echo "【11.6】★ API 代理透传模式(/apixp/<slug>/<尾部> → 内部 REST 服务)"
# 与 vendor 模式同一把 sk_ Key + 配额 + 用量统计,但**不再由网关拼体**——
# 客户端带什么方法/body/Content-Type 原样给上游。判据同样是看靶机实际收到的
# path / method / 头 / body,只看网关 200 证明不了「原样透传」。
ck "无 Key → 401" 401 "$(code "$GW/apixp/ocr/ocr")"
ck "坏 Key → 401" 401 "$(code -H 'X-API-Key: sk_nope' "$GW/apixp/ocr/ocr")"
ck "不存在的 slug → 404" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apixp/ghost/foo")"
ck "★已停用的 slug → 404" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apixp/offline/foo")"
ck "★passthrough 空 Key 也放行(内部服务免鉴权)" 200 \
"$(code -H "X-API-Key: $ALICE_KEY" -X POST "$GW/apixp/ocr/ocr" -H 'Content-Type: application/json' -d '{"image_base64":"AAAA"}')"
ck " 靶机 path = 路径尾部 /ocr(base + 尾部)" "/ocr" "$(up_field path)"
ck " ★方法原样透传(POST 没被改)" "POST" "$(up_field method)"
ck " ★请求体原样透传" yes "$(up_field body | grep -q 'image_base64' && echo yes || echo no)"
ck " ★空 Key 不下发 X-API-Key(上游收不到该头)" "" "$(up_hdr x-api-key)"
ck " ★下行 sk_ Key 没被带出去" "" "$(up_hdr authorization)"
# 路径尾部可多级:/apixp/ocr/ocr/base64 → 上游 /ocr/base64
ck "★多级尾部透传 → 200" 200 \
"$(code -H "X-API-Key: $ALICE_KEY" -X POST "$GW/apixp/ocr/ocr/base64" -H 'Content-Type: application/json' -d '{"image_base64":"BBBB"}')"
ck " 靶机 path = /ocr/base64" "/ocr/base64" "$(up_field path)"
# query 也随尾部一起透传(变量式 proxy_pass 不自动追加,由模块拼进 $apixp_target)
ck "★query 随尾部透传 → 200" 200 \
"$(code -H "X-API-Key: $ALICE_KEY" -X POST "$GW/apixp/ocr/ocr/url?src=http://example.com/x" -H 'Content-Type: application/json' -d '{"x":1}')"
ck " 靶机 path 含 query" yes "$(up_field path | grep -q '/ocr/url?src=' && echo yes || echo no)"
# 带 Key 的 passthrough:X-API-Key 必须被下发
ck "★带 Key 的 passthrough → 200" 200 \
"$(code -H "X-API-Key: $ALICE_KEY" -X POST "$GW/apixp/ocrk/health" -H 'Content-Type: application/json' -d '{}')"
ck " ★X-API-Key 被下发到上游" "ak_internal_ocr_key" "$(up_hdr x-api-key)"
ck " ★path 尾部 = /health" "/health" "$(up_field path)"
# mode 与路径前缀必须一致:vendor slug 不能走 /apixp/,passthrough slug 不能走 /apix/
ck "★vendor slug 走 /apixp/ → 404(路径与模式不匹配)" 404 \
"$(code -H "X-API-Key: $ALICE_KEY" "$GW/apixp/search?q=1")"
ck "★passthrough slug 走 /apix/ → 404(路径与模式不匹配)" 404 \
"$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/ocr")"
# 管理台:mode 字段回显 + 调用地址用 /apixp/ 前缀
ck "管理台列表含 ocr 且 mode=passthrough" yes "$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c '
import sys, json
d = json.load(sys.stdin)["data"]
it = next((x for x in d["items"] if x["slug"] == "ocr"), None)
print("yes" if it and it.get("mode") == "passthrough" else "no")')"
ck " ★ocr 调用地址用 /apixp/ 前缀" yes "$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c '
import sys, json
d = json.load(sys.stdin)["data"]
it = next((x for x in d["items"] if x["slug"] == "ocr"), None)
print("yes" if it and "/apixp/ocr" in it.get("url","") else "no")')"
# 探测:passthrough 走 GET 打上游根地址验连通性
ck "★探测(passthrough)→ 透传 200" 200 "$(code -b "$JAR" -X POST \
"$API/api/admin/apix/test?slug=ocr" -H 'Content-Type: application/json' -d '{}')"
ck " 探测是 GET(非 POST 预估)" "GET" "$(up_field method)"
ck " 探测打上游根 /" "/" "$(up_field path)"
ck " 探测也空 Key(不下发 X-API-Key)" "" "$(up_hdr x-api-key)"
ck "★内部透传探测口外部直连 → 404" 404 \
"$(code -b "$JAR" -X POST "$API/_apixp_probe/ocr" -d '{}')"
ck " 不存在的 slug 透传探测 → 404" 404 "$(code -b "$JAR" -X POST \
"$API/api/admin/apix/test?slug=ghost" -H 'Content-Type: application/json' -d '{}')"
ck "★用量统计按 api/<slug> 归组(passthrough 同样计)" yes \
"$(curl -s -b "$JAR" $API/api/admin/usage | has '"slot": *"api/ocr"')"
echo
echo "【12】★ 上游健康标记(失效自动缓存 + 模型发现附可用地址/状态)"
# 健康缓存由 log 阶段的**真实转发**写入(uh:<slot>)——「测试」按钮走同一条
# proxy_pass 链路,探测结果也天然更新缓存,无需第二套标记逻辑。
# 语义:上游回了头 = 活着(4xx 是它对请求内容的意见,不是机器死了);
# 5xx / 连不上(502) / 等头超时(504) = 失效;没被任何请求打过 = unknown。
# 缓存只用于展示(列表标红 / 模型发现附状态),转发路径不读它。
up_field_of() { # $1=slot $2=字段 → 取上游列表里该 slot 的字段值
curl -s -b "$JAR" $API/api/admin/upstreams | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
u = next((x for x in items if x.get("slot") == sys.argv[1]), None)
v = u.get(sys.argv[2]) if u else "miss"
print("miss" if v is None else v)
' "$1" "$2"
}
xm_status() { # /v1/models 扩展块里某 slot 的健康状态
curl -s -H "Authorization: Bearer $HKEY" $GW/v1/models | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
u = next((x for x in d.get("x_ipass_upstreams", []) if x.get("slot") == sys.argv[1]), {})
print(u.get("status", "miss"))
' "$1"
}
ck "创建临时上游 health1 → 200" 200 \
"$(code -b "$JAR" -X POST $API/api/admin/upstreams/save -H 'Content-Type: application/json' \
-d '{"slot":"health1","base":"http://127.0.0.1:18999","auth":"Bearer sk-H1","models":["m1"]}')"
ck "★初始状态 = unknown(没被任何真实请求打过)" unknown "$(up_field_of health1 status)"
# 临时账号(空白名单 = 不限)换一把干净 Key 来打靶机注入的 500
curl -s -b "$JAR" -X POST $API/api/admin/users/create -H 'Content-Type: application/json' \
-d "$(jbody username healthprobe password 123456 display_name hp)" >/dev/null
HKEY=$(curl -s -b "$JAR" $API/api/admin/users | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
print(next((u["api_key"] for u in items if u.get("username") == "healthprobe"), ""))
')
BOOM='{"model":"m1","max_tokens":1,"boom":true,"messages":[{"role":"user","content":"x"}]}'
ck "boom 请求 → 靶机注入的 500 原样透传给客户端" 500 \
"$(code -H "Authorization: Bearer $HKEY" -X POST $GW/health1/v1/chat/completions \
-H 'Content-Type: application/json' -d "$BOOM")"
sleep 0.3
ck "★上游 500 → health1 自动标记为 failed(无需人工探测)" failed "$(up_field_of health1 status)"
ck " 缓存里记录的状态码 = 500" 500 "$(up_field_of health1 status_code)"
ck "★/v1/models 顶层附 x_ipass_upstreams(地址 + 模型 + 状态)" yes \
"$(curl -s -H "Authorization: Bearer $HKEY" $GW/v1/models | has '"x_ipass_upstreams"')"
ck " 其中 health1 同样标 failed" failed "$(xm_status health1)"
ck " 给出 anthropic 形式的 base_url" yes \
"$(curl -s -H "Authorization: Bearer $HKEY" $GW/v1/models | has '"anthropic_base_url"')"
ck "★账号态 /api/v1/upstreams 也带状态与双协议地址" yes \
"$(curl -s -H "X-API-Key: $HKEY" $API/api/v1/upstreams | has '"anthropic_base_url"')"
OKBODY='{"model":"m1","max_tokens":1,"messages":[{"role":"user","content":"ping"}]}'
ck "恢复:正常请求 → 200" 200 \
"$(code -H "Authorization: Bearer $HKEY" -X POST $GW/health1/v1/chat/completions \
-H 'Content-Type: application/json' -d "$OKBODY")"
sleep 0.3
ck "★下次成功转发自动转回 ok(自愈,无需清缓存)" ok "$(up_field_of health1 status)"
# 清理临时数据
curl -s -b "$JAR" -X POST $API/api/admin/upstreams/delete -H 'Content-Type: application/json' \
-d "$(jbody slot health1)" >/dev/null
ck "删除后列表不再出现 health1" miss "$(up_field_of health1 status)"
curl -s -b "$JAR" -X POST $API/api/admin/users/delete -H 'Content-Type: application/json' \
-d "$(jbody username healthprobe)" >/dev/null
echo
echo "【13】对话审计(meta.jsonl:账号/上游/状态 + 智能 excerpt + 全文 hash)"
AUDIT=$RUN/audit
sleep 0.3
# 1.0.18 起不再把请求体复制进审计目录(旧版每请求 700 KB+),只写一行 meta。
# 1.0.27 起 excerpt 改为智能摘录:短请求保留原文;长请求解析 JSON 提取
# model + 最后一个 user content(按字符截),跳过 context_management / tools
# 等结构噪声,输出 {"model":"…","content":"…"}。
meta_n() { cat "$AUDIT"/*/meta.jsonl 2>/dev/null | awk 'NF{n++} END{print n+0}'; }
body_files() { find "$AUDIT" -name '*.json' 2>/dev/null | wc -l | tr -d ' '; }
BODY_BASE=$(body_files)
N1=$(meta_n)
ck "转发的请求都落了 meta 行" yes "$([ "$N1" -ge 3 ] && echo yes || echo no)"
ck "meta 行记录账号/上游/状态" yes \
"$(grep -q '"user":"alice"' "$AUDIT"/*/meta.jsonl 2>/dev/null && grep -q '"slot":"openai"' "$AUDIT"/*/meta.jsonl && grep -q '"status":200' "$AUDIT"/*/meta.jsonl && echo yes || echo no)"
# ★ 长请求体:1.0.27 智能摘录优先把 user content 摘出,跳过 context_management / tools。
# 这条断言是 1.0.18 的核心取舍(元数据 + 摘要),现在升级为「摘要含用户内容、
# 不含结构噪声」。
# 测试用 400 字节 + 'AAA…' 占位 + 'ZZ_TAIL_MARK' 尾巴,确保 body_bytes > 100 走智能路径。
PAD=$(printf 'A%.0s' $(seq 1 400))
LBODY="{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":\"${PAD}ZZ_TAIL_MARK\"}]}"
code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
-H 'Content-Type: application/json' -d "$LBODY" >/dev/null
sleep 0.3
# 取这一笔长请求的 excerpt(按 body_bytes > 400 + user=alice + status=200 唯一定位)
EXCERPT=$("$PY" -c "
import json, sys
for f in sys.argv[1:]:
for l in open(f, encoding='utf-8'):
d = json.loads(l)
if d.get('user') == 'alice' and d.get('status') == 200 and d.get('body_bytes', 0) > 400:
print(d.get('excerpt',''))
sys.exit(0)
" "$AUDIT"/*/meta.jsonl 2>/dev/null)
# ★ 智能摘录:长请求 excerpt 是 {"model":"…","content":"<user 内容>"} 结构
# - 必须含 model 字段(保留诊断价值)
# - 必须含 user content 关键片段(跳过结构噪声,摘出真正想问的)
# - 不应含 'ZZ_TAIL_MARK'(尾部内容仍未落盘)
# - 不应含 AAA 占位(这是 user content,智能摘录**全部** user 内容到 content 字段;
# 原 1.0.18 前 100 字节全是结构噪声反而看不到 AAA)
ck "★智能 excerpt 含 model 字段" yes \
"$(printf '%s' "$EXCERPT" | grep -q '"model":"gpt-4o"' && echo yes || echo no)"
ck "★智能 excerpt 含 user content(AAA 占位)" yes \
"$(printf '%s' "$EXCERPT" | grep -q 'AAAA' && echo yes || echo no)"
ck "★智能 excerpt 不含尾部标记(脱敏与不落盘)" yes \
"$(if printf '%s' "$EXCERPT" | grep -q 'ZZ_TAIL_MARK'; then echo no; else echo yes; fi)"
ck "★智能 excerpt 不含结构噪声(context_management / tools)" yes \
"$(if printf '%s' "$EXCERPT" | grep -q 'context_management\|\"tools\"'; then echo no; else echo yes; fi)"
ck "★body_bytes = 请求体真实长度(智能摘录不影响)" "$(printf '%s' "$LBODY" | wc -c | tr -d ' ')" \
"$("$PY" -c "
import json, sys
n = ''
for f in sys.argv[1:]:
for l in open(f, encoding='utf-8'):
d = json.loads(l)
if d.get('user') == 'alice' and d.get('body_bytes', 0) > 400:
n = d['body_bytes']
print(n)
" "$AUDIT"/*/meta.jsonl 2>/dev/null)"
ck "★body_hash 是全文指纹(16 hex,智能摘录不影响)" yes "$("$PY" -c "
import json, re, sys
res = 'no'
for f in sys.argv[1:]:
for l in open(f, encoding='utf-8'):
d = json.loads(l)
if d.get('user') == 'alice' and d.get('body_bytes', 0) > 400:
res = 'yes' if re.fullmatch(r'[0-9a-f]{16}', d.get('body_hash', '')) else 'no'
print(res)
" "$AUDIT"/*/meta.jsonl 2>/dev/null)"
ck "★不再生成请求体副本(本轮新增 *.json = 0)" 0 "$(( $(body_files) - BODY_BASE ))"
ck "★审计目录里搜不到请求体尾部" yes \
"$(if grep -q 'ZZ_TAIL_MARK' "$AUDIT"/*/meta.jsonl 2>/dev/null; then echo no; else echo yes; fi)"
# ★ 短请求体(≤ 100B):excerpt 应保留原文(含 model + messages 结构)
SBODY="{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}"
code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
-H 'Content-Type: application/json' -d "$SBODY" >/dev/null
sleep 0.3
# ⚠️ meta.jsonl 里 excerpt 是 JSON 字符串,其内的 "messages" 被转义成 \"messages\"——
# 直接 grep '"messages"' 匹不到;用不带引号的 'messages' 即可(包含 messages 数组外层
# + content 里的 messages 都不会误匹,下面 JSON 解析做严格断言)。
ck "★短请求(≤100B)excerpt 保留原文(含 messages 结构)" yes \
"$("$PY" -c "
import json, sys
res = 'no'
for f in sys.argv[1:]:
for l in open(f, encoding='utf-8'):
d = json.loads(l)
if d.get('user') != 'alice' or d.get('status') != 200 or d.get('body_bytes', 0) > 100:
continue
# 短请求 excerpt 应该是原文(含 messages 数组);智能摘录只摘 model + content
try:
ex = json.loads(d.get('excerpt',''))
if 'messages' in ex: res = 'yes'
except Exception: pass
print(res)
" "$AUDIT"/*/meta.jsonl 2>/dev/null)"
N2=$(meta_n)
code -H "Authorization: Bearer sk_deadbeef" -X POST $GW/openai/v1/chat/completions \
-H 'Content-Type: application/json' -d "$BODY" >/dev/null
sleep 0.3
ck "被拒请求(401)不入档" 0 "$(( $(meta_n) - N2 ))"
echo "【14】token 计量(body filter 旁路抓 usage → utt:/utms:)"
# 非 stream usage 固定 7/3;带 usage_probe 的流式在 SSE 尾部给 11/22
curl -s -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
-H 'Content-Type: application/json' -d "$BODY" >/dev/null
curl -s -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
-H 'Content-Type: application/json' -d '{"model":"gpt-4o","stream":true,"usage_probe":true}' >/dev/null
sleep 0.3
TOK=$(curl -s -b "$JAR" $API/api/admin/usage)
echo "【多账号路由】粘性绑定 + 额度耗尽自动切换"
# 用 admin(白名单为空=不限;本套件中途会删除 bob,故不复用 bob)
ck "admin 基础鉴权(openai,白名单不限)→ 200" 200 "$(code -H "Authorization: Bearer $ADMIN_KEY" -X POST $GW/openai/v1/chat/completions -H 'Content-Type: application/json' -d '{"model":"gpt-4o","messages":[{"role":"user","content":"hi"}],"max_tokens":8}')"
acct_of() { curl -s -D - -o /dev/null -X POST "$GW/$1/v1/chat/completions" \
-H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \
-d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}' \
| grep -i '^X-Upstream-Acct' | tr -d '\r' | awk '{print $2}'; }
code_of() { curl -s -o /dev/null -w '%{http_code}' -X POST "$GW/$1/v1/chat/completions" \
-H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \
-d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}'; }
ck "sticky 首次请求命中 acct=0" 0 "$(acct_of sticky sticky-model)"
ck "sticky 首次请求 200" 200 "$(code_of sticky sticky-model)"
ck "sticky 粘性:第 2 次仍 acct=0" 0 "$(acct_of sticky sticky-model)"
ck "sticky 粘性:第 3 次仍 acct=0" 0 "$(acct_of sticky sticky-model)"
# 切换:multi slot accounts[0]=bad(命中假上游 UPSTREAM_FAIL_AUTH → 401+marker)→ 自动切 acct=1
mh() { curl -s -D - -o /dev/null -X POST "$GW/$1/v1/chat/completions" \
-H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \
-d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}' \
| grep -iE '^HTTP/|^X-Upstream-Acct' | tr -d '\r'; }
M1=$(mh multi multi-model)
ck "multi 初次请求打到 acct=0(bad)" 0 "$(echo "$M1" | awk '/X-Upstream-Acct/{print $2}')"
ck "multi 初次请求 401(触发冷却)" 401 "$(echo "$M1" | awk '/^HTTP/{print $2}')"
M2=$(mh multi multi-model)
ck "multi 冷却中切到 acct=1" 1 "$(echo "$M2" | awk '/X-Upstream-Acct/{print $2}')"
ck "multi 切到 acct=1 后 200" 200 "$(echo "$M2" | awk '/^HTTP/{print $2}')"
M3=$(mh multi multi-model)
ck "multi 冷却期间保持 acct=1(不回退坏账号)" 1 "$(echo "$M3" | awk '/X-Upstream-Acct/{print $2}')"
# 靶机验证:再发一次确认切换后的账号,读靶机实际收到的 auth(证明下行 Key 真被换成对应账号 Key)
M4=$(mh multi multi-model)
ck "multi 切换后下游 Key 真换成 acct-good" "Bearer acct-good" "$(up_hdr Authorization)"
echo "【探测恢复闭环】退避冷却 + 真实请求探测 + 绑定流量保护"
# fixture `probe`:#0=probe-flip(每账号只失败第一次)、#1=probe-dead(永远耗尽)。
# conf 里 ipass_probe_min 1 → 首轮退避 1s,翻倍 2s;冷却硬顶 30min。
# 剧本逐拍(时间锚 = 各次标记后的 next_probe):
ph() { curl -s -D - -o /dev/null -X POST "$GW/probe/v1/chat/completions" \
-H "Authorization: Bearer $1" -H 'Content-Type: application/json' \
-d '{"model":"probe-model","messages":[{"role":"user","content":"hi"}],"max_tokens":8}' \
| grep -iE '^HTTP/|^X-Upstream-Acct' | tr -d '\r'; }
p_code() { echo "$1" | awk '/^HTTP/{print $2}'; }
p_acct() { echo "$1" | awk '/X-Upstream-Acct/{print $2}'; }
P1=$(ph "$PROBE1_KEY")
ck "probe 冷分配落 #0(FNV 钉死)且首触 401" 0 "$(p_acct "$P1")"
ck "probe #0 首触 401 → 进冷却" 401 "$(p_code "$P1")"
P2=$(ph "$PROBE1_KEY")
ck "probe 绑定 #0 冷却中 → 退到 #1(也 401,被标记)" 401 "$(p_code "$P2")"
ck "probe 第二落点 #1" 1 "$(p_acct "$P2")"
sleep 2.0 # 首轮退避 1s 到期(#0/#1 标记时刻差 <0.5s,2s 留足余量;#0 的 next_probe 更早)
P3=$(ph "$PROBE1_KEY")
ck "★probe 全冷却下最早到期的 #0 被真实请求探测 → 200 回队" 200 "$(p_code "$P3")"
ck "★probe 探测落在 #0" 0 "$(p_acct "$P3")"
P4=$(ph "$PROBE1_KEY")
ck "★probe 绑定已复活的 #0 → 不被 #1 探测劫持(绑定流量保护)" 0 "$(p_acct "$P4")"
ck "probe 绑定保护请求正常 200" 200 "$(p_code "$P4")"
P5=$(ph "$PROBE2_KEY")
ck "★probe 无绑定流量被劫持去探测已到期 #1 → 吃真实 401(代价被退避封顶)" 401 "$(p_code "$P5")"
ck "probe 劫持落在 #1" 1 "$(p_acct "$P5")"
P6=$(ph "$PROBE2_KEY")
ck "probe 探测失败窗口即关闭(退避翻倍 2s)→ 回健康 #0" 0 "$(p_acct "$P6")"
sleep 3.0 # #1 第二轮退避 2s 到期(同样留余量)
P7=$(ph "$PROBE3_KEY")
ck "★probe #1 再次到期仍被探测(指数退避、不撒手)" 401 "$(p_code "$P7")"
ck "probe 第二轮探测仍落 #1" 1 "$(p_acct "$P7")"
UPS2=$(curl -s -b "$JAR" $API/api/admin/upstreams)
acct_view() { # $1=快照变量名内容 $2=slot $3=idx $4=路径 → 原始值(bool 小写)
printf '%s' "$1" | "$PY" -c '
import sys, json
slot, idx, path = sys.argv[2], int(sys.argv[3]), sys.argv[4]
d = json.loads(sys.argv[1])
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
for u in items:
if u.get("slot") == slot:
for a in u.get("accounts", []):
if a.get("idx") == idx:
cur = a
for k in path.split("."):
cur = cur.get(k) if isinstance(cur, dict) else None
print(str(cur).lower()); sys.exit(0)
print("missing")
' "$1" "$2" "$3" "$4"; }
ck "★probe 投影:#0 冷却已解除(自动回队)" false "$(acct_view "$UPS2" probe 0 cooled)"
ck "★probe 投影:#1 仍在冷却(退避持续)" true "$(acct_view "$UPS2" probe 1 cooled)"
ck "probe #0 至少记 1 次耗尽" yes "$([ "$(acct_view "$UPS2" probe 0 load.exhausts)" -ge 1 ] && echo yes || echo no)"
ck "probe #1 至少记 3 次耗尽(标记+两轮探测)" yes "$([ "$(acct_view "$UPS2" probe 1 load.exhausts)" -ge 3 ] && echo yes || echo no)"
# ---- 管理台「测试」按钮不得写路由状态(线上事故回归)----
# 事故复现:面板测试走 /_upstream_test,账号由 pick 随机选;上游一旦回 401+marker,
# 旧逻辑会把"管理员手抖点了一下测试"当成"这个账号额度耗尽",冷却一张好钥匙,
# 恢复要走完一轮退避/探测。fixture `probetest`:单账号 Bearer probe-dead
# (上游靶机识别后永远回 401+`status_code:1004` 标记),exhaust_status=[401] 让
# status 通道也命中——彻底钉死「测试按钮不写冷却」这条。
TB=$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=probetest" -H 'Content-Type: application/json' -d "$PING")
ck "probetest admin test → 上游真实 401 透传" 401 "$TB"
UPS_PT=$(curl -s -b "$JAR" $API/api/admin/upstreams)
ck "★admin test 后账号未进冷却(冷却不被测试按钮污染)" false "$(acct_view "$UPS_PT" probetest 0 cooled)"
ck "★admin test 不记录耗尽计数" 0 "$(acct_view "$UPS_PT" probetest 0 load.exhausts)"
# 真实流量同形态 → 标记通道必须照常工作(不能为避 bug 把真标记也关掉)
PT1=$(mh probetest probetest-model)
ck "probetest 真实流量 → 401(标记通道触发)" 401 "$(echo "$PT1" | awk '/^HTTP/{print $2}')"
UPS_PT2=$(curl -s -b "$JAR" $API/api/admin/upstreams)
ck "★真实流量 401+marker → 进冷却(标记通道保留)" true "$(acct_view "$UPS_PT2" probetest 0 cooled)"
ck "★真实流量耗尽计数 ≥1" yes "$([ "$(acct_view "$UPS_PT2" probetest 0 load.exhausts)" -ge 1 ] && echo yes || echo no)"
# ---- auth 规范化(normalize_auth)----
# fixture `normalize` 账号 auth 写成裸 key(`sk-NORM-NO-SCHEME`,无 `Bearer `),
# 期望:转发时网关自动补 `Bearer `,靶机实收完整头值(不会因缺 scheme 触发 1004)。
N1=$(mh normalize normalize-model)
ck "normalize 初次 200(裸 key 被补 Bearer 后 200)" 200 "$(echo "$N1" | awk '/^HTTP/{print $2}')"
ck "normalize 靶机实收 Bearer sk-NORM-NO-SCHEME" "Bearer sk-NORM-NO-SCHEME" "$(up_hdr Authorization)"
# ---- 账号级负载统计(管理台「密钥 / 负载」列的数据源)----
# 每次真实转发在 log 阶段落到「本次选中的那个账号」的日桶 uda:<slot>:<idx>:<day>;
# 额度耗尽另记 uex:。用同一份 /api/admin/upstreams 投影断言负载分布是否正确。
UPS=$(curl -s -b "$JAR" $API/api/admin/upstreams)
acct_num() { # $1=slot $2=idx $3=度量路径(如 load.requests)→ 取账号级数值
printf '%s' "$UPS" | "$PY" -c '
import sys, json
slot, idx, path = sys.argv[1], int(sys.argv[2]), sys.argv[3]
d = json.load(sys.stdin)
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
for u in items:
if u.get("slot") == slot:
for a in u.get("accounts", []):
if a.get("idx") == idx:
cur = a
for k in path.split("."):
cur = cur.get(k) if isinstance(cur, dict) else None
print(cur if cur is not None else 0)
sys.exit(0)
print(0)
' "$1" "$2" "$3"
}
ck "★sticky 账号#0 记到今日请求(负载分布生效)" yes "$([ "$(acct_num sticky 0 load.requests)" -ge 3 ] && echo yes || echo no)"
ck "★sticky 账号#1 未被分配(0 请求,粘性=集中在一把 Key)" 0 "$(acct_num sticky 1 load.requests)"
ck "★multi 账号#0 记到 1 次额度耗尽" yes "$([ "$(acct_num multi 0 load.exhausts)" -ge 1 ] && echo yes || echo no)"
ck "★multi 账号#1 记到今日请求(切换后负载落在新 Key)" yes "$([ "$(acct_num multi 1 load.requests)" -ge 3 ] && echo yes || echo no)"
ck "账号级趋势序列含 14 天" 14 \
"$(printf '%s' "$UPS" | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
for u in items:
if u.get("slot") == "sticky":
t = u.get("account_trend") or [{}]
print(len(t[0].get("requests") or []))
break
else:
print(0)
')"
# 日期标签与趋势序列逐位对齐(图表悬停提示用;由后端算,避免前后端各自推导时区)
ck "账号趋势带 14 个日期标签(与序列对齐)" 14 \
"$(printf '%s' "$UPS" | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
for u in items:
if u.get("slot") == "sticky":
print(len(u.get("trend_labels") or []))
break
else:
print(0)
')"
# ---- 趋势(日桶)----
# 口径与请求级用量一致:只统计真正转发出去的请求与上游 usage 里的 token
TR=$(curl -s -b "$JAR" "$API/api/admin/trend?days=7")
ck "趋势接口 → 200" 200 "$(code -b "$JAR" "$API/api/admin/trend?days=7")"
ck "趋势默认含 14 天" 14 "$(printf '%s' "$(curl -s -b "$JAR" $API/api/admin/trend)" | "$PY" -c 'import json,sys;print(len(json.load(sys.stdin)["data"]["labels"]))' 2>/dev/null)"
ck "趋势天数上限 90" 90 "$(printf '%s' "$(curl -s -b "$JAR" "$API/api/admin/trend?days=99999")" | "$PY" -c 'import json,sys;print(json.load(sys.stdin)["data"]["days"])' 2>/dev/null)"
ck "7 天区间 → 7 个标签" 7 "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;print(len(json.load(sys.stdin)["data"]["labels"]))' 2>/dev/null)"
ck "今日请求已进日桶" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if sum(d["total"]["requests"])>0 else "no")' 2>/dev/null)"
ck "今日 token 已进日桶" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if sum(d["total"]["in"])+sum(d["total"]["out"])>0 else "no")' 2>/dev/null)"
ck "趋势含按账号维度" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if any(u["username"]=="alice" for u in d["users"]) else "no")' 2>/dev/null)"
ck "趋势含按 API 组维度" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if any(not s["empty"] for s in d["slots"]) else "no")' 2>/dev/null)"
tok_ok() { "$PY" -c "
import json, sys
d = json.loads(sys.argv[1])['data']
ok_u = ok_s = False
for u in d.get('users', []):
if u['username'] == 'alice':
t = u.get('tokens', {})
ok_u = t.get('in_total', 0) >= 18 and t.get('out_total', 0) >= 25
for m in u.get('models', []):
if m['slot'] == 'openai':
ok_u = ok_u and m.get('tokens', {}).get('in_total', 0) >= 18
for s in d.get('models', []):
if s['slot'] == 'openai':
ts = s.get('tokens', {})
ok_s = ts.get('in_total', 0) >= 18 and ts.get('out_total', 0) >= 25
print('yes' if ok_u and ok_s else 'no')
" "$1"; }
ck "usage API 出现 token 维度(账号/上游明细/上游聚合)" yes "$(tok_ok "$TOK")"
ck "审计 meta 行记录 token" yes "$("$PY" -c "
import json, sys
hit = False
for f in sys.argv[1:]:
for l in open(f, encoding='utf-8'):
d = json.loads(l)
if d.get('user') == 'alice' and d.get('tin', 0) > 0:
hit = True
print('yes' if hit else 'no')
" "$AUDIT"/*/meta.jsonl 2>/dev/null)"
echo
echo "【15】★ 运行期统计落盘 / 重启恢复(发版不再清零用量与趋势)"
# 背景:用量/趋势只活在共享内存里,进程退出即消失 → 1.0.18 之前每次发版统计归零。
# 现在:退出钩子(exit_process)落盘 + 记账路径按间隔节流落盘;启动时 put-if-absent 恢复。
# 本段刻意放在**最后**:它要停掉再拉起 nginx,之后的断言环境会变(会话丢、共享内存重建)。
STATS=$RUN/data/llm-stats.json
NBI=${NGINX_BIN:-}
ck "本套件把落盘间隔压到 1 秒(nginx.conf.in)" 1 \
"$(curl -s -b "$JAR" "$API/api/admin/stats/persist" | jqv data.interval_secs)"
# 跑几笔请求,让统计有东西可落
for _ in 1 2 3; do
code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
-H 'Content-Type: application/json' -d "$BODY" >/dev/null
done
sleep 1.5 # > 间隔 1 秒:保证周期性落盘已经跑过一次
ck "★周期落盘已产出快照文件" yes "$([ -s "$STATS" ] && echo yes || echo no)"
# 落盘时刻必须从共享内存读(`spd:last`),而不是各 worker 自己的静态量 —— 否则
# 「日志明明落了 N 次、状态页却说本进程还没落过」(1.0.18 首次线上验证实测)。
# 本套件开 2 个 worker,正是能暴露该分歧的拓扑;断言看的是「任意 worker 都报得出」。
ck "★状态页报告的上次落盘时刻非空(跨 worker 一致)" yes \
"$(curl -s -b "$JAR" "$API/api/admin/stats/persist" | "$PY" -c "
import json, sys
d = json.load(sys.stdin)['data']
print('yes' if d.get('last_dump_at', 0) > 0 and d.get('last_dump_text') else 'no')
" 2>/dev/null)"
ck "★快照含账号级用量键(ust:alice)" yes \
"$("$PY" -c "
import json, sys
ok = 'no'
try:
d = json.load(open(sys.argv[1], encoding='utf-8'))
ok = 'yes' if any(k.startswith('ust:alice') for k in d.get('keys', {})) else 'no'
except Exception:
pass
print(ok)
" "$STATS" 2>/dev/null)"
ck "★快照含趋势日桶键(ud:)" yes \
"$("$PY" -c "
import json, sys
ok = 'no'
try:
d = json.load(open(sys.argv[1], encoding='utf-8'))
ok = 'yes' if any(k.startswith('ud:') for k in d.get('keys', {})) else 'no'
except Exception:
pass
print(ok)
" "$STATS" 2>/dev/null)"
ck "★快照不含凭据类配置键(up:/upa:/ax:)" yes \
"$("$PY" -c "
import json, sys
bad = 'no'
try:
d = json.load(open(sys.argv[1], encoding='utf-8'))
bad = 'yes' if any(k.split(':', 1)[0] + ':' in ('up:', 'upa:', 'upe:', 'ax:', 'ukey:', 'sub:') for k in d.get('keys', {})) else 'no'
except Exception:
pass
print('yes' if bad == 'no' else 'no')
" "$STATS" 2>/dev/null)"
T0=$(curl -s -b "$JAR" $API/api/admin/usage | "$PY" -c "
import json, sys
d = json.load(sys.stdin)['data']
for u in d.get('users', []):
if u['username'] == 'alice':
print(u['total']); break
else:
print(-1)
" 2>/dev/null)
TR0=$(curl -s -b "$JAR" "$API/api/admin/trend?days=7" | "$PY" -c "
import json, sys
print(sum(json.load(sys.stdin)['data']['total']['requests']))
" 2>/dev/null)
ck "重启前 alice 用量 > 0" yes "$([ "${T0:-0}" -gt 0 ] && echo yes || echo no)"
# ---- 优雅停机(= 生产 STOPSIGNAL SIGQUIT,会调到 worker 的 exit_process 钩子)----
if [ -n "$NBI" ]; then
"$NBI" -p "$RUN" -c "$RUN/nginx.conf" -s quit >/dev/null 2>&1
for _ in $(seq 1 30); do
code -s -o /dev/null --max-time 0.3 $GW/ >/dev/null 2>&1 || break
sleep 0.1
done
"$NBI" -p "$RUN" -c "$RUN/nginx.conf" >>"$RUN/logs/restart.out" 2>&1
sleep 0.5
fi
ck "重启后 LLM 端口可用" yes \
"$(for _ in $(seq 1 40); do [ "$(code -H 'Authorization: Bearer en_bad' -X POST $GW/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY" 2>/dev/null)" = "401" ] && { echo yes; break; }; sleep 0.1; done)"
# 会话在共享内存里(sess:),重启即失效 —— 这不是本功能的范畴,重新登录即可
JAR2=$RUN/cookie2.jar
curl -s -c "$JAR2" -X POST $API/api/admin/login -H 'Content-Type: application/json' \
-d "$(jbody username admin password 123456)" >/dev/null
PST=$(curl -s -b "$JAR2" "$API/api/admin/stats/persist")
ck "★重启后报告已恢复的键数 > 0" yes \
"$(printf '%s' "$PST" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if d.get("restored_keys",0)>0 else "no")' 2>/dev/null)"
ck "★恢复数 = 快照键数(没漏键)" yes \
"$(printf '%s' "$PST" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if d.get("restore_total",0)>0 and d.get("restored_keys",0)==d.get("restore_total",-1) else "no")' 2>/dev/null)"
T1=$(curl -s -b "$JAR2" $API/api/admin/usage | "$PY" -c "
import json, sys
d = json.load(sys.stdin)['data']
for u in d.get('users', []):
if u['username'] == 'alice':
print(u['total']); break
else:
print(-1)
" 2>/dev/null)
ck "★重启后 alice 累计用量不变(未被清零)" "$T0" "$T1"
# 再打一笔:应从恢复后的基数 +1,而不是从 0 重新长起来
code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
-H 'Content-Type: application/json' -d "$BODY" >/dev/null
T2=$(curl -s -b "$JAR2" $API/api/admin/usage | "$PY" -c "
import json, sys
d = json.load(sys.stdin)['data']
for u in d.get('users', []):
if u['username'] == 'alice':
print(u['total']); break
else:
print(-1)
" 2>/dev/null)
ck "★重启后继续累加(T0+1,而非从 0 重来)" "$(( T0 + 1 ))" "$T2"
ck "★重启后趋势日桶仍在(历史没丢)" yes \
"$(curl -s -b "$JAR2" "$API/api/admin/trend?days=7" | "$PY" -c "
import json, sys
print('yes' if sum(json.load(sys.stdin)['data']['total']['requests']) >= $TR0 else 'no')
" 2>/dev/null)"
ck "★重启后账号级负载仍可读(uda: 恢复)" yes \
"$(curl -s -b "$JAR2" $API/api/admin/upstreams | "$PY" -c "
import json, sys
d = json.load(sys.stdin)['data']
ok = False
for s in d.get('items', []):
if s.get('slot') == 'openai':
for a in s.get('accounts', []):
if (a.get('load') or {}).get('requests', 0) > 0:
ok = True
print('yes' if ok else 'no')
" 2>/dev/null)"
echo
echo "============================================================"
printf " 通过 \033[32m%d\033[0m 失败 \033[31m%d\033[0m\n" "$pass" "$fail"
echo "============================================================"
[ "$fail" -eq 0 ] || exit 1