线上事故回归:面板「测试」按钮走 /_upstream_test 真实转发,账号由 pick 随机 选;上游一旦回耗尽形态(状态码命中 upe 或 body 命中 marker),旧逻辑会把 「管理员手抖点测试」当成「账号额度耗尽」冷却一把好钥匙,恢复要走完整退避/ 探测。13:50 一次面板测试轮把 minimax 三把好 key 全部点进冷却,整池不可用。 * handlers.rs LlmHealthHandler:is_probe URI 前缀请求**不写冷却/耗尽**;只 跑 mark_health(健康展示仍需)。2xx clear_cool 保留——测试成功 = 该账号此刻 健康,是正向证据。bump_acct_load 保留(测试也是真转发,统计有用)。 * tests/llm/e2e.sh:新增「管理台测试按钮不得写路由状态」剧本,6 条断言; fixture probetest 单账号 Bearer probe-dead + exhaust_status=[401] + markers 一并命中——status 通道 + body 通道都能触发,真实分流修复前 cool 必须为 true,修复后为 false;真实流量同时仍正常冷却(不能为避 bug 把真标记 也关掉)。 * tests/llm/fixtures/llm-upstreams.json:加 probetest 槽位(同形 401+marker)。 回归:bash tests/run.sh llm → 270/0 全绿(原 262 + 8)。
1119 lines
65 KiB
Bash
Executable File
1119 lines
65 KiB
Bash
Executable File
#!/bin/bash
|
||
# ============================================================================
|
||
# LLM 代理 · 本机回归
|
||
#
|
||
# 一般不用直接跑,走统一入口: bash tests/run.sh llm
|
||
# 直接跑需自行保证:假上游(tests/llm/upstream.py)已在 18999 监听,
|
||
# nginx 已按 tests/llm/nginx.conf.in 渲染并启动,且 data/ 用的是 fixtures 里
|
||
# 那两个固定文件(alice 白名单=openai、bob 白名单为空=不限)。
|
||
#
|
||
# ★ 核心:不只验网关自己的响应码,而是**读靶机实际收到的请求**——
|
||
# 「下游 Key 是否真被换成上游真 Key」只有看靶机才能证明。
|
||
#
|
||
# 可覆盖:GW / UPSTREAM_LOG / PY
|
||
# ============================================================================
|
||
set -u
|
||
|
||
HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||
RUN=${RUN:-$(cd "$HERE/.." && pwd)/.run/llm}
|
||
|
||
GW=${GW:-http://127.0.0.1:18090}
|
||
# 管理端口(同一份 nginx / 同一块共享内存)——用量与模型清单从这边读
|
||
API=${API:-http://127.0.0.1:18091}
|
||
UP_LOG=${UPSTREAM_LOG:-$RUN/upstream.log}
|
||
PY=${PY:-python3}
|
||
|
||
# fixtures 里的固定 Key(见 fixtures/users.json)
|
||
ALICE_KEY=sk_alice_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||
BOB_KEY=sk_bob_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
|
||
# 多账号路由验证用 admin:白名单为空=不限、无配额限制、且本套件中途会删除 bob 故不复用
|
||
ADMIN_KEY=sk_admin_0000000000000000000000000000000000000000000000ab
|
||
|
||
pass=0; fail=0
|
||
ck() {
|
||
if [ "$2" = "$3" ]; then
|
||
printf " \033[32m✓\033[0m %s\n" "$1"; pass=$((pass+1))
|
||
else
|
||
printf " \033[31m✗\033[0m %s 期望=[%s] 实际=[%s]\n" "$1" "$2" "$3"; fail=$((fail+1))
|
||
fi
|
||
}
|
||
code() { curl -s -o /dev/null -w '%{http_code}' "$@"; }
|
||
jbody() { local out='{' sep=''; while [ $# -gt 1 ]; do out="$out$sep\"$1\":\"$2\""; sep=','; shift 2; done; printf '%s}' "$out"; }
|
||
# 靶机日志:取最后一条记录的某个字段
|
||
up_field() {
|
||
"$PY" -c "
|
||
import json, sys
|
||
last = None
|
||
try:
|
||
for l in open(sys.argv[1], encoding='utf-8'):
|
||
l = l.strip()
|
||
if l: last = l
|
||
except FileNotFoundError:
|
||
pass
|
||
print(json.loads(last).get(sys.argv[2], '') if last else '')
|
||
" "$UP_LOG" "$1" 2>/dev/null
|
||
}
|
||
up_count() { if [ -f "$UP_LOG" ]; then awk 'NF{n++} END{print n+0}' "$UP_LOG"; else echo 0; fi }
|
||
# 靶机日志:取最后一条记录里**某个请求头**的值(头名大小写不敏感)。
|
||
# 专门为「出站头策略」服务:要证明某个头没被发出去 / 某个头还活着,
|
||
# 只能看上游实际收到了什么。取不到的空值 = 该头确实没发。
|
||
up_hdr() {
|
||
"$PY" -c "
|
||
import json, sys
|
||
last = None
|
||
try:
|
||
for l in open(sys.argv[1], encoding='utf-8'):
|
||
l = l.strip()
|
||
if l: last = l
|
||
except FileNotFoundError:
|
||
pass
|
||
h = (json.loads(last).get('headers') or {}) if last else {}
|
||
print(h.get(sys.argv[2].lower(), ''))
|
||
" "$UP_LOG" "$1" 2>/dev/null
|
||
}
|
||
post() { curl -s -H "Authorization: Bearer $1" -X POST "$GW$2" -H 'Content-Type: application/json' -d "$3"; }
|
||
|
||
# 从 stdin 的 JSON 里按点分路径取值
|
||
jqv() {
|
||
"$PY" -c "
|
||
import sys, json
|
||
try: d = json.load(sys.stdin)
|
||
except Exception: print(''); sys.exit()
|
||
for k in sys.argv[1].split('.'):
|
||
if d is None: break
|
||
d = d[int(k)] if k.isdigit() and isinstance(d, list) else (d.get(k) if isinstance(d, dict) else None)
|
||
print('' if d is None else d)
|
||
" "$1" 2>/dev/null
|
||
}
|
||
# 在 stdin 里找子串(命中输出 yes,否则 no)
|
||
has() { if grep -q "$1"; then echo yes; else echo no; fi; }
|
||
# 反向断言:**不**出现该子串才算过
|
||
nhas() { if grep -q "$1"; then echo no; else echo yes; fi; }
|
||
|
||
echo "【1】鉴权(OpenAI 兼容错误体)"
|
||
BODY=$(jbody model gpt-4o)
|
||
ck "无 Key → 401" 401 "$(code -X POST $GW/openai/v1/chat/completions)"
|
||
ck "错误 Key → 401" 401 "$(code -X POST $GW/openai/v1/chat/completions -H 'Authorization: Bearer sk_deadbeef')"
|
||
ck "错误体是 OpenAI 形状(含 error.type)" yes \
|
||
"$(curl -s -X POST $GW/openai/v1/chat/completions | grep -q '"error"' && echo yes || echo no)"
|
||
ck "路径缺 slot → 400" 400 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/)"
|
||
ck "未知上游 → 404" 404 "$(code -H "Authorization: Bearer $BOB_KEY" -X POST $GW/nope/v1/chat/completions)"
|
||
|
||
echo
|
||
echo "【2】上游解析与白名单"
|
||
ck "alice + openai(在白名单)→ 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
|
||
ck "响应体来自上游(含 pong)" yes "$(post "$ALICE_KEY" /openai/v1/chat/completions "$BODY" | grep -q pong && echo yes || echo no)"
|
||
ck "alice + other(不在白名单)→ 403" 403 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/other/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
|
||
ck "alice + downstream(已停用)→ 404" 404 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/downstream/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
|
||
ck "bob + other(空白名单=不限)→ 200" 200 "$(code -H "Authorization: Bearer $BOB_KEY" -X POST $GW/other/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
|
||
|
||
echo
|
||
echo "【3】★ Key 替换 / 路径 / 头(读靶机实际收到的请求)"
|
||
N0=$(up_count)
|
||
post "$ALICE_KEY" "/openai/v1/chat/completions?a=1" "$BODY" >/dev/null
|
||
ck "靶机恰好收到 1 次转发" 1 "$(( $(up_count) - N0 ))"
|
||
ck "★Authorization 已换成上游真 Key" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
|
||
ck "★下游 Key 未泄漏给上游" yes "$(up_field auth | grep -q 'sk_alice_' && echo leaked || echo yes)"
|
||
ck "Host 头 = 上游主机(非下游域名)" "127.0.0.1:18999" "$(up_field host)"
|
||
ck "★路径去 slot 且保留 query" "/v1/chat/completions?a=1" "$(up_field path)"
|
||
ck "请求体原样透传" "$BODY" "$(up_field body)"
|
||
|
||
# ★ 出站头策略(对齐 LiteLLM 的两层:默认不放行 + 凭据头显式抹空)。
|
||
# 构造最坏情形:客户端把**网关自己的 Key** 塞进每一个凭据头,
|
||
# 同时带上两套协议的扩展头,然后看上游究竟收到了什么。
|
||
# 注意这里**故意不发 Authorization**,顺带验证「入站只靠 x-api-key 也能过网关」。
|
||
curl -s -o /dev/null -X POST "$GW/openai/v1/chat/completions" \
|
||
-H "x-api-key: $ALICE_KEY" \
|
||
-H "api-key: $ALICE_KEY" \
|
||
-H "x-goog-api-key: $ALICE_KEY" \
|
||
-H 'anthropic-version: 2023-06-01' \
|
||
-H 'anthropic-beta: prompt-caching-2024-07-31' \
|
||
-H 'openai-beta: assistants=v2' \
|
||
-H 'Content-Type: application/json' -d "$BODY"
|
||
ck "★x-api-key 未泄漏给上游(默认摘除清单)" "" "$(up_hdr x-api-key)"
|
||
ck "★api-key 未泄漏给上游" "" "$(up_hdr api-key)"
|
||
ck "★x-goog-api-key 未泄漏给上游" "" "$(up_hdr x-goog-api-key)"
|
||
ck "★Authorization 仍是上游真 Key(客户端塞不进凭据)" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
|
||
ck "★Content-Type 已注入(漏了所有 POST 都会废)" "application/json" "$(up_hdr content-type)"
|
||
ck " Accept 已注入(SSE 协商要靠它)" "*/*" "$(up_hdr accept)"
|
||
ck "★anthropic-version 已注入(Anthropic 协议必填)" "2023-06-01" "$(up_hdr anthropic-version)"
|
||
ck "★anthropic-beta 已注入(Claude Code 扩展依赖)" "prompt-caching-2024-07-31" "$(up_hdr anthropic-beta)"
|
||
ck " openai-beta 已注入" "assistants=v2" "$(up_hdr openai-beta)"
|
||
ck " User-Agent 已透传(厂商侧排障要靠它)" yes \
|
||
"$([ -n "$(up_hdr user-agent)" ] && echo yes || echo no)"
|
||
|
||
echo
|
||
echo "【4】上游 base 带路径时的拼接"
|
||
post "$BOB_KEY" /other/v1/chat/completions "$BODY" >/dev/null
|
||
ck "★base=/llm + 客户端带 /v1 → 剥掉(base 自带路径)" "/llm/chat/completions" "$(up_field path)"
|
||
ck "用的是该上游自己的 Key" "Bearer sk-OTHER-UPSTREAM" "$(up_field auth)"
|
||
|
||
# ★ 版本段去重(实测踩过的坑:厂商 base_url 自带 /v1,客户端也带 /v1 → /v1/v1 → 上游 404 空响应)
|
||
post "$BOB_KEY" /deepseeklike/v1/chat/completions "$BODY" >/dev/null
|
||
ck "★base 自带 /v1 + 客户端带 /v1 → 只保留一份" "/v1/chat/completions" "$(up_field path)"
|
||
post "$BOB_KEY" /deepseeklike/chat/completions "$BODY" >/dev/null
|
||
ck " 客户端不带 /v1 → 照常前缀上 base 的 /v1" "/v1/chat/completions" "$(up_field path)"
|
||
post "$BOB_KEY" /deepseeklike/v2/chat/completions "$BODY" >/dev/null
|
||
ck " 不同版本段不误判为重复" "/v1/v2/chat/completions" "$(up_field path)"
|
||
|
||
# ★ 「统一注册」= base 只写到 host,一条记录同时服务两套协议。
|
||
# 这是 LiteLLM(/anthropic/* 前缀)/ one-api(根地址 + 路径天然区分)/
|
||
# Higress(按路径后缀判协议)三个开源实现的共同做法:
|
||
# **协议由客户端路径表达,网关不做任何推断、不需要分发代码**。
|
||
# 夹具里 openai 的 base = http://127.0.0.1:18999(纯 host),正好验这条。
|
||
post "$ALICE_KEY" "/openai/anthropic/v1/messages" "$BODY" >/dev/null
|
||
ck "★Anthropic 路径原样拼在 host 粒度 base 之后(裸 host 客户端 /v1 保留)" "/anthropic/v1/messages" "$(up_field path)"
|
||
ck " 同一条记录也换上游真 Key(不是只有 OpenAI 路径才换)" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
|
||
|
||
echo
|
||
echo "【5】★ 流式(SSE 不被缓冲)"
|
||
SSE=$(GW="$GW" KEY="$ALICE_KEY" "$PY" "$HERE/sse_test.py" 2>&1)
|
||
ck "Content-Type 是 text/event-stream" "text/event-stream" "$(echo "$SSE" | sed -n 's/^CT=//p')"
|
||
ck "收到 3 个 data 块" 3 "$(echo "$SSE" | sed -n 's/^CHUNKS=//p')"
|
||
FIRST=$(echo "$SSE" | sed -n 's/^FIRST=//p')
|
||
TOTAL=$(echo "$SSE" | sed -n 's/^TOTAL=//p')
|
||
ck "★首块 < 0.5s 到达(未被攒包)" yes "$(awk -v f="${FIRST:-9}" 'BEGIN{print (f<0.5)?"yes":"no"}')"
|
||
ck "总耗时 ≥ 1.0s(确实分了 3 次推)" yes "$(awk -v t="${TOTAL:-0}" 'BEGIN{print (t>=1.0)?"yes":"no"}')"
|
||
|
||
echo
|
||
echo "【6】★ 限流与每日配额(carol rpm=2、dave daily=2,见 fixtures/users.json)"
|
||
CAROL_KEY=sk_carol_cccccccccccccccccccccccccccccccccccccccccccccc
|
||
DAVE_KEY=sk_dave_dddddddddddddddddddddddddddddddddddddddddddddddd
|
||
# 探测闭环组专用(合成数据)。probe1 的 FNV64(key)%2=0 → 冷分配必落 probe 槽 #0;
|
||
# probe2/3 只在探测分支(先于哈希)被使用,落点不依赖哈希。
|
||
PROBE1_KEY=sk_probe_one_00000000000000000000000000000000
|
||
PROBE2_KEY=sk_probe_two_00000000000000000000000000000001
|
||
PROBE3_KEY=sk_probe_three_0000000000000000000000000000000
|
||
cpost() { code -H "Authorization: Bearer $1" -X POST "$GW/other/v1/chat/completions" -H 'Content-Type: application/json' -d "$BODY"; }
|
||
ck "carol 第 1 发 → 200" 200 "$(cpost "$CAROL_KEY")"
|
||
ck "carol 第 2 发 → 200" 200 "$(cpost "$CAROL_KEY")"
|
||
ck "carol 第 3 发 → 429(rpm=2)" 429 "$(cpost "$CAROL_KEY")"
|
||
ck "429 是 rate_limit_error / rate_limit_exceeded" yes \
|
||
"$(post "$CAROL_KEY" /other/v1/chat/completions "$BODY" | grep -q 'rate_limit' && echo yes || echo no)"
|
||
ck "dave 第 1 发 → 200" 200 "$(cpost "$DAVE_KEY")"
|
||
ck "dave 第 2 发 → 200" 200 "$(cpost "$DAVE_KEY")"
|
||
ck "dave 第 3 发 → 429(daily=2)" 429 "$(cpost "$DAVE_KEY")"
|
||
ck "429 是 insufficient_quota" yes \
|
||
"$(post "$DAVE_KEY" /other/v1/chat/completions "$BODY" | grep -q 'insufficient_quota' && echo yes || echo no)"
|
||
ck "carol 被限后 alice 不受影响(按 Key 隔离)→ 200" 200 \
|
||
"$(code -H "Authorization: Bearer $ALICE_KEY" -X POST "$GW/openai/v1/chat/completions" -H 'Content-Type: application/json' -d "$BODY")"
|
||
|
||
if [ "${HAVE_TLS:-0}" = "1" ]; then
|
||
echo
|
||
echo "【7】★ HTTPS(18443 自签,模板 tests/llm/tls.conf.in)"
|
||
HTTPS_BASE=https://127.0.0.1:18443
|
||
ck "https + 错误 Key → 401(TLS 层下鉴权仍生效)" 401 \
|
||
"$(code -k -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Authorization: Bearer nope')"
|
||
ck "https + alice → 200" 200 \
|
||
"$(code -k -H "Authorization: Bearer $ALICE_KEY" -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
|
||
ck "https 响应体来自上游" yes \
|
||
"$(curl -sk -H "Authorization: Bearer $ALICE_KEY" -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY" | grep -q pong && echo yes || echo no)"
|
||
else
|
||
echo
|
||
echo "【7】HTTPS:跳过(本机 nginx 无 http_ssl_module 或无 openssl)"
|
||
fi
|
||
|
||
echo
|
||
echo "【8】★ 模型发现(网关本地应答,不去上游)"
|
||
M0=$(up_count)
|
||
ck "无 Key → 401" 401 "$(code $GW/v1/models)"
|
||
ck "alice GET /v1/models → 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models)"
|
||
ck " 形状是 OpenAI 的 object=list" yes \
|
||
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | has '"object":"list"')"
|
||
ck " 含白名单内上游的模型 gpt-4o" yes \
|
||
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | has 'gpt-4o')"
|
||
ck " 不含白名单外上游的模型 claude-3-5-sonnet" yes \
|
||
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | nhas 'claude-3-5-sonnet')"
|
||
ck " 不含已停用上游的模型 should-not-appear" yes \
|
||
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | nhas 'should-not-appear')"
|
||
ck "bob(白名单为空=不限)能看到 other 的模型" yes \
|
||
"$(curl -s -H "Authorization: Bearer $BOB_KEY" $GW/v1/models | has 'claude-3-5-sonnet')"
|
||
ck "GET /models 同样本地应答 → 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/models)"
|
||
ck "★整个模型发现过程 0 次上游调用" 0 "$(( $(up_count) - M0 ))"
|
||
|
||
# 带 slot 的 /v1/models 是**另一条路**:照常透传给厂商,取"那家厂商的"模型清单
|
||
N0=$(up_count)
|
||
ck "★/<slot>/v1/models 仍透传上游 → 200" 200 \
|
||
"$(code -H "Authorization: Bearer $ALICE_KEY" $GW/openai/v1/models)"
|
||
ck " 上游确实收到了 1 次请求" 1 "$(( $(up_count) - N0 ))"
|
||
ck " 转发路径未剥错(/v1/models)" "/v1/models" "$(up_field path)"
|
||
ck " 响应体来自上游(不是网关本地那份)" yes \
|
||
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/openai/v1/models | has 'upstream-model')"
|
||
|
||
echo
|
||
echo "【9】★ 账号态 API:可用模型 + 自己的用量"
|
||
# base_url 跟随**调用方所用的端口**(88 部署必须带 :88;443 反代后面则不带)——
|
||
# 见 rust/src/upstreams.rs 的 base_url 注释,写死任何一种都会在另一种部署里给出错地址。
|
||
API_PORT="${API##*:}"
|
||
ck "无 Key 查模型 → 401" 401 "$(code $API/api/v1/models)"
|
||
ck "无 Key 查用量 → 401" 401 "$(code $API/api/v1/usage)"
|
||
ck "无 Key 查 me → 401" 401 "$(code $API/api/v1/me)"
|
||
ck "alice 查 me → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/me)"
|
||
# ★ /me 必须带 recent 块(1.0.29 同事反馈「一次拿全我最近调用」)
|
||
# - calls_7d ≥ 1(alice 前面已调过 openai / other,应有 audit meta.jsonl)
|
||
# - by_slot 数组非空(含调过的上游 slot)
|
||
# - hour_24h 是 24 长度的数组
|
||
ME_DATA=$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/me)
|
||
ck "★me.recent.calls_7d ≥ 1" yes \
|
||
"$("$PY" -c "
|
||
import json, sys
|
||
d = json.loads(sys.argv[1])
|
||
r = d.get('data', {}).get('recent', {})
|
||
print('yes' if r.get('calls_7d', 0) >= 1 else 'no')
|
||
" "$ME_DATA")"
|
||
ck "★me.recent.by_slot 非空" yes \
|
||
"$("$PY" -c "
|
||
import json, sys
|
||
d = json.loads(sys.argv[1])
|
||
r = d.get('data', {}).get('recent', {})
|
||
print('yes' if len(r.get('by_slot', [])) >= 1 else 'no')
|
||
" "$ME_DATA")"
|
||
ck "★me.recent.hour_24h 是 24 长度数组" yes \
|
||
"$("$PY" -c "
|
||
import json, sys
|
||
d = json.loads(sys.argv[1])
|
||
r = d.get('data', {}).get('recent', {})
|
||
h = r.get('hour_24h', [])
|
||
print('yes' if isinstance(h, list) and len(h) == 24 else 'no')
|
||
" "$ME_DATA")"
|
||
ck "★me.recent.by_slot 字段完整" yes \
|
||
"$("$PY" -c "
|
||
import json, sys
|
||
d = json.loads(sys.argv[1])
|
||
r = d.get('data', {}).get('recent', {})
|
||
slots = r.get('by_slot', [])
|
||
ok = bool(slots) and all(
|
||
set(['slot','calls_7d','calls_24h']).issubset(s.keys()) for s in slots
|
||
)
|
||
print('yes' if ok else 'no')
|
||
" "$ME_DATA")"
|
||
ck "alice 查模型 → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/models)"
|
||
ck " 模型清单里带 base_url(lm 域名 + 端口 + slot)" yes \
|
||
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has "lm.mayishangshu.cn:$API_PORT/openai")"
|
||
# ★ base_url / anthropic_base_url 是**纯网关出口地址**(只含 slot,不带厂商路径、不带 /v1):
|
||
# 厂商版本/协议路径段(/v1 / /compatible-mode/v1 / /api/coding/paas/v4 / /anthropic …)
|
||
# 原样存在上游的 base / base_anthropic 里,转发时由 join_target 拼到 SDK 余下路径前。
|
||
# 这样 OpenAI SDK 填 `…/openai`、Anthropic SDK 填 `…/openai/anthropic` 即可,网关不猜路径。
|
||
# (校验完整字段值而非整份响应子串——anthropic_base_url 同段会出现,避免误伤。)
|
||
ck "★base_url 是纯 slot(无 /v1、不带上游 base 路径)" yes \
|
||
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has "\"base_url\":\"https://lm.mayishangshu.cn:$API_PORT/openai\"")"
|
||
ck "★base_url 不含 /v1(版本段交给上游 base)" no \
|
||
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has "\"base_url\":\"https://lm.mayishangshu.cn:$API_PORT/openai/v1\"")"
|
||
ck "★base_url 纯 slot:other 也不带 /llm(路径在上游 base 里)" yes \
|
||
"$(curl -s -H "X-API-Key: $BOB_KEY" $API/api/v1/models | has "lm.mayishangshu.cn:$API_PORT/other\"")"
|
||
ck "★anthropic_base_url 是 /<slot>/anthropic(Anthropic SDK 内部拼 /v1/messages)" yes \
|
||
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has "anthropic_base_url\":\"https://lm.mayishangshu.cn:$API_PORT/openai/anthropic\"")"
|
||
# ★ 端口自适应:Host 不带端口(走 443 反代)时 base_url 也不该带端口。
|
||
# 写死 88 → 443 部署下客户端连到另一个端口的服务;裸域名 → 88 部署下被出口设备抢答
|
||
# (实测 HTTP 200 + 一个 HTML 登录页,是最难看出问题的一种失败)。
|
||
ck "★Host 无端口 → base_url 不带端口" yes \
|
||
"$(curl -s -H "X-API-Key: $ALICE_KEY" -H 'Host: lm.example' $API/api/v1/models | has 'lm.mayishangshu.cn/openai')"
|
||
ck "★Host 带端口 → base_url 带同一端口" yes \
|
||
"$(curl -s -H "X-API-Key: $ALICE_KEY" -H "Host: lm.example:$API_PORT" $API/api/v1/models | has "lm.mayishangshu.cn:$API_PORT/openai")"
|
||
|
||
echo
|
||
echo "【9.5】★ 双协议地址:OpenAI 与 Anthropic 路径各走各的(智谱/阿里形态)"
|
||
# glm 上游 OpenAI 路径 /api/coding/paas/v4、Anthropic 路径 /api/anthropic,完全不同。
|
||
# 网关按「路径里有没有 /anthropic 段」分流到 base / base_anthropic,其余路径原样拼。
|
||
# 真实客户端统一带 /v1(见 /openai/v1/chat/completions 约定),所以这里也发 /v1,
|
||
# 验证 join_target 会剥掉客户端多余的 /v1(否则 glm 会被拼成 …/v4/v1 被智谱 404)。
|
||
# - `/glm/v1/chat/completions` → 上游 /api/coding/paas/v4/chat/completions
|
||
# - `/glm/anthropic/v1/messages` → 上游 /api/anthropic/v1/messages
|
||
# 用 admin key(绕过白名单)直打,up_field path 读靶机实际收到的路径。
|
||
GLM_BODY='{"model":"glm-4","max_tokens":1,"messages":[{"role":"user","content":"hi"}]}'
|
||
ck "★glm OpenAI 协议(带 /v1)→ 200" 200 \
|
||
"$(code -X POST -H "X-API-Key: $ADMIN_KEY" -H 'Content-Type: application/json' \
|
||
-d "$GLM_BODY" $GW/glm/v1/chat/completions)"
|
||
ck " 靶机路径 = base + /chat/completions(客户端 /v1 已剥)" \
|
||
"/api/coding/paas/v4/chat/completions" "$(up_field path)"
|
||
ck "★glm Anthropic 协议 → 200" 200 \
|
||
"$(code -X POST -H "X-API-Key: $ADMIN_KEY" -H 'Content-Type: application/json' \
|
||
-d "$GLM_BODY" $GW/glm/anthropic/v1/messages)"
|
||
ck " 靶机路径 = base_anthropic + /v1/messages(/anthropic 段已剥)" \
|
||
"/api/anthropic/v1/messages" "$(up_field path)"
|
||
|
||
ALICE_USAGE=$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/usage)
|
||
ck "alice 查用量 → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/usage)"
|
||
ALICE_TOTAL=$(echo "$ALICE_USAGE" | jqv data.total)
|
||
ck "★alice 累计用量 > 0(前面真发过请求)" yes \
|
||
"$(awk -v t="${ALICE_TOTAL:-0}" 'BEGIN{print (t>0)?"yes":"no"}')"
|
||
ck " 今日 = 累计(都是刚发的)" "$ALICE_TOTAL" "$(echo "$ALICE_USAGE" | jqv data.today)"
|
||
ck " 分上游明细含 openai" yes "$(echo "$ALICE_USAGE" | has '"slot": *"openai"')"
|
||
ck "★被拒的越权/未知上游不进明细(无 other/downstream)" yes \
|
||
"$(echo "$ALICE_USAGE" | nhas '"slot": *"other"')"
|
||
ck " 回显限额供客户端自我节流" yes "$(echo "$ALICE_USAGE" | has '"limits"')"
|
||
|
||
# ★ 关键语义:统计的是"真的转发出去"的请求 —— 超限被 429 的那一发不计
|
||
CAROL_TOTAL=$(curl -s -H "X-API-Key: $CAROL_KEY" $API/api/v1/usage | jqv data.total)
|
||
ck "★carol 发了 3 次、第 3 次被限流 → 用量只记 2" 2 "${CAROL_TOTAL:-x}"
|
||
DAVE_TOTAL=$(curl -s -H "X-API-Key: $DAVE_KEY" $API/api/v1/usage | jqv data.total)
|
||
ck "★dave 同上(每日配额)→ 用量只记 2" 2 "${DAVE_TOTAL:-x}"
|
||
ck "bob 的明细含 other(他真调过)" yes \
|
||
"$(curl -s -H "X-API-Key: $BOB_KEY" $API/api/v1/usage | has '"slot": *"other"')"
|
||
|
||
echo
|
||
echo "【10】管理员用量总览 + 删账号清理用量键"
|
||
ck "无会话 → 401" 401 "$(code $API/api/admin/usage)"
|
||
ck "不是接口路径 → 404" 404 "$(code $API/api/v1/nope)"
|
||
|
||
JAR=$RUN/cookie.jar
|
||
LOGIN=$(curl -s -c "$JAR" -X POST $API/api/admin/login -H 'Content-Type: application/json' \
|
||
-d "$(jbody username admin password 123456)")
|
||
ck "管理员登录 → 200" 200 "$(echo "$LOGIN" | jqv code)"
|
||
|
||
USAGE=$(curl -s -b "$JAR" $API/api/admin/usage)
|
||
ck "用量总览含 alice" yes "$(echo "$USAGE" | has '"username": *"alice"')"
|
||
# ★ 上游视图返回 base_anthropic:双协议地址各厂商路径不同,必须随视图给出
|
||
# (智谱 /api/coding/paas/v4 ↔ /api/anthropic、阿里 /compatible-mode/v1 ↔ /apps/anthropic)。
|
||
ck "★上游视图返回 base_anthropic 字段" yes \
|
||
"$(curl -s -b "$JAR" $API/api/admin/upstreams | has '"base_anthropic"')"
|
||
ck " 总览含按上游分组(openai)" yes "$(echo "$USAGE" | has '"slot": *"openai"')"
|
||
ck " 全局合计 > 0" yes "$(echo "$USAGE" | jqv data.total.total | awk '{print ($1>0)?"yes":"no"}')"
|
||
ck "★删除前用量榜里有 bob(他真调过 other)" yes "$(echo "$USAGE" | has '"username": *"bob"')"
|
||
|
||
# ★ 删除账号必须**连带清掉用量键**(`ust:` / `ums:`)——它们是按前缀扫出来出报表的,
|
||
# 不清就会让已删账号一直挂在用量榜上(与"删账号必须摘 ukey:"同类的正确性问题)。
|
||
curl -s -b "$JAR" -X POST $API/api/admin/users/delete -H 'Content-Type: application/json' \
|
||
-d "$(jbody username bob)" >/dev/null
|
||
ck "★bob 已被删除(他的 Key 立即失效 → 401)" 401 "$(code -H "X-API-Key: $BOB_KEY" $API/api/v1/usage)"
|
||
|
||
USAGE2=$(curl -s -b "$JAR" $API/api/admin/usage)
|
||
ck "★删除后用量榜不再出现 bob" yes "$(echo "$USAGE2" | nhas 'bob')"
|
||
ck " 但 alice 仍在榜单上" yes "$(echo "$USAGE2" | has '"username": *"alice"')"
|
||
|
||
echo
|
||
echo "【11】★ 上游连通性测试(管理台「测试」按钮的后端链路)"
|
||
# 管理台把探测体原样发上来,网关只负责「定位上游 + 换真 Key + 转发 + 透传状态」。
|
||
# 判据照旧:不只看 HTTP 200,还要看靶机实际收到的 path / auth。
|
||
PING='{"model":"ds-chat","max_tokens":1,"stream":false,"messages":[{"role":"user","content":"ping"}]}'
|
||
ck "无会话测试 → 401" 401 \
|
||
"$(code -X POST "$API/api/admin/upstreams/test?slot=openai" -d '{}')"
|
||
ck "★admin 测 openai → 透传上游 200" 200 \
|
||
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=openai" -H 'Content-Type: application/json' -d "$PING")"
|
||
ck " 靶机收到的 path 不含 slot 段" "/v1/chat/completions" "$(up_field path)"
|
||
ck " 换上了 openai 上游的真 Key" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
|
||
ck " ★探测体原样到达靶机(model=ds-chat,未被网关注入体覆盖)" "yes" \
|
||
"$(up_field body | grep -q 'ds-chat' && echo yes || echo no)"
|
||
ck "★base 自带 /v1 的上游(deepseeklike)→ 去重后 200" 200 \
|
||
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=deepseeklike" -H 'Content-Type: application/json' -d "$PING")"
|
||
ck " 靶机 path 仍是 /v1/chat/completions(无 /v1/v1、无 slot 段)" "/v1/chat/completions" "$(up_field path)"
|
||
ck " 换上了 deepseeklike 的真 Key" "Bearer sk-DS-REAL" "$(up_field auth)"
|
||
ck "★已停用的上游测不了(shm 已摘除 → 404)" 404 \
|
||
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=downstream" -d '{}')"
|
||
ck "不存在的 slot → 404 信封" 404 \
|
||
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=ghost" -d '{}')"
|
||
ck "★内部 location 外部直连 → 404(不能绕过 admin 会话触达上游)" 404 \
|
||
"$(code -b "$JAR" -X POST "$API/_upstream_test/openai/v1/chat/completions" -d '{}')"
|
||
# —— 静态回归闸 ——
|
||
# 生产 conf 与测试模板是两份拷贝,历史事故(面板测 minimax 报
|
||
# `unknown model 'test' (2013)`)正是二者漂移:1.0.43 为当时的「空 body 探测」
|
||
# 兜底,在 conf/nginx.conf 的 /_upstream_test/ 注入了 proxy_set_body dummy 体,
|
||
# 而 nginx 的 proxy_set_body 是**无条件覆盖**——把面板发来的「第一个登记模型」
|
||
# 探测体整个换掉,严格校验模型名的厂商全部误报「不通」,本夹具却因模板没这行
|
||
# 而全绿。探测体必须原样转发,这道闸防注入行只在线上配置里复活。
|
||
# 只看生效行:conf 注释里「别加 proxy_set_body」的告诫会字面命中 grep
|
||
UB_BLOCK=$(awk '/location \/_upstream_test\//{f=1} f{print} f&&/^ \}$/{exit}' "$HERE/../../conf/nginx.conf" | grep -v '^[[:space:]]*#')
|
||
ck "★生产 conf 探测块无 proxy_set_body(探测体原样转发)" "no" \
|
||
"$(echo "$UB_BLOCK" | grep -q proxy_set_body && echo yes || echo no)"
|
||
|
||
echo
|
||
echo "【11.5】★ API 代理(/apix/<slug> → 外部供应商)"
|
||
# 与 LLM 代理同一把 sk_ Key,但上游形态不同:供应商只有一个收口,资产由
|
||
# **请求体里的 asset_id** 决定,所以请求体必须由网关拼——判据是看靶机
|
||
# 收到的 path / 头 / body 三件事,只看网关自己的 200 什么都证明不了。
|
||
ck "无 Key → 401" 401 "$(code "$GW/apix/search?q=hello")"
|
||
ck "坏 Key → 401" 401 "$(code -H 'X-API-Key: sk_nope' "$GW/apix/search?q=hello")"
|
||
ck "不存在的 slug → 404" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/ghost?q=1")"
|
||
ck "★已停用的 slug → 404(shm 里已摘除)" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/offline?q=1")"
|
||
ck "★正常调用 → 透传上游 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/search?q=hello")"
|
||
ck " 靶机 path = 供应商的收口" "/api/v1/ai/request" "$(up_field path)"
|
||
ck " ★GET 被改写成 POST(proxy_method + proxy_set_body)" "POST" "$(up_field method)"
|
||
ck " ★换上了供应商真 Key" "ak_test_supplier_key" "$(up_hdr x-api-key)"
|
||
ck " ★调用体由网关拼:asset_id 来自配置" yes \
|
||
"$(up_field body | grep -q '"asset_id":"searxng-search"' && echo yes || echo no)"
|
||
ck " ★query 参数进 call_data" yes \
|
||
"$(up_field body | grep -q '"q":"hello"' && echo yes || echo no)"
|
||
ck " ★下行 Key 没被带出去" yes "$(up_field body | nhas 'sk_alice')"
|
||
ck " 下游 Authorization 未透传" "" "$(up_hdr authorization)"
|
||
# x-api-key 头**会**有值——那正是换上的供应商真 Key。要证的是它不再以下游的
|
||
# sk_ 开头:客户端用自己的 Key 换来供应商凭据,而不是凭据被原样透传。
|
||
AXH=$(up_hdr x-api-key)
|
||
ck " ★下游 x-api-key 已被换掉(不再是 sk_ 开头)" no \
|
||
"$(printf '%s' "$AXH" | grep -q '^sk_' && echo yes || echo no)"
|
||
|
||
# 管理台:清单 / 凭据掩码 / 停启用写穿
|
||
ck "管理台列表含 search" yes "$(curl -s -b "$JAR" $API/api/admin/apix | has '"slug": *"search"')"
|
||
ck " ★条目 Key 只回显掩码(不回显真值)" no \
|
||
"$(curl -s -b "$JAR" $API/api/admin/apix | has 'ak_test_supplier_key')"
|
||
ck " 条目 base 回显" "http://127.0.0.1:18999" \
|
||
"$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c '
|
||
import sys, json
|
||
d = json.load(sys.stdin)["data"]
|
||
it = next((x for x in d["items"] if x["slug"] == "search"), None)
|
||
print(it.get("base") if it else "miss")')"
|
||
ck " 供应商名回显" "vendor-a" \
|
||
"$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c '
|
||
import sys, json
|
||
d = json.load(sys.stdin)["data"]
|
||
it = next((x for x in d["items"] if x["slug"] == "search"), None)
|
||
print(it.get("supplier") if it else "miss")')"
|
||
ck " 已停用条目 enabled=false" "False" "$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c '
|
||
import sys, json
|
||
d = json.load(sys.stdin)["data"]
|
||
it = next((x for x in d["items"] if x["slug"] == "offline"), None)
|
||
print(str(it.get("enabled")) if it else "miss")')"
|
||
ck "账号态接口只列已启用的(offline 不出现)" "3" \
|
||
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/apix | jqv data.count)"
|
||
|
||
ck "非法 slug → 400" 400 "$(code -b "$JAR" -X POST $API/api/admin/apix/save \
|
||
-H 'Content-Type: application/json' -d "$(jbody slug 'Bad Slug!' asset_id x)")"
|
||
ck "缺 asset_id → 400" 400 "$(code -b "$JAR" -X POST $API/api/admin/apix/save \
|
||
-H 'Content-Type: application/json' -d "$(jbody slug med asset_id '')")"
|
||
ck "★vendor 新建缺 Key → 400(vendor 必须 Key)" 400 \
|
||
"$(code -b "$JAR" -X POST $API/api/admin/apix/save \
|
||
-H 'Content-Type: application/json' -d "$(jbody slug vendorbare asset_id x base http://127.0.0.1:18999)")"
|
||
ck "★passthrough 新建可空 Key → 200(1.0.20 现场 bug 防回归)" 200 \
|
||
"$(code -b "$JAR" -X POST $API/api/admin/apix/save \
|
||
-H 'Content-Type: application/json' -d "$(jbody slug pt_nokey mode passthrough base http://127.0.0.1:18999)")"
|
||
ck "新建代理 med → 200" 200 "$(code -b "$JAR" -X POST $API/api/admin/apix/save \
|
||
-H 'Content-Type: application/json' -d "$(jbody slug med asset_id asset_26476ef1-4687-45 name 疾病百科 params kw base http://127.0.0.1:18999 key ak_test_supplier_key supplier vendor-a)")"
|
||
ck " ★新代理立即可用(写穿 shm)" 200 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/med?kw=感冒")"
|
||
ck " 靶机 body 带新 asset_id 与新参数" yes \
|
||
"$(up_field body | grep -q '"asset_id":"asset_26476ef1-4687-45"' && echo yes || echo no)"
|
||
ck " 参数名用的是 med 声明的 kw" yes "$(up_field body | grep -q '"kw":"' && echo yes || echo no)"
|
||
|
||
curl -s -b "$JAR" -X POST $API/api/admin/apix/save -H 'Content-Type: application/json' \
|
||
-d "$(jbody slug search asset_id searxng-search enabled false)" >/dev/null
|
||
ck "★停用后立即 404(写穿 shm,无回源兜底)" 404 \
|
||
"$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/search?q=1")"
|
||
curl -s -b "$JAR" -X POST $API/api/admin/apix/save -H 'Content-Type: application/json' \
|
||
-d "$(jbody slug search asset_id searxng-search enabled true)" >/dev/null
|
||
ck " 重新启用后又能调通" 200 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/search?q=1")"
|
||
curl -s -b "$JAR" -X POST $API/api/admin/apix/delete -H 'Content-Type: application/json' \
|
||
-d "$(jbody slug med)" >/dev/null
|
||
ck "删除后再调 → 404" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/med?kw=x")"
|
||
|
||
# 探测:走供应商的**免费**预估接口,不执行资产、不扣费
|
||
ck "★探测(免费预估)→ 透传 200" 200 "$(code -b "$JAR" -X POST \
|
||
"$API/api/admin/apix/test?slug=search" -H 'Content-Type: application/json' -d '{}')"
|
||
ck " 探测打的是 /api/v1/ai/estimate" "/api/v1/ai/estimate" "$(up_field path)"
|
||
ck " 探测也换了供应商真 Key" "ak_test_supplier_key" "$(up_hdr x-api-key)"
|
||
ck "★内部探测口外部直连 → 404" 404 \
|
||
"$(code -b "$JAR" -X POST "$API/_apix_probe/search" -d '{}')"
|
||
ck " 不存在的 slug 探测 → 404" 404 "$(code -b "$JAR" -X POST \
|
||
"$API/api/admin/apix/test?slug=ghost" -H 'Content-Type: application/json' -d '{}')"
|
||
ck "★用量统计按 api/<slug> 归组(与 LLM 上游天然分开)" yes \
|
||
"$(curl -s -b "$JAR" $API/api/admin/usage | has '"slot": *"api/search"')"
|
||
ck "★趋势含 api/<slug> 维度" yes "$(curl -s -b "$JAR" "$API/api/admin/trend?days=7" | has '"slot": *"api/search"')"
|
||
ck "★usage_hint 写入与回显(1.0.22 能力目录 hint)" yes \
|
||
"$(curl -s -b "$JAR" -X POST $API/api/admin/apix/save \
|
||
-H 'Content-Type: application/json' \
|
||
-d "$(jbody slug hintsrc asset_id asset_hint_26476ef1 base http://127.0.0.1:18999 key ak_test_supplier_key usage_hint 'POST /<tail> multipart fields=[file]; POST /<tail>/base64 json {image_base64:string}; Swagger: http://x/docs')" >/dev/null; \
|
||
curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c '
|
||
import sys, json
|
||
d = json.load(sys.stdin)
|
||
it = next((x for x in d["data"]["items"] if x["slug"] == "hintsrc"), None)
|
||
print("yes" if it and "POST /<tail>" in it.get("usage_hint","") else "no")')"
|
||
# 清理
|
||
curl -s -b "$JAR" -X POST $API/api/admin/apix/delete -H 'Content-Type: application/json' \
|
||
-d "$(jbody slug hintsrc)" >/dev/null
|
||
|
||
echo
|
||
echo "【11.6】★ API 代理透传模式(/apixp/<slug>/<尾部> → 内部 REST 服务)"
|
||
# 与 vendor 模式同一把 sk_ Key + 配额 + 用量统计,但**不再由网关拼体**——
|
||
# 客户端带什么方法/body/Content-Type 原样给上游。判据同样是看靶机实际收到的
|
||
# path / method / 头 / body,只看网关 200 证明不了「原样透传」。
|
||
ck "无 Key → 401" 401 "$(code "$GW/apixp/ocr/ocr")"
|
||
ck "坏 Key → 401" 401 "$(code -H 'X-API-Key: sk_nope' "$GW/apixp/ocr/ocr")"
|
||
ck "不存在的 slug → 404" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apixp/ghost/foo")"
|
||
ck "★已停用的 slug → 404" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apixp/offline/foo")"
|
||
ck "★passthrough 空 Key 也放行(内部服务免鉴权)" 200 \
|
||
"$(code -H "X-API-Key: $ALICE_KEY" -X POST "$GW/apixp/ocr/ocr" -H 'Content-Type: application/json' -d '{"image_base64":"AAAA"}')"
|
||
ck " 靶机 path = 路径尾部 /ocr(base + 尾部)" "/ocr" "$(up_field path)"
|
||
ck " ★方法原样透传(POST 没被改)" "POST" "$(up_field method)"
|
||
ck " ★请求体原样透传" yes "$(up_field body | grep -q 'image_base64' && echo yes || echo no)"
|
||
ck " ★空 Key 不下发 X-API-Key(上游收不到该头)" "" "$(up_hdr x-api-key)"
|
||
ck " ★下行 sk_ Key 没被带出去" "" "$(up_hdr authorization)"
|
||
# 路径尾部可多级:/apixp/ocr/ocr/base64 → 上游 /ocr/base64
|
||
ck "★多级尾部透传 → 200" 200 \
|
||
"$(code -H "X-API-Key: $ALICE_KEY" -X POST "$GW/apixp/ocr/ocr/base64" -H 'Content-Type: application/json' -d '{"image_base64":"BBBB"}')"
|
||
ck " 靶机 path = /ocr/base64" "/ocr/base64" "$(up_field path)"
|
||
# query 也随尾部一起透传(变量式 proxy_pass 不自动追加,由模块拼进 $apixp_target)
|
||
ck "★query 随尾部透传 → 200" 200 \
|
||
"$(code -H "X-API-Key: $ALICE_KEY" -X POST "$GW/apixp/ocr/ocr/url?src=http://example.com/x" -H 'Content-Type: application/json' -d '{"x":1}')"
|
||
ck " 靶机 path 含 query" yes "$(up_field path | grep -q '/ocr/url?src=' && echo yes || echo no)"
|
||
# 带 Key 的 passthrough:X-API-Key 必须被下发
|
||
ck "★带 Key 的 passthrough → 200" 200 \
|
||
"$(code -H "X-API-Key: $ALICE_KEY" -X POST "$GW/apixp/ocrk/health" -H 'Content-Type: application/json' -d '{}')"
|
||
ck " ★X-API-Key 被下发到上游" "ak_internal_ocr_key" "$(up_hdr x-api-key)"
|
||
ck " ★path 尾部 = /health" "/health" "$(up_field path)"
|
||
# mode 与路径前缀必须一致:vendor slug 不能走 /apixp/,passthrough slug 不能走 /apix/
|
||
ck "★vendor slug 走 /apixp/ → 404(路径与模式不匹配)" 404 \
|
||
"$(code -H "X-API-Key: $ALICE_KEY" "$GW/apixp/search?q=1")"
|
||
ck "★passthrough slug 走 /apix/ → 404(路径与模式不匹配)" 404 \
|
||
"$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/ocr")"
|
||
# 管理台:mode 字段回显 + 调用地址用 /apixp/ 前缀
|
||
ck "管理台列表含 ocr 且 mode=passthrough" yes "$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c '
|
||
import sys, json
|
||
d = json.load(sys.stdin)["data"]
|
||
it = next((x for x in d["items"] if x["slug"] == "ocr"), None)
|
||
print("yes" if it and it.get("mode") == "passthrough" else "no")')"
|
||
ck " ★ocr 调用地址用 /apixp/ 前缀" yes "$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c '
|
||
import sys, json
|
||
d = json.load(sys.stdin)["data"]
|
||
it = next((x for x in d["items"] if x["slug"] == "ocr"), None)
|
||
print("yes" if it and "/apixp/ocr" in it.get("url","") else "no")')"
|
||
# 探测:passthrough 走 GET 打上游根地址验连通性
|
||
ck "★探测(passthrough)→ 透传 200" 200 "$(code -b "$JAR" -X POST \
|
||
"$API/api/admin/apix/test?slug=ocr" -H 'Content-Type: application/json' -d '{}')"
|
||
ck " 探测是 GET(非 POST 预估)" "GET" "$(up_field method)"
|
||
ck " 探测打上游根 /" "/" "$(up_field path)"
|
||
ck " 探测也空 Key(不下发 X-API-Key)" "" "$(up_hdr x-api-key)"
|
||
ck "★内部透传探测口外部直连 → 404" 404 \
|
||
"$(code -b "$JAR" -X POST "$API/_apixp_probe/ocr" -d '{}')"
|
||
ck " 不存在的 slug 透传探测 → 404" 404 "$(code -b "$JAR" -X POST \
|
||
"$API/api/admin/apix/test?slug=ghost" -H 'Content-Type: application/json' -d '{}')"
|
||
ck "★用量统计按 api/<slug> 归组(passthrough 同样计)" yes \
|
||
"$(curl -s -b "$JAR" $API/api/admin/usage | has '"slot": *"api/ocr"')"
|
||
|
||
echo
|
||
echo "【12】★ 上游健康标记(失效自动缓存 + 模型发现附可用地址/状态)"
|
||
# 健康缓存由 log 阶段的**真实转发**写入(uh:<slot>)——「测试」按钮走同一条
|
||
# proxy_pass 链路,探测结果也天然更新缓存,无需第二套标记逻辑。
|
||
# 语义:上游回了头 = 活着(4xx 是它对请求内容的意见,不是机器死了);
|
||
# 5xx / 连不上(502) / 等头超时(504) = 失效;没被任何请求打过 = unknown。
|
||
# 缓存只用于展示(列表标红 / 模型发现附状态),转发路径不读它。
|
||
up_field_of() { # $1=slot $2=字段 → 取上游列表里该 slot 的字段值
|
||
curl -s -b "$JAR" $API/api/admin/upstreams | "$PY" -c '
|
||
import sys, json
|
||
d = json.load(sys.stdin)
|
||
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
|
||
u = next((x for x in items if x.get("slot") == sys.argv[1]), None)
|
||
v = u.get(sys.argv[2]) if u else "miss"
|
||
print("miss" if v is None else v)
|
||
' "$1" "$2"
|
||
}
|
||
xm_status() { # /v1/models 扩展块里某 slot 的健康状态
|
||
curl -s -H "Authorization: Bearer $HKEY" $GW/v1/models | "$PY" -c '
|
||
import sys, json
|
||
d = json.load(sys.stdin)
|
||
u = next((x for x in d.get("x_ipass_upstreams", []) if x.get("slot") == sys.argv[1]), {})
|
||
print(u.get("status", "miss"))
|
||
' "$1"
|
||
}
|
||
|
||
ck "创建临时上游 health1 → 200" 200 \
|
||
"$(code -b "$JAR" -X POST $API/api/admin/upstreams/save -H 'Content-Type: application/json' \
|
||
-d '{"slot":"health1","base":"http://127.0.0.1:18999","auth":"Bearer sk-H1","models":["m1"]}')"
|
||
ck "★初始状态 = unknown(没被任何真实请求打过)" unknown "$(up_field_of health1 status)"
|
||
|
||
# 临时账号(空白名单 = 不限)换一把干净 Key 来打靶机注入的 500
|
||
curl -s -b "$JAR" -X POST $API/api/admin/users/create -H 'Content-Type: application/json' \
|
||
-d "$(jbody username healthprobe password 123456 display_name hp)" >/dev/null
|
||
HKEY=$(curl -s -b "$JAR" $API/api/admin/users | "$PY" -c '
|
||
import sys, json
|
||
d = json.load(sys.stdin)
|
||
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
|
||
print(next((u["api_key"] for u in items if u.get("username") == "healthprobe"), ""))
|
||
')
|
||
|
||
BOOM='{"model":"m1","max_tokens":1,"boom":true,"messages":[{"role":"user","content":"x"}]}'
|
||
ck "boom 请求 → 靶机注入的 500 原样透传给客户端" 500 \
|
||
"$(code -H "Authorization: Bearer $HKEY" -X POST $GW/health1/v1/chat/completions \
|
||
-H 'Content-Type: application/json' -d "$BOOM")"
|
||
sleep 0.3
|
||
ck "★上游 500 → health1 自动标记为 failed(无需人工探测)" failed "$(up_field_of health1 status)"
|
||
ck " 缓存里记录的状态码 = 500" 500 "$(up_field_of health1 status_code)"
|
||
ck "★/v1/models 顶层附 x_ipass_upstreams(地址 + 模型 + 状态)" yes \
|
||
"$(curl -s -H "Authorization: Bearer $HKEY" $GW/v1/models | has '"x_ipass_upstreams"')"
|
||
ck " 其中 health1 同样标 failed" failed "$(xm_status health1)"
|
||
ck " 给出 anthropic 形式的 base_url" yes \
|
||
"$(curl -s -H "Authorization: Bearer $HKEY" $GW/v1/models | has '"anthropic_base_url"')"
|
||
ck "★账号态 /api/v1/upstreams 也带状态与双协议地址" yes \
|
||
"$(curl -s -H "X-API-Key: $HKEY" $API/api/v1/upstreams | has '"anthropic_base_url"')"
|
||
|
||
OKBODY='{"model":"m1","max_tokens":1,"messages":[{"role":"user","content":"ping"}]}'
|
||
ck "恢复:正常请求 → 200" 200 \
|
||
"$(code -H "Authorization: Bearer $HKEY" -X POST $GW/health1/v1/chat/completions \
|
||
-H 'Content-Type: application/json' -d "$OKBODY")"
|
||
sleep 0.3
|
||
ck "★下次成功转发自动转回 ok(自愈,无需清缓存)" ok "$(up_field_of health1 status)"
|
||
|
||
# 清理临时数据
|
||
curl -s -b "$JAR" -X POST $API/api/admin/upstreams/delete -H 'Content-Type: application/json' \
|
||
-d "$(jbody slot health1)" >/dev/null
|
||
ck "删除后列表不再出现 health1" miss "$(up_field_of health1 status)"
|
||
curl -s -b "$JAR" -X POST $API/api/admin/users/delete -H 'Content-Type: application/json' \
|
||
-d "$(jbody username healthprobe)" >/dev/null
|
||
|
||
echo
|
||
echo "【13】对话审计(meta.jsonl:账号/上游/状态 + 智能 excerpt + 全文 hash)"
|
||
AUDIT=$RUN/audit
|
||
sleep 0.3
|
||
# 1.0.18 起不再把请求体复制进审计目录(旧版每请求 700 KB+),只写一行 meta。
|
||
# 1.0.27 起 excerpt 改为智能摘录:短请求保留原文;长请求解析 JSON 提取
|
||
# model + 最后一个 user content(按字符截),跳过 context_management / tools
|
||
# 等结构噪声,输出 {"model":"…","content":"…"}。
|
||
meta_n() { cat "$AUDIT"/*/meta.jsonl 2>/dev/null | awk 'NF{n++} END{print n+0}'; }
|
||
body_files() { find "$AUDIT" -name '*.json' 2>/dev/null | wc -l | tr -d ' '; }
|
||
BODY_BASE=$(body_files)
|
||
N1=$(meta_n)
|
||
ck "转发的请求都落了 meta 行" yes "$([ "$N1" -ge 3 ] && echo yes || echo no)"
|
||
ck "meta 行记录账号/上游/状态" yes \
|
||
"$(grep -q '"user":"alice"' "$AUDIT"/*/meta.jsonl 2>/dev/null && grep -q '"slot":"openai"' "$AUDIT"/*/meta.jsonl && grep -q '"status":200' "$AUDIT"/*/meta.jsonl && echo yes || echo no)"
|
||
|
||
# ★ 长请求体:1.0.27 智能摘录优先把 user content 摘出,跳过 context_management / tools。
|
||
# 这条断言是 1.0.18 的核心取舍(元数据 + 摘要),现在升级为「摘要含用户内容、
|
||
# 不含结构噪声」。
|
||
# 测试用 400 字节 + 'AAA…' 占位 + 'ZZ_TAIL_MARK' 尾巴,确保 body_bytes > 100 走智能路径。
|
||
PAD=$(printf 'A%.0s' $(seq 1 400))
|
||
LBODY="{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":\"${PAD}ZZ_TAIL_MARK\"}]}"
|
||
code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
|
||
-H 'Content-Type: application/json' -d "$LBODY" >/dev/null
|
||
sleep 0.3
|
||
|
||
# 取这一笔长请求的 excerpt(按 body_bytes > 400 + user=alice + status=200 唯一定位)
|
||
EXCERPT=$("$PY" -c "
|
||
import json, sys
|
||
for f in sys.argv[1:]:
|
||
for l in open(f, encoding='utf-8'):
|
||
d = json.loads(l)
|
||
if d.get('user') == 'alice' and d.get('status') == 200 and d.get('body_bytes', 0) > 400:
|
||
print(d.get('excerpt',''))
|
||
sys.exit(0)
|
||
" "$AUDIT"/*/meta.jsonl 2>/dev/null)
|
||
|
||
# ★ 智能摘录:长请求 excerpt 是 {"model":"…","content":"<user 内容>"} 结构
|
||
# - 必须含 model 字段(保留诊断价值)
|
||
# - 必须含 user content 关键片段(跳过结构噪声,摘出真正想问的)
|
||
# - 不应含 'ZZ_TAIL_MARK'(尾部内容仍未落盘)
|
||
# - 不应含 AAA 占位(这是 user content,智能摘录**全部** user 内容到 content 字段;
|
||
# 原 1.0.18 前 100 字节全是结构噪声反而看不到 AAA)
|
||
ck "★智能 excerpt 含 model 字段" yes \
|
||
"$(printf '%s' "$EXCERPT" | grep -q '"model":"gpt-4o"' && echo yes || echo no)"
|
||
ck "★智能 excerpt 含 user content(AAA 占位)" yes \
|
||
"$(printf '%s' "$EXCERPT" | grep -q 'AAAA' && echo yes || echo no)"
|
||
ck "★智能 excerpt 不含尾部标记(脱敏与不落盘)" yes \
|
||
"$(if printf '%s' "$EXCERPT" | grep -q 'ZZ_TAIL_MARK'; then echo no; else echo yes; fi)"
|
||
ck "★智能 excerpt 不含结构噪声(context_management / tools)" yes \
|
||
"$(if printf '%s' "$EXCERPT" | grep -q 'context_management\|\"tools\"'; then echo no; else echo yes; fi)"
|
||
|
||
ck "★body_bytes = 请求体真实长度(智能摘录不影响)" "$(printf '%s' "$LBODY" | wc -c | tr -d ' ')" \
|
||
"$("$PY" -c "
|
||
import json, sys
|
||
n = ''
|
||
for f in sys.argv[1:]:
|
||
for l in open(f, encoding='utf-8'):
|
||
d = json.loads(l)
|
||
if d.get('user') == 'alice' and d.get('body_bytes', 0) > 400:
|
||
n = d['body_bytes']
|
||
print(n)
|
||
" "$AUDIT"/*/meta.jsonl 2>/dev/null)"
|
||
ck "★body_hash 是全文指纹(16 hex,智能摘录不影响)" yes "$("$PY" -c "
|
||
import json, re, sys
|
||
res = 'no'
|
||
for f in sys.argv[1:]:
|
||
for l in open(f, encoding='utf-8'):
|
||
d = json.loads(l)
|
||
if d.get('user') == 'alice' and d.get('body_bytes', 0) > 400:
|
||
res = 'yes' if re.fullmatch(r'[0-9a-f]{16}', d.get('body_hash', '')) else 'no'
|
||
print(res)
|
||
" "$AUDIT"/*/meta.jsonl 2>/dev/null)"
|
||
ck "★不再生成请求体副本(本轮新增 *.json = 0)" 0 "$(( $(body_files) - BODY_BASE ))"
|
||
ck "★审计目录里搜不到请求体尾部" yes \
|
||
"$(if grep -q 'ZZ_TAIL_MARK' "$AUDIT"/*/meta.jsonl 2>/dev/null; then echo no; else echo yes; fi)"
|
||
|
||
# ★ 短请求体(≤ 100B):excerpt 应保留原文(含 model + messages 结构)
|
||
SBODY="{\"model\":\"gpt-4o\",\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}"
|
||
code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
|
||
-H 'Content-Type: application/json' -d "$SBODY" >/dev/null
|
||
sleep 0.3
|
||
# ⚠️ meta.jsonl 里 excerpt 是 JSON 字符串,其内的 "messages" 被转义成 \"messages\"——
|
||
# 直接 grep '"messages"' 匹不到;用不带引号的 'messages' 即可(包含 messages 数组外层
|
||
# + content 里的 messages 都不会误匹,下面 JSON 解析做严格断言)。
|
||
ck "★短请求(≤100B)excerpt 保留原文(含 messages 结构)" yes \
|
||
"$("$PY" -c "
|
||
import json, sys
|
||
res = 'no'
|
||
for f in sys.argv[1:]:
|
||
for l in open(f, encoding='utf-8'):
|
||
d = json.loads(l)
|
||
if d.get('user') != 'alice' or d.get('status') != 200 or d.get('body_bytes', 0) > 100:
|
||
continue
|
||
# 短请求 excerpt 应该是原文(含 messages 数组);智能摘录只摘 model + content
|
||
try:
|
||
ex = json.loads(d.get('excerpt',''))
|
||
if 'messages' in ex: res = 'yes'
|
||
except Exception: pass
|
||
print(res)
|
||
" "$AUDIT"/*/meta.jsonl 2>/dev/null)"
|
||
|
||
N2=$(meta_n)
|
||
code -H "Authorization: Bearer sk_deadbeef" -X POST $GW/openai/v1/chat/completions \
|
||
-H 'Content-Type: application/json' -d "$BODY" >/dev/null
|
||
sleep 0.3
|
||
ck "被拒请求(401)不入档" 0 "$(( $(meta_n) - N2 ))"
|
||
|
||
echo "【14】token 计量(body filter 旁路抓 usage → utt:/utms:)"
|
||
# 非 stream usage 固定 7/3;带 usage_probe 的流式在 SSE 尾部给 11/22
|
||
curl -s -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
|
||
-H 'Content-Type: application/json' -d "$BODY" >/dev/null
|
||
curl -s -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
|
||
-H 'Content-Type: application/json' -d '{"model":"gpt-4o","stream":true,"usage_probe":true}' >/dev/null
|
||
sleep 0.3
|
||
TOK=$(curl -s -b "$JAR" $API/api/admin/usage)
|
||
|
||
echo "【多账号路由】粘性绑定 + 额度耗尽自动切换"
|
||
# 用 admin(白名单为空=不限;本套件中途会删除 bob,故不复用 bob)
|
||
ck "admin 基础鉴权(openai,白名单不限)→ 200" 200 "$(code -H "Authorization: Bearer $ADMIN_KEY" -X POST $GW/openai/v1/chat/completions -H 'Content-Type: application/json' -d '{"model":"gpt-4o","messages":[{"role":"user","content":"hi"}],"max_tokens":8}')"
|
||
acct_of() { curl -s -D - -o /dev/null -X POST "$GW/$1/v1/chat/completions" \
|
||
-H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \
|
||
-d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}' \
|
||
| grep -i '^X-Upstream-Acct' | tr -d '\r' | awk '{print $2}'; }
|
||
code_of() { curl -s -o /dev/null -w '%{http_code}' -X POST "$GW/$1/v1/chat/completions" \
|
||
-H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \
|
||
-d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}'; }
|
||
|
||
ck "sticky 首次请求命中 acct=0" 0 "$(acct_of sticky sticky-model)"
|
||
ck "sticky 首次请求 200" 200 "$(code_of sticky sticky-model)"
|
||
ck "sticky 粘性:第 2 次仍 acct=0" 0 "$(acct_of sticky sticky-model)"
|
||
ck "sticky 粘性:第 3 次仍 acct=0" 0 "$(acct_of sticky sticky-model)"
|
||
|
||
# 切换:multi slot accounts[0]=bad(命中假上游 UPSTREAM_FAIL_AUTH → 401+marker)→ 自动切 acct=1
|
||
mh() { curl -s -D - -o /dev/null -X POST "$GW/$1/v1/chat/completions" \
|
||
-H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \
|
||
-d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}' \
|
||
| grep -iE '^HTTP/|^X-Upstream-Acct' | tr -d '\r'; }
|
||
M1=$(mh multi multi-model)
|
||
ck "multi 初次请求打到 acct=0(bad)" 0 "$(echo "$M1" | awk '/X-Upstream-Acct/{print $2}')"
|
||
ck "multi 初次请求 401(触发冷却)" 401 "$(echo "$M1" | awk '/^HTTP/{print $2}')"
|
||
M2=$(mh multi multi-model)
|
||
ck "multi 冷却中切到 acct=1" 1 "$(echo "$M2" | awk '/X-Upstream-Acct/{print $2}')"
|
||
ck "multi 切到 acct=1 后 200" 200 "$(echo "$M2" | awk '/^HTTP/{print $2}')"
|
||
M3=$(mh multi multi-model)
|
||
ck "multi 冷却期间保持 acct=1(不回退坏账号)" 1 "$(echo "$M3" | awk '/X-Upstream-Acct/{print $2}')"
|
||
# 靶机验证:再发一次确认切换后的账号,读靶机实际收到的 auth(证明下行 Key 真被换成对应账号 Key)
|
||
M4=$(mh multi multi-model)
|
||
ck "multi 切换后下游 Key 真换成 acct-good" "Bearer acct-good" "$(up_hdr Authorization)"
|
||
|
||
echo "【探测恢复闭环】退避冷却 + 真实请求探测 + 绑定流量保护"
|
||
# fixture `probe`:#0=probe-flip(每账号只失败第一次)、#1=probe-dead(永远耗尽)。
|
||
# conf 里 ipass_probe_min 1 → 首轮退避 1s,翻倍 2s;冷却硬顶 30min。
|
||
# 剧本逐拍(时间锚 = 各次标记后的 next_probe):
|
||
ph() { curl -s -D - -o /dev/null -X POST "$GW/probe/v1/chat/completions" \
|
||
-H "Authorization: Bearer $1" -H 'Content-Type: application/json' \
|
||
-d '{"model":"probe-model","messages":[{"role":"user","content":"hi"}],"max_tokens":8}' \
|
||
| grep -iE '^HTTP/|^X-Upstream-Acct' | tr -d '\r'; }
|
||
p_code() { echo "$1" | awk '/^HTTP/{print $2}'; }
|
||
p_acct() { echo "$1" | awk '/X-Upstream-Acct/{print $2}'; }
|
||
|
||
P1=$(ph "$PROBE1_KEY")
|
||
ck "probe 冷分配落 #0(FNV 钉死)且首触 401" 0 "$(p_acct "$P1")"
|
||
ck "probe #0 首触 401 → 进冷却" 401 "$(p_code "$P1")"
|
||
P2=$(ph "$PROBE1_KEY")
|
||
ck "probe 绑定 #0 冷却中 → 退到 #1(也 401,被标记)" 401 "$(p_code "$P2")"
|
||
ck "probe 第二落点 #1" 1 "$(p_acct "$P2")"
|
||
sleep 2.0 # 首轮退避 1s 到期(#0/#1 标记时刻差 <0.5s,2s 留足余量;#0 的 next_probe 更早)
|
||
P3=$(ph "$PROBE1_KEY")
|
||
ck "★probe 全冷却下最早到期的 #0 被真实请求探测 → 200 回队" 200 "$(p_code "$P3")"
|
||
ck "★probe 探测落在 #0" 0 "$(p_acct "$P3")"
|
||
P4=$(ph "$PROBE1_KEY")
|
||
ck "★probe 绑定已复活的 #0 → 不被 #1 探测劫持(绑定流量保护)" 0 "$(p_acct "$P4")"
|
||
ck "probe 绑定保护请求正常 200" 200 "$(p_code "$P4")"
|
||
P5=$(ph "$PROBE2_KEY")
|
||
ck "★probe 无绑定流量被劫持去探测已到期 #1 → 吃真实 401(代价被退避封顶)" 401 "$(p_code "$P5")"
|
||
ck "probe 劫持落在 #1" 1 "$(p_acct "$P5")"
|
||
P6=$(ph "$PROBE2_KEY")
|
||
ck "probe 探测失败窗口即关闭(退避翻倍 2s)→ 回健康 #0" 0 "$(p_acct "$P6")"
|
||
sleep 3.0 # #1 第二轮退避 2s 到期(同样留余量)
|
||
P7=$(ph "$PROBE3_KEY")
|
||
ck "★probe #1 再次到期仍被探测(指数退避、不撒手)" 401 "$(p_code "$P7")"
|
||
ck "probe 第二轮探测仍落 #1" 1 "$(p_acct "$P7")"
|
||
|
||
UPS2=$(curl -s -b "$JAR" $API/api/admin/upstreams)
|
||
acct_view() { # $1=快照变量名内容 $2=slot $3=idx $4=路径 → 原始值(bool 小写)
|
||
printf '%s' "$1" | "$PY" -c '
|
||
import sys, json
|
||
slot, idx, path = sys.argv[2], int(sys.argv[3]), sys.argv[4]
|
||
d = json.loads(sys.argv[1])
|
||
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
|
||
for u in items:
|
||
if u.get("slot") == slot:
|
||
for a in u.get("accounts", []):
|
||
if a.get("idx") == idx:
|
||
cur = a
|
||
for k in path.split("."):
|
||
cur = cur.get(k) if isinstance(cur, dict) else None
|
||
print(str(cur).lower()); sys.exit(0)
|
||
print("missing")
|
||
' "$1" "$2" "$3" "$4"; }
|
||
ck "★probe 投影:#0 冷却已解除(自动回队)" false "$(acct_view "$UPS2" probe 0 cooled)"
|
||
ck "★probe 投影:#1 仍在冷却(退避持续)" true "$(acct_view "$UPS2" probe 1 cooled)"
|
||
ck "probe #0 至少记 1 次耗尽" yes "$([ "$(acct_view "$UPS2" probe 0 load.exhausts)" -ge 1 ] && echo yes || echo no)"
|
||
ck "probe #1 至少记 3 次耗尽(标记+两轮探测)" yes "$([ "$(acct_view "$UPS2" probe 1 load.exhausts)" -ge 3 ] && echo yes || echo no)"
|
||
|
||
# ---- 管理台「测试」按钮不得写路由状态(线上事故回归)----
|
||
# 事故复现:面板测试走 /_upstream_test,账号由 pick 随机选;上游一旦回 401+marker,
|
||
# 旧逻辑会把"管理员手抖点了一下测试"当成"这个账号额度耗尽",冷却一张好钥匙,
|
||
# 恢复要走完一轮退避/探测。fixture `probetest`:单账号 Bearer probe-dead
|
||
# (上游靶机识别后永远回 401+`status_code:1004` 标记),exhaust_status=[401] 让
|
||
# status 通道也命中——彻底钉死「测试按钮不写冷却」这条。
|
||
TB=$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=probetest" -H 'Content-Type: application/json' -d "$PING")
|
||
ck "probetest admin test → 上游真实 401 透传" 401 "$TB"
|
||
UPS_PT=$(curl -s -b "$JAR" $API/api/admin/upstreams)
|
||
ck "★admin test 后账号未进冷却(冷却不被测试按钮污染)" false "$(acct_view "$UPS_PT" probetest 0 cooled)"
|
||
ck "★admin test 不记录耗尽计数" 0 "$(acct_view "$UPS_PT" probetest 0 load.exhausts)"
|
||
# 真实流量同形态 → 标记通道必须照常工作(不能为避 bug 把真标记也关掉)
|
||
PT1=$(mh probetest probetest-model)
|
||
ck "probetest 真实流量 → 401(标记通道触发)" 401 "$(echo "$PT1" | awk '/^HTTP/{print $2}')"
|
||
UPS_PT2=$(curl -s -b "$JAR" $API/api/admin/upstreams)
|
||
ck "★真实流量 401+marker → 进冷却(标记通道保留)" true "$(acct_view "$UPS_PT2" probetest 0 cooled)"
|
||
ck "★真实流量耗尽计数 ≥1" yes "$([ "$(acct_view "$UPS_PT2" probetest 0 load.exhausts)" -ge 1 ] && echo yes || echo no)"
|
||
|
||
# ---- auth 规范化(normalize_auth)----
|
||
# fixture `normalize` 账号 auth 写成裸 key(`sk-NORM-NO-SCHEME`,无 `Bearer `),
|
||
# 期望:转发时网关自动补 `Bearer `,靶机实收完整头值(不会因缺 scheme 触发 1004)。
|
||
N1=$(mh normalize normalize-model)
|
||
ck "normalize 初次 200(裸 key 被补 Bearer 后 200)" 200 "$(echo "$N1" | awk '/^HTTP/{print $2}')"
|
||
ck "normalize 靶机实收 Bearer sk-NORM-NO-SCHEME" "Bearer sk-NORM-NO-SCHEME" "$(up_hdr Authorization)"
|
||
|
||
# ---- 账号级负载统计(管理台「密钥 / 负载」列的数据源)----
|
||
# 每次真实转发在 log 阶段落到「本次选中的那个账号」的日桶 uda:<slot>:<idx>:<day>;
|
||
# 额度耗尽另记 uex:。用同一份 /api/admin/upstreams 投影断言负载分布是否正确。
|
||
UPS=$(curl -s -b "$JAR" $API/api/admin/upstreams)
|
||
acct_num() { # $1=slot $2=idx $3=度量路径(如 load.requests)→ 取账号级数值
|
||
printf '%s' "$UPS" | "$PY" -c '
|
||
import sys, json
|
||
slot, idx, path = sys.argv[1], int(sys.argv[2]), sys.argv[3]
|
||
d = json.load(sys.stdin)
|
||
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
|
||
for u in items:
|
||
if u.get("slot") == slot:
|
||
for a in u.get("accounts", []):
|
||
if a.get("idx") == idx:
|
||
cur = a
|
||
for k in path.split("."):
|
||
cur = cur.get(k) if isinstance(cur, dict) else None
|
||
print(cur if cur is not None else 0)
|
||
sys.exit(0)
|
||
print(0)
|
||
' "$1" "$2" "$3"
|
||
}
|
||
ck "★sticky 账号#0 记到今日请求(负载分布生效)" yes "$([ "$(acct_num sticky 0 load.requests)" -ge 3 ] && echo yes || echo no)"
|
||
ck "★sticky 账号#1 未被分配(0 请求,粘性=集中在一把 Key)" 0 "$(acct_num sticky 1 load.requests)"
|
||
ck "★multi 账号#0 记到 1 次额度耗尽" yes "$([ "$(acct_num multi 0 load.exhausts)" -ge 1 ] && echo yes || echo no)"
|
||
ck "★multi 账号#1 记到今日请求(切换后负载落在新 Key)" yes "$([ "$(acct_num multi 1 load.requests)" -ge 3 ] && echo yes || echo no)"
|
||
ck "账号级趋势序列含 14 天" 14 \
|
||
"$(printf '%s' "$UPS" | "$PY" -c '
|
||
import sys, json
|
||
d = json.load(sys.stdin)
|
||
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
|
||
for u in items:
|
||
if u.get("slot") == "sticky":
|
||
t = u.get("account_trend") or [{}]
|
||
print(len(t[0].get("requests") or []))
|
||
break
|
||
else:
|
||
print(0)
|
||
')"
|
||
# 日期标签与趋势序列逐位对齐(图表悬停提示用;由后端算,避免前后端各自推导时区)
|
||
ck "账号趋势带 14 个日期标签(与序列对齐)" 14 \
|
||
"$(printf '%s' "$UPS" | "$PY" -c '
|
||
import sys, json
|
||
d = json.load(sys.stdin)
|
||
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
|
||
for u in items:
|
||
if u.get("slot") == "sticky":
|
||
print(len(u.get("trend_labels") or []))
|
||
break
|
||
else:
|
||
print(0)
|
||
')"
|
||
|
||
# ---- 趋势(日桶)----
|
||
# 口径与请求级用量一致:只统计真正转发出去的请求与上游 usage 里的 token
|
||
TR=$(curl -s -b "$JAR" "$API/api/admin/trend?days=7")
|
||
ck "趋势接口 → 200" 200 "$(code -b "$JAR" "$API/api/admin/trend?days=7")"
|
||
ck "趋势默认含 14 天" 14 "$(printf '%s' "$(curl -s -b "$JAR" $API/api/admin/trend)" | "$PY" -c 'import json,sys;print(len(json.load(sys.stdin)["data"]["labels"]))' 2>/dev/null)"
|
||
ck "趋势天数上限 90" 90 "$(printf '%s' "$(curl -s -b "$JAR" "$API/api/admin/trend?days=99999")" | "$PY" -c 'import json,sys;print(json.load(sys.stdin)["data"]["days"])' 2>/dev/null)"
|
||
ck "7 天区间 → 7 个标签" 7 "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;print(len(json.load(sys.stdin)["data"]["labels"]))' 2>/dev/null)"
|
||
ck "今日请求已进日桶" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if sum(d["total"]["requests"])>0 else "no")' 2>/dev/null)"
|
||
ck "今日 token 已进日桶" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if sum(d["total"]["in"])+sum(d["total"]["out"])>0 else "no")' 2>/dev/null)"
|
||
ck "趋势含按账号维度" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if any(u["username"]=="alice" for u in d["users"]) else "no")' 2>/dev/null)"
|
||
ck "趋势含按 API 组维度" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if any(not s["empty"] for s in d["slots"]) else "no")' 2>/dev/null)"
|
||
tok_ok() { "$PY" -c "
|
||
import json, sys
|
||
d = json.loads(sys.argv[1])['data']
|
||
ok_u = ok_s = False
|
||
for u in d.get('users', []):
|
||
if u['username'] == 'alice':
|
||
t = u.get('tokens', {})
|
||
ok_u = t.get('in_total', 0) >= 18 and t.get('out_total', 0) >= 25
|
||
for m in u.get('models', []):
|
||
if m['slot'] == 'openai':
|
||
ok_u = ok_u and m.get('tokens', {}).get('in_total', 0) >= 18
|
||
for s in d.get('models', []):
|
||
if s['slot'] == 'openai':
|
||
ts = s.get('tokens', {})
|
||
ok_s = ts.get('in_total', 0) >= 18 and ts.get('out_total', 0) >= 25
|
||
print('yes' if ok_u and ok_s else 'no')
|
||
" "$1"; }
|
||
ck "usage API 出现 token 维度(账号/上游明细/上游聚合)" yes "$(tok_ok "$TOK")"
|
||
ck "审计 meta 行记录 token" yes "$("$PY" -c "
|
||
import json, sys
|
||
hit = False
|
||
for f in sys.argv[1:]:
|
||
for l in open(f, encoding='utf-8'):
|
||
d = json.loads(l)
|
||
if d.get('user') == 'alice' and d.get('tin', 0) > 0:
|
||
hit = True
|
||
print('yes' if hit else 'no')
|
||
" "$AUDIT"/*/meta.jsonl 2>/dev/null)"
|
||
|
||
echo
|
||
echo "【15】★ 运行期统计落盘 / 重启恢复(发版不再清零用量与趋势)"
|
||
# 背景:用量/趋势只活在共享内存里,进程退出即消失 → 1.0.18 之前每次发版统计归零。
|
||
# 现在:退出钩子(exit_process)落盘 + 记账路径按间隔节流落盘;启动时 put-if-absent 恢复。
|
||
# 本段刻意放在**最后**:它要停掉再拉起 nginx,之后的断言环境会变(会话丢、共享内存重建)。
|
||
STATS=$RUN/data/llm-stats.json
|
||
NBI=${NGINX_BIN:-}
|
||
ck "本套件把落盘间隔压到 1 秒(nginx.conf.in)" 1 \
|
||
"$(curl -s -b "$JAR" "$API/api/admin/stats/persist" | jqv data.interval_secs)"
|
||
|
||
# 跑几笔请求,让统计有东西可落
|
||
for _ in 1 2 3; do
|
||
code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
|
||
-H 'Content-Type: application/json' -d "$BODY" >/dev/null
|
||
done
|
||
sleep 1.5 # > 间隔 1 秒:保证周期性落盘已经跑过一次
|
||
|
||
ck "★周期落盘已产出快照文件" yes "$([ -s "$STATS" ] && echo yes || echo no)"
|
||
# 落盘时刻必须从共享内存读(`spd:last`),而不是各 worker 自己的静态量 —— 否则
|
||
# 「日志明明落了 N 次、状态页却说本进程还没落过」(1.0.18 首次线上验证实测)。
|
||
# 本套件开 2 个 worker,正是能暴露该分歧的拓扑;断言看的是「任意 worker 都报得出」。
|
||
ck "★状态页报告的上次落盘时刻非空(跨 worker 一致)" yes \
|
||
"$(curl -s -b "$JAR" "$API/api/admin/stats/persist" | "$PY" -c "
|
||
import json, sys
|
||
d = json.load(sys.stdin)['data']
|
||
print('yes' if d.get('last_dump_at', 0) > 0 and d.get('last_dump_text') else 'no')
|
||
" 2>/dev/null)"
|
||
ck "★快照含账号级用量键(ust:alice)" yes \
|
||
"$("$PY" -c "
|
||
import json, sys
|
||
ok = 'no'
|
||
try:
|
||
d = json.load(open(sys.argv[1], encoding='utf-8'))
|
||
ok = 'yes' if any(k.startswith('ust:alice') for k in d.get('keys', {})) else 'no'
|
||
except Exception:
|
||
pass
|
||
print(ok)
|
||
" "$STATS" 2>/dev/null)"
|
||
ck "★快照含趋势日桶键(ud:)" yes \
|
||
"$("$PY" -c "
|
||
import json, sys
|
||
ok = 'no'
|
||
try:
|
||
d = json.load(open(sys.argv[1], encoding='utf-8'))
|
||
ok = 'yes' if any(k.startswith('ud:') for k in d.get('keys', {})) else 'no'
|
||
except Exception:
|
||
pass
|
||
print(ok)
|
||
" "$STATS" 2>/dev/null)"
|
||
ck "★快照不含凭据类配置键(up:/upa:/ax:)" yes \
|
||
"$("$PY" -c "
|
||
import json, sys
|
||
bad = 'no'
|
||
try:
|
||
d = json.load(open(sys.argv[1], encoding='utf-8'))
|
||
bad = 'yes' if any(k.split(':', 1)[0] + ':' in ('up:', 'upa:', 'upe:', 'ax:', 'ukey:', 'sub:') for k in d.get('keys', {})) else 'no'
|
||
except Exception:
|
||
pass
|
||
print('yes' if bad == 'no' else 'no')
|
||
" "$STATS" 2>/dev/null)"
|
||
|
||
T0=$(curl -s -b "$JAR" $API/api/admin/usage | "$PY" -c "
|
||
import json, sys
|
||
d = json.load(sys.stdin)['data']
|
||
for u in d.get('users', []):
|
||
if u['username'] == 'alice':
|
||
print(u['total']); break
|
||
else:
|
||
print(-1)
|
||
" 2>/dev/null)
|
||
TR0=$(curl -s -b "$JAR" "$API/api/admin/trend?days=7" | "$PY" -c "
|
||
import json, sys
|
||
print(sum(json.load(sys.stdin)['data']['total']['requests']))
|
||
" 2>/dev/null)
|
||
ck "重启前 alice 用量 > 0" yes "$([ "${T0:-0}" -gt 0 ] && echo yes || echo no)"
|
||
|
||
# ---- 优雅停机(= 生产 STOPSIGNAL SIGQUIT,会调到 worker 的 exit_process 钩子)----
|
||
if [ -n "$NBI" ]; then
|
||
"$NBI" -p "$RUN" -c "$RUN/nginx.conf" -s quit >/dev/null 2>&1
|
||
for _ in $(seq 1 30); do
|
||
code -s -o /dev/null --max-time 0.3 $GW/ >/dev/null 2>&1 || break
|
||
sleep 0.1
|
||
done
|
||
"$NBI" -p "$RUN" -c "$RUN/nginx.conf" >>"$RUN/logs/restart.out" 2>&1
|
||
sleep 0.5
|
||
fi
|
||
|
||
ck "重启后 LLM 端口可用" yes \
|
||
"$(for _ in $(seq 1 40); do [ "$(code -H 'Authorization: Bearer en_bad' -X POST $GW/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY" 2>/dev/null)" = "401" ] && { echo yes; break; }; sleep 0.1; done)"
|
||
|
||
# 会话在共享内存里(sess:),重启即失效 —— 这不是本功能的范畴,重新登录即可
|
||
JAR2=$RUN/cookie2.jar
|
||
curl -s -c "$JAR2" -X POST $API/api/admin/login -H 'Content-Type: application/json' \
|
||
-d "$(jbody username admin password 123456)" >/dev/null
|
||
PST=$(curl -s -b "$JAR2" "$API/api/admin/stats/persist")
|
||
ck "★重启后报告已恢复的键数 > 0" yes \
|
||
"$(printf '%s' "$PST" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if d.get("restored_keys",0)>0 else "no")' 2>/dev/null)"
|
||
ck "★恢复数 = 快照键数(没漏键)" yes \
|
||
"$(printf '%s' "$PST" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if d.get("restore_total",0)>0 and d.get("restored_keys",0)==d.get("restore_total",-1) else "no")' 2>/dev/null)"
|
||
|
||
T1=$(curl -s -b "$JAR2" $API/api/admin/usage | "$PY" -c "
|
||
import json, sys
|
||
d = json.load(sys.stdin)['data']
|
||
for u in d.get('users', []):
|
||
if u['username'] == 'alice':
|
||
print(u['total']); break
|
||
else:
|
||
print(-1)
|
||
" 2>/dev/null)
|
||
ck "★重启后 alice 累计用量不变(未被清零)" "$T0" "$T1"
|
||
# 再打一笔:应从恢复后的基数 +1,而不是从 0 重新长起来
|
||
code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
|
||
-H 'Content-Type: application/json' -d "$BODY" >/dev/null
|
||
T2=$(curl -s -b "$JAR2" $API/api/admin/usage | "$PY" -c "
|
||
import json, sys
|
||
d = json.load(sys.stdin)['data']
|
||
for u in d.get('users', []):
|
||
if u['username'] == 'alice':
|
||
print(u['total']); break
|
||
else:
|
||
print(-1)
|
||
" 2>/dev/null)
|
||
ck "★重启后继续累加(T0+1,而非从 0 重来)" "$(( T0 + 1 ))" "$T2"
|
||
ck "★重启后趋势日桶仍在(历史没丢)" yes \
|
||
"$(curl -s -b "$JAR2" "$API/api/admin/trend?days=7" | "$PY" -c "
|
||
import json, sys
|
||
print('yes' if sum(json.load(sys.stdin)['data']['total']['requests']) >= $TR0 else 'no')
|
||
" 2>/dev/null)"
|
||
ck "★重启后账号级负载仍可读(uda: 恢复)" yes \
|
||
"$(curl -s -b "$JAR2" $API/api/admin/upstreams | "$PY" -c "
|
||
import json, sys
|
||
d = json.load(sys.stdin)['data']
|
||
ok = False
|
||
for s in d.get('items', []):
|
||
if s.get('slot') == 'openai':
|
||
for a in s.get('accounts', []):
|
||
if (a.get('load') or {}).get('requests', 0) > 0:
|
||
ok = True
|
||
print('yes' if ok else 'no')
|
||
" 2>/dev/null)"
|
||
|
||
echo
|
||
echo "============================================================"
|
||
printf " 通过 \033[32m%d\033[0m 失败 \033[31m%d\033[0m\n" "$pass" "$fail"
|
||
echo "============================================================"
|
||
[ "$fail" -eq 0 ] || exit 1
|