Files
ipass__ipass/tests/llm/e2e.sh
T
admin bfa3cf8ff8 feat(llm): 上游多账号路由 + 账号级负载统计 (1.0.14)
一个 slot 挂多把上游账号 Key,解决厂商「单账号额度/并发受限」与
「同一用户缓存命中」两个矛盾诉求。

多账号路由 (plugin-llm)
- models: Upstream.accounts[] (auth/note/resets) + 上游级
  exhaust_status/exhaust_markers;顶层 auth 保留兼容(accounts 为空时合成单账号)
- upstreams::pick_account:粘性绑定 ukb:<key> 优先 → 无绑定时
  FNV-1a(下游 Key) % 可用数 冷分散 → 绑定账号冷却则重新分散 → 全冷却取最早恢复
  * 分散避开厂商单账号 agent 并发上限;落点固定保住 prompt/token 缓存
- cool_until_until 天锚定冷却:boundary = day0 + ceil(已过/窗口)*窗口,
  封顶次日 0 点(MiniMax 5h 窗口、24=5*4+4 → 末段 20:00-24:00 只有 4h 随天重置)
- 耗尽双检测 (log 阶段):$upstream_status 命中 exhaust_status,或
  usage_filter 流式扫到 marker → $llm_acct 定位本次账号 → 写冷却 uac:
- shm: up:/upa:/uac:/ukb:/upe: 五类前缀;$llm_acct 变量 (NOCACHEABLE)

账号级负载统计(问题发现与跟踪)
- 新增 uda:<slot>:<idx>:<day>(请求/token)与 uex:(耗尽次数) 日桶,
  复用内核 usage::bump_day —— 无数据库、无新进程(项目红线)
- 计数点只在 log 阶段一次;pick_account 会被 $llm_target/$llm_auth/$llm_acct
  三个变量各触发一次,放进那里会三倍计数
- view() 投影带每账号 load 与 account_trend + trend_labels(日期由后端算)
- 管理台「上游」页密钥列:各 Key 今日请求/token/耗尽 + 负载比例条 +
  14 天迷你趋势(自绘 SVG,零第三方依赖)+ 冷却/耗尽徽标 + 上游合计

测试与验证
- 回归 279 条全绿(网关 102 + LLM 177),新增 6 条多账号/负载断言
- 假上游新增 UPSTREAM_FAIL_AUTH 模拟账号额度耗尽;靶机验证 Key 真被替换
- 前端以真实 admin.css + upstreams.js 渲染路径做无头浏览器断言:
  比例条与负载成比例(412 vs 96 → 100%/23%)、每账号 14 条块、
  悬停峰值日期与数据逐位吻合、单账号/无账号兜底正常、无横向溢出、零 JS 错误

文档
- 新增 docs/llm/MULTI_ACCOUNT_ROUTING.md(数据模型/路由/冷却数学/shm 布局/负载统计)
- readme §11 数据模型 + §15 上游配置 + §15 用量统计 + §12.6 tag 表 + §16 变更记录
- 修正 docs/llm/*.md 中插件化前的过时路径(rust/src/* → crates/plugin-llm/src/*)
2026-09-14 11:57:08 +08:00

649 lines
39 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/bash
# ============================================================================
# LLM 代理 · 本机回归
#
# 一般不用直接跑,走统一入口: bash tests/run.sh llm
# 直接跑需自行保证:假上游(tests/llm/upstream.py)已在 18999 监听,
# nginx 已按 tests/llm/nginx.conf.in 渲染并启动,且 data/ 用的是 fixtures 里
# 那两个固定文件(alice 白名单=openai、bob 白名单为空=不限)。
#
# ★ 核心:不只验网关自己的响应码,而是**读靶机实际收到的请求**——
# 「下游 Key 是否真被换成上游真 Key」只有看靶机才能证明。
#
# 可覆盖:GW / UPSTREAM_LOG / PY
# ============================================================================
set -u
HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
RUN=${RUN:-$(cd "$HERE/.." && pwd)/.run/llm}
GW=${GW:-http://127.0.0.1:18090}
# 管理端口(同一份 nginx / 同一块共享内存)——用量与模型清单从这边读
API=${API:-http://127.0.0.1:18091}
UP_LOG=${UPSTREAM_LOG:-$RUN/upstream.log}
PY=${PY:-python3}
# fixtures 里的固定 Key(见 fixtures/users.json)
ALICE_KEY=sk_alice_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
BOB_KEY=sk_bob_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
# 多账号路由验证用 admin:白名单为空=不限、无配额限制、且本套件中途会删除 bob 故不复用
ADMIN_KEY=sk_admin_0000000000000000000000000000000000000000000000ab
pass=0; fail=0
ck() {
if [ "$2" = "$3" ]; then
printf " \033[32m✓\033[0m %s\n" "$1"; pass=$((pass+1))
else
printf " \033[31m✗\033[0m %s 期望=[%s] 实际=[%s]\n" "$1" "$2" "$3"; fail=$((fail+1))
fi
}
code() { curl -s -o /dev/null -w '%{http_code}' "$@"; }
jbody() { local out='{' sep=''; while [ $# -gt 1 ]; do out="$out$sep\"$1\":\"$2\""; sep=','; shift 2; done; printf '%s}' "$out"; }
# 靶机日志:取最后一条记录的某个字段
up_field() {
"$PY" -c "
import json, sys
last = None
try:
for l in open(sys.argv[1], encoding='utf-8'):
l = l.strip()
if l: last = l
except FileNotFoundError:
pass
print(json.loads(last).get(sys.argv[2], '') if last else '')
" "$UP_LOG" "$1" 2>/dev/null
}
up_count() { if [ -f "$UP_LOG" ]; then awk 'NF{n++} END{print n+0}' "$UP_LOG"; else echo 0; fi }
# 靶机日志:取最后一条记录里**某个请求头**的值(头名大小写不敏感)。
# 专门为「出站头策略」服务:要证明某个头没被发出去 / 某个头还活着,
# 只能看上游实际收到了什么。取不到的空值 = 该头确实没发。
up_hdr() {
"$PY" -c "
import json, sys
last = None
try:
for l in open(sys.argv[1], encoding='utf-8'):
l = l.strip()
if l: last = l
except FileNotFoundError:
pass
h = (json.loads(last).get('headers') or {}) if last else {}
print(h.get(sys.argv[2].lower(), ''))
" "$UP_LOG" "$1" 2>/dev/null
}
post() { curl -s -H "Authorization: Bearer $1" -X POST "$GW$2" -H 'Content-Type: application/json' -d "$3"; }
# 从 stdin 的 JSON 里按点分路径取值
jqv() {
"$PY" -c "
import sys, json
try: d = json.load(sys.stdin)
except Exception: print(''); sys.exit()
for k in sys.argv[1].split('.'):
if d is None: break
d = d[int(k)] if k.isdigit() and isinstance(d, list) else (d.get(k) if isinstance(d, dict) else None)
print('' if d is None else d)
" "$1" 2>/dev/null
}
# 在 stdin 里找子串(命中输出 yes,否则 no)
has() { if grep -q "$1"; then echo yes; else echo no; fi; }
# 反向断言:**不**出现该子串才算过
nhas() { if grep -q "$1"; then echo no; else echo yes; fi; }
echo "【1】鉴权(OpenAI 兼容错误体)"
BODY=$(jbody model gpt-4o)
ck "无 Key → 401" 401 "$(code -X POST $GW/openai/v1/chat/completions)"
ck "错误 Key → 401" 401 "$(code -X POST $GW/openai/v1/chat/completions -H 'Authorization: Bearer sk_deadbeef')"
ck "错误体是 OpenAI 形状(含 error.type)" yes \
"$(curl -s -X POST $GW/openai/v1/chat/completions | grep -q '"error"' && echo yes || echo no)"
ck "路径缺 slot → 400" 400 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/)"
ck "未知上游 → 404" 404 "$(code -H "Authorization: Bearer $BOB_KEY" -X POST $GW/nope/v1/chat/completions)"
echo
echo "【2】上游解析与白名单"
ck "alice + openai(在白名单)→ 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "响应体来自上游(含 pong)" yes "$(post "$ALICE_KEY" /openai/v1/chat/completions "$BODY" | grep -q pong && echo yes || echo no)"
ck "alice + other(不在白名单)→ 403" 403 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/other/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "alice + downstream(已停用)→ 404" 404 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/downstream/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "bob + other(空白名单=不限)→ 200" 200 "$(code -H "Authorization: Bearer $BOB_KEY" -X POST $GW/other/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
echo
echo "【3】★ Key 替换 / 路径 / 头(读靶机实际收到的请求)"
N0=$(up_count)
post "$ALICE_KEY" "/openai/v1/chat/completions?a=1" "$BODY" >/dev/null
ck "靶机恰好收到 1 次转发" 1 "$(( $(up_count) - N0 ))"
ck "★Authorization 已换成上游真 Key" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
ck "★下游 Key 未泄漏给上游" yes "$(up_field auth | grep -q 'sk_alice_' && echo leaked || echo yes)"
ck "Host 头 = 上游主机(非下游域名)" "127.0.0.1:18999" "$(up_field host)"
ck "★路径去 slot 且保留 query" "/v1/chat/completions?a=1" "$(up_field path)"
ck "请求体原样透传" "$BODY" "$(up_field body)"
# ★ 出站头策略(对齐 LiteLLM 的两层:默认不放行 + 凭据头显式抹空)。
# 构造最坏情形:客户端把**网关自己的 Key** 塞进每一个凭据头,
# 同时带上两套协议的扩展头,然后看上游究竟收到了什么。
# 注意这里**故意不发 Authorization**,顺带验证「入站只靠 x-api-key 也能过网关」。
curl -s -o /dev/null -X POST "$GW/openai/v1/chat/completions" \
-H "x-api-key: $ALICE_KEY" \
-H "api-key: $ALICE_KEY" \
-H "x-goog-api-key: $ALICE_KEY" \
-H 'anthropic-version: 2023-06-01' \
-H 'anthropic-beta: prompt-caching-2024-07-31' \
-H 'openai-beta: assistants=v2' \
-H 'Content-Type: application/json' -d "$BODY"
ck "★x-api-key 未泄漏给上游(默认摘除清单)" "" "$(up_hdr x-api-key)"
ck "★api-key 未泄漏给上游" "" "$(up_hdr api-key)"
ck "★x-goog-api-key 未泄漏给上游" "" "$(up_hdr x-goog-api-key)"
ck "★Authorization 仍是上游真 Key(客户端塞不进凭据)" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
ck "★Content-Type 已注入(漏了所有 POST 都会废)" "application/json" "$(up_hdr content-type)"
ck " Accept 已注入(SSE 协商要靠它)" "*/*" "$(up_hdr accept)"
ck "★anthropic-version 已注入(Anthropic 协议必填)" "2023-06-01" "$(up_hdr anthropic-version)"
ck "★anthropic-beta 已注入(Claude Code 扩展依赖)" "prompt-caching-2024-07-31" "$(up_hdr anthropic-beta)"
ck " openai-beta 已注入" "assistants=v2" "$(up_hdr openai-beta)"
ck " User-Agent 已透传(厂商侧排障要靠它)" yes \
"$([ -n "$(up_hdr user-agent)" ] && echo yes || echo no)"
echo
echo "【4】上游 base 带路径时的拼接"
post "$BOB_KEY" /other/v1/chat/completions "$BODY" >/dev/null
ck "★base=/llm + path 正确拼接" "/llm/v1/chat/completions" "$(up_field path)"
ck "用的是该上游自己的 Key" "Bearer sk-OTHER-UPSTREAM" "$(up_field auth)"
# ★ 版本段去重(实测踩过的坑:厂商 base_url 自带 /v1,客户端也带 /v1 → /v1/v1 → 上游 404 空响应)
post "$BOB_KEY" /deepseeklike/v1/chat/completions "$BODY" >/dev/null
ck "★base 自带 /v1 + 客户端带 /v1 → 只保留一份" "/v1/chat/completions" "$(up_field path)"
post "$BOB_KEY" /deepseeklike/chat/completions "$BODY" >/dev/null
ck " 客户端不带 /v1 → 照常前缀上 base 的 /v1" "/v1/chat/completions" "$(up_field path)"
post "$BOB_KEY" /deepseeklike/v2/chat/completions "$BODY" >/dev/null
ck " 不同版本段不误判为重复" "/v1/v2/chat/completions" "$(up_field path)"
# ★ 「统一注册」= base 只写到 host,一条记录同时服务两套协议。
# 这是 LiteLLM(/anthropic/* 前缀)/ one-api(根地址 + 路径天然区分)/
# Higress(按路径后缀判协议)三个开源实现的共同做法:
# **协议由客户端路径表达,网关不做任何推断、不需要分发代码**。
# 夹具里 openai 的 base = http://127.0.0.1:18999(纯 host),正好验这条。
post "$ALICE_KEY" "/openai/anthropic/v1/messages" "$BODY" >/dev/null
ck "★Anthropic 路径原样拼在 host 粒度 base 之后" "/anthropic/v1/messages" "$(up_field path)"
ck " 同一条记录也换上游真 Key(不是只有 OpenAI 路径才换)" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
echo
echo "【5】★ 流式(SSE 不被缓冲)"
SSE=$(GW="$GW" KEY="$ALICE_KEY" "$PY" "$HERE/sse_test.py" 2>&1)
ck "Content-Type 是 text/event-stream" "text/event-stream" "$(echo "$SSE" | sed -n 's/^CT=//p')"
ck "收到 3 个 data 块" 3 "$(echo "$SSE" | sed -n 's/^CHUNKS=//p')"
FIRST=$(echo "$SSE" | sed -n 's/^FIRST=//p')
TOTAL=$(echo "$SSE" | sed -n 's/^TOTAL=//p')
ck "★首块 < 0.5s 到达(未被攒包)" yes "$(awk -v f="${FIRST:-9}" 'BEGIN{print (f<0.5)?"yes":"no"}')"
ck "总耗时 ≥ 1.0s(确实分了 3 次推)" yes "$(awk -v t="${TOTAL:-0}" 'BEGIN{print (t>=1.0)?"yes":"no"}')"
echo
echo "【6】★ 限流与每日配额(carol rpm=2、dave daily=2,见 fixtures/users.json)"
CAROL_KEY=sk_carol_cccccccccccccccccccccccccccccccccccccccccccccc
DAVE_KEY=sk_dave_dddddddddddddddddddddddddddddddddddddddddddddddd
cpost() { code -H "Authorization: Bearer $1" -X POST "$GW/other/v1/chat/completions" -H 'Content-Type: application/json' -d "$BODY"; }
ck "carol 第 1 发 → 200" 200 "$(cpost "$CAROL_KEY")"
ck "carol 第 2 发 → 200" 200 "$(cpost "$CAROL_KEY")"
ck "carol 第 3 发 → 429(rpm=2)" 429 "$(cpost "$CAROL_KEY")"
ck "429 是 rate_limit_error / rate_limit_exceeded" yes \
"$(post "$CAROL_KEY" /other/v1/chat/completions "$BODY" | grep -q 'rate_limit' && echo yes || echo no)"
ck "dave 第 1 发 → 200" 200 "$(cpost "$DAVE_KEY")"
ck "dave 第 2 发 → 200" 200 "$(cpost "$DAVE_KEY")"
ck "dave 第 3 发 → 429(daily=2)" 429 "$(cpost "$DAVE_KEY")"
ck "429 是 insufficient_quota" yes \
"$(post "$DAVE_KEY" /other/v1/chat/completions "$BODY" | grep -q 'insufficient_quota' && echo yes || echo no)"
ck "carol 被限后 alice 不受影响(按 Key 隔离)→ 200" 200 \
"$(code -H "Authorization: Bearer $ALICE_KEY" -X POST "$GW/openai/v1/chat/completions" -H 'Content-Type: application/json' -d "$BODY")"
if [ "${HAVE_TLS:-0}" = "1" ]; then
echo
echo "【7】★ HTTPS(18443 自签,模板 tests/llm/tls.conf.in)"
HTTPS_BASE=https://127.0.0.1:18443
ck "https + 错误 Key → 401(TLS 层下鉴权仍生效)" 401 \
"$(code -k -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Authorization: Bearer nope')"
ck "https + alice → 200" 200 \
"$(code -k -H "Authorization: Bearer $ALICE_KEY" -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "https 响应体来自上游" yes \
"$(curl -sk -H "Authorization: Bearer $ALICE_KEY" -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY" | grep -q pong && echo yes || echo no)"
else
echo
echo "【7】HTTPS:跳过(本机 nginx 无 http_ssl_module 或无 openssl)"
fi
echo
echo "【8】★ 模型发现(网关本地应答,不去上游)"
M0=$(up_count)
ck "无 Key → 401" 401 "$(code $GW/v1/models)"
ck "alice GET /v1/models → 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models)"
ck " 形状是 OpenAI 的 object=list" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | has '"object":"list"')"
ck " 含白名单内上游的模型 gpt-4o" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | has 'gpt-4o')"
ck " 不含白名单外上游的模型 claude-3-5-sonnet" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | nhas 'claude-3-5-sonnet')"
ck " 不含已停用上游的模型 should-not-appear" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | nhas 'should-not-appear')"
ck "bob(白名单为空=不限)能看到 other 的模型" yes \
"$(curl -s -H "Authorization: Bearer $BOB_KEY" $GW/v1/models | has 'claude-3-5-sonnet')"
ck "GET /models 同样本地应答 → 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/models)"
ck "★整个模型发现过程 0 次上游调用" 0 "$(( $(up_count) - M0 ))"
# 带 slot 的 /v1/models 是**另一条路**:照常透传给厂商,取"那家厂商的"模型清单
N0=$(up_count)
ck "★/<slot>/v1/models 仍透传上游 → 200" 200 \
"$(code -H "Authorization: Bearer $ALICE_KEY" $GW/openai/v1/models)"
ck " 上游确实收到了 1 次请求" 1 "$(( $(up_count) - N0 ))"
ck " 转发路径未剥错(/v1/models)" "/v1/models" "$(up_field path)"
ck " 响应体来自上游(不是网关本地那份)" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/openai/v1/models | has 'upstream-model')"
echo
echo "【9】★ 账号态 API:可用模型 + 自己的用量"
# base_url 跟随**调用方所用的端口**(88 部署必须带 :88;443 反代后面则不带)——
# 见 rust/src/upstreams.rs 的 base_url 注释,写死任何一种都会在另一种部署里给出错地址。
API_PORT="${API##*:}"
ck "无 Key 查模型 → 401" 401 "$(code $API/api/v1/models)"
ck "无 Key 查用量 → 401" 401 "$(code $API/api/v1/usage)"
ck "alice 查模型 → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/models)"
ck " 模型清单里带 base_url(lm 域名 + 端口 + slot)" yes \
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has "lm.mayishangshu.cn:$API_PORT/openai")"
# ★ base_url 必须**保留上游 base 的路径**:openai 的 base 无路径 → 无;other 的 base 带 /llm → 带 /llm。
# 搞错这一条会让"base 自带 /v1"的上游变成 /v1/v1/... → 上游 404 且响应体为空,极难排查。
# (校验 base_url 字段的**完整值**而不是整份响应的子串——后者会被同段新增的
# anthropic_base_url(…/openai/anthropic)误伤。)
ck "★base_url 不凭空拼 /v1(base 无路径就不带)" yes \
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has "\"base_url\":\"https://lm.mayishangshu.cn:$API_PORT/openai\"")"
ck "★base_url 保留上游 base 的路径 /llm" yes \
"$(curl -s -H "X-API-Key: $BOB_KEY" $API/api/v1/models | has "lm.mayishangshu.cn:$API_PORT/other/llm")"
# ★ 端口自适应:Host 不带端口(走 443 反代)时 base_url 也不该带端口。
# 写死 88 → 443 部署下客户端连到另一个端口的服务;裸域名 → 88 部署下被出口设备抢答
# (实测 HTTP 200 + 一个 HTML 登录页,是最难看出问题的一种失败)。
ck "★Host 无端口 → base_url 不带端口" yes \
"$(curl -s -H "X-API-Key: $ALICE_KEY" -H 'Host: lm.example' $API/api/v1/models | has 'lm.mayishangshu.cn/openai')"
ck "★Host 带端口 → base_url 带同一端口" yes \
"$(curl -s -H "X-API-Key: $ALICE_KEY" -H "Host: lm.example:$API_PORT" $API/api/v1/models | has "lm.mayishangshu.cn:$API_PORT/openai")"
ALICE_USAGE=$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/usage)
ck "alice 查用量 → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/usage)"
ALICE_TOTAL=$(echo "$ALICE_USAGE" | jqv data.total)
ck "★alice 累计用量 > 0(前面真发过请求)" yes \
"$(awk -v t="${ALICE_TOTAL:-0}" 'BEGIN{print (t>0)?"yes":"no"}')"
ck " 今日 = 累计(都是刚发的)" "$ALICE_TOTAL" "$(echo "$ALICE_USAGE" | jqv data.today)"
ck " 分上游明细含 openai" yes "$(echo "$ALICE_USAGE" | has '"slot": *"openai"')"
ck "★被拒的越权/未知上游不进明细(无 other/downstream)" yes \
"$(echo "$ALICE_USAGE" | nhas '"slot": *"other"')"
ck " 回显限额供客户端自我节流" yes "$(echo "$ALICE_USAGE" | has '"limits"')"
# ★ 关键语义:统计的是"真的转发出去"的请求 —— 超限被 429 的那一发不计
CAROL_TOTAL=$(curl -s -H "X-API-Key: $CAROL_KEY" $API/api/v1/usage | jqv data.total)
ck "★carol 发了 3 次、第 3 次被限流 → 用量只记 2" 2 "${CAROL_TOTAL:-x}"
DAVE_TOTAL=$(curl -s -H "X-API-Key: $DAVE_KEY" $API/api/v1/usage | jqv data.total)
ck "★dave 同上(每日配额)→ 用量只记 2" 2 "${DAVE_TOTAL:-x}"
ck "bob 的明细含 other(他真调过)" yes \
"$(curl -s -H "X-API-Key: $BOB_KEY" $API/api/v1/usage | has '"slot": *"other"')"
echo
echo "【10】管理员用量总览 + 删账号清理用量键"
ck "无会话 → 401" 401 "$(code $API/api/admin/usage)"
ck "不是接口路径 → 404" 404 "$(code $API/api/v1/nope)"
JAR=$RUN/cookie.jar
LOGIN=$(curl -s -c "$JAR" -X POST $API/api/admin/login -H 'Content-Type: application/json' \
-d "$(jbody username admin password 123456)")
ck "管理员登录 → 200" 200 "$(echo "$LOGIN" | jqv code)"
USAGE=$(curl -s -b "$JAR" $API/api/admin/usage)
ck "用量总览含 alice" yes "$(echo "$USAGE" | has '"username": *"alice"')"
ck " 总览含按上游分组(openai)" yes "$(echo "$USAGE" | has '"slot": *"openai"')"
ck " 全局合计 > 0" yes "$(echo "$USAGE" | jqv data.total.total | awk '{print ($1>0)?"yes":"no"}')"
ck "★删除前用量榜里有 bob(他真调过 other)" yes "$(echo "$USAGE" | has '"username": *"bob"')"
# ★ 删除账号必须**连带清掉用量键**(`ust:` / `ums:`)——它们是按前缀扫出来出报表的,
# 不清就会让已删账号一直挂在用量榜上(与"删账号必须摘 ukey:"同类的正确性问题)。
curl -s -b "$JAR" -X POST $API/api/admin/users/delete -H 'Content-Type: application/json' \
-d "$(jbody username bob)" >/dev/null
ck "★bob 已被删除(他的 Key 立即失效 → 401)" 401 "$(code -H "X-API-Key: $BOB_KEY" $API/api/v1/usage)"
USAGE2=$(curl -s -b "$JAR" $API/api/admin/usage)
ck "★删除后用量榜不再出现 bob" yes "$(echo "$USAGE2" | nhas 'bob')"
ck " 但 alice 仍在榜单上" yes "$(echo "$USAGE2" | has '"username": *"alice"')"
echo
echo "【11】★ 上游连通性测试(管理台「测试」按钮的后端链路)"
# 管理台把探测体原样发上来,网关只负责「定位上游 + 换真 Key + 转发 + 透传状态」。
# 判据照旧:不只看 HTTP 200,还要看靶机实际收到的 path / auth。
PING='{"model":"ds-chat","max_tokens":1,"stream":false,"messages":[{"role":"user","content":"ping"}]}'
ck "无会话测试 → 401" 401 \
"$(code -X POST "$API/api/admin/upstreams/test?slot=openai" -d '{}')"
ck "★admin 测 openai → 透传上游 200" 200 \
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=openai" -H 'Content-Type: application/json' -d "$PING")"
ck " 靶机收到的 path 不含 slot 段" "/v1/chat/completions" "$(up_field path)"
ck " 换上了 openai 上游的真 Key" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
ck "★base 自带 /v1 的上游(deepseeklike)→ 去重后 200" 200 \
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=deepseeklike" -H 'Content-Type: application/json' -d "$PING")"
ck " 靶机 path 仍是 /v1/chat/completions(无 /v1/v1、无 slot 段)" "/v1/chat/completions" "$(up_field path)"
ck " 换上了 deepseeklike 的真 Key" "Bearer sk-DS-REAL" "$(up_field auth)"
ck "★已停用的上游测不了(shm 已摘除 → 404)" 404 \
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=downstream" -d '{}')"
ck "不存在的 slot → 404 信封" 404 \
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=ghost" -d '{}')"
ck "★内部 location 外部直连 → 404(不能绕过 admin 会话触达上游)" 404 \
"$(code -b "$JAR" -X POST "$API/_upstream_test/openai/v1/chat/completions" -d '{}')"
echo
echo "【11.5】★ API 代理(/apix/<slug> → 外部供应商)"
# 与 LLM 代理同一把 sk_ Key,但上游形态不同:供应商只有一个收口,资产由
# **请求体里的 asset_id** 决定,所以请求体必须由网关拼——判据是看靶机
# 收到的 path / 头 / body 三件事,只看网关自己的 200 什么都证明不了。
ck "无 Key → 401" 401 "$(code "$GW/apix/search?q=hello")"
ck "坏 Key → 401" 401 "$(code -H 'X-API-Key: sk_nope' "$GW/apix/search?q=hello")"
ck "不存在的 slug → 404" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/ghost?q=1")"
ck "★已停用的 slug → 404(shm 里已摘除)" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/offline?q=1")"
ck "★正常调用 → 透传上游 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/search?q=hello")"
ck " 靶机 path = 供应商的收口" "/api/v1/ai/request" "$(up_field path)"
ck " ★GET 被改写成 POST(proxy_method + proxy_set_body)" "POST" "$(up_field method)"
ck " ★换上了供应商真 Key" "ak_test_supplier_key" "$(up_hdr x-api-key)"
ck " ★调用体由网关拼:asset_id 来自配置" yes \
"$(up_field body | grep -q '"asset_id":"searxng-search"' && echo yes || echo no)"
ck " ★query 参数进 call_data" yes \
"$(up_field body | grep -q '"q":"hello"' && echo yes || echo no)"
ck " ★下行 Key 没被带出去" yes "$(up_field body | nhas 'sk_alice')"
ck " 下游 Authorization 未透传" "" "$(up_hdr authorization)"
# x-api-key 头**会**有值——那正是换上的供应商真 Key。要证的是它不再以下游的
# sk_ 开头:客户端用自己的 Key 换来供应商凭据,而不是凭据被原样透传。
AXH=$(up_hdr x-api-key)
ck " ★下游 x-api-key 已被换掉(不再是 sk_ 开头)" no \
"$(printf '%s' "$AXH" | grep -q '^sk_' && echo yes || echo no)"
# 管理台:清单 / 凭据掩码 / 停启用写穿
ck "管理台列表含 search" yes "$(curl -s -b "$JAR" $API/api/admin/apix | has '"slug": *"search"')"
ck " ★条目 Key 只回显掩码(不回显真值)" no \
"$(curl -s -b "$JAR" $API/api/admin/apix | has 'ak_test_supplier_key')"
ck " 条目 base 回显" "http://127.0.0.1:18999" \
"$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c '
import sys, json
d = json.load(sys.stdin)["data"]
it = next((x for x in d["items"] if x["slug"] == "search"), None)
print(it.get("base") if it else "miss")')"
ck " 供应商名回显" "vendor-a" \
"$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c '
import sys, json
d = json.load(sys.stdin)["data"]
it = next((x for x in d["items"] if x["slug"] == "search"), None)
print(it.get("supplier") if it else "miss")')"
ck " 已停用条目 enabled=false" "False" "$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c '
import sys, json
d = json.load(sys.stdin)["data"]
it = next((x for x in d["items"] if x["slug"] == "offline"), None)
print(str(it.get("enabled")) if it else "miss")')"
ck "账号态接口只列已启用的(offline 不出现)" "1" \
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/apix | jqv data.count)"
ck "非法 slug → 400" 400 "$(code -b "$JAR" -X POST $API/api/admin/apix/save \
-H 'Content-Type: application/json' -d "$(jbody slug 'Bad Slug!' asset_id x)")"
ck "缺 asset_id → 400" 400 "$(code -b "$JAR" -X POST $API/api/admin/apix/save \
-H 'Content-Type: application/json' -d "$(jbody slug med asset_id '')")"
ck "新建代理 med → 200" 200 "$(code -b "$JAR" -X POST $API/api/admin/apix/save \
-H 'Content-Type: application/json' -d "$(jbody slug med asset_id asset_26476ef1-4687-45 name 疾病百科 params kw base http://127.0.0.1:18999 key ak_test_supplier_key supplier vendor-a)")"
ck " ★新代理立即可用(写穿 shm)" 200 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/med?kw=感冒")"
ck " 靶机 body 带新 asset_id 与新参数" yes \
"$(up_field body | grep -q '"asset_id":"asset_26476ef1-4687-45"' && echo yes || echo no)"
ck " 参数名用的是 med 声明的 kw" yes "$(up_field body | grep -q '"kw":"' && echo yes || echo no)"
curl -s -b "$JAR" -X POST $API/api/admin/apix/save -H 'Content-Type: application/json' \
-d "$(jbody slug search asset_id searxng-search enabled false)" >/dev/null
ck "★停用后立即 404(写穿 shm,无回源兜底)" 404 \
"$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/search?q=1")"
curl -s -b "$JAR" -X POST $API/api/admin/apix/save -H 'Content-Type: application/json' \
-d "$(jbody slug search asset_id searxng-search enabled true)" >/dev/null
ck " 重新启用后又能调通" 200 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/search?q=1")"
curl -s -b "$JAR" -X POST $API/api/admin/apix/delete -H 'Content-Type: application/json' \
-d "$(jbody slug med)" >/dev/null
ck "删除后再调 → 404" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/med?kw=x")"
# 探测:走供应商的**免费**预估接口,不执行资产、不扣费
ck "★探测(免费预估)→ 透传 200" 200 "$(code -b "$JAR" -X POST \
"$API/api/admin/apix/test?slug=search" -H 'Content-Type: application/json' -d '{}')"
ck " 探测打的是 /api/v1/ai/estimate" "/api/v1/ai/estimate" "$(up_field path)"
ck " 探测也换了供应商真 Key" "ak_test_supplier_key" "$(up_hdr x-api-key)"
ck "★内部探测口外部直连 → 404" 404 \
"$(code -b "$JAR" -X POST "$API/_apix_probe/search" -d '{}')"
ck " 不存在的 slug 探测 → 404" 404 "$(code -b "$JAR" -X POST \
"$API/api/admin/apix/test?slug=ghost" -H 'Content-Type: application/json' -d '{}')"
ck "★用量统计按 api/<slug> 归组(与 LLM 上游天然分开)" yes \
"$(curl -s -b "$JAR" $API/api/admin/usage | has '"slot": *"api/search"')"
ck "★趋势含 api/<slug> 维度" yes "$(curl -s -b "$JAR" "$API/api/admin/trend?days=7" | has '"slot": *"api/search"')"
echo
echo "【12】★ 上游健康标记(失效自动缓存 + 模型发现附可用地址/状态)"
# 健康缓存由 log 阶段的**真实转发**写入(uh:<slot>)——「测试」按钮走同一条
# proxy_pass 链路,探测结果也天然更新缓存,无需第二套标记逻辑。
# 语义:上游回了头 = 活着(4xx 是它对请求内容的意见,不是机器死了);
# 5xx / 连不上(502) / 等头超时(504) = 失效;没被任何请求打过 = unknown。
# 缓存只用于展示(列表标红 / 模型发现附状态),转发路径不读它。
up_field_of() { # $1=slot $2=字段 → 取上游列表里该 slot 的字段值
curl -s -b "$JAR" $API/api/admin/upstreams | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
u = next((x for x in items if x.get("slot") == sys.argv[1]), None)
v = u.get(sys.argv[2]) if u else "miss"
print("miss" if v is None else v)
' "$1" "$2"
}
xm_status() { # /v1/models 扩展块里某 slot 的健康状态
curl -s -H "Authorization: Bearer $HKEY" $GW/v1/models | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
u = next((x for x in d.get("x_ipass_upstreams", []) if x.get("slot") == sys.argv[1]), {})
print(u.get("status", "miss"))
' "$1"
}
ck "创建临时上游 health1 → 200" 200 \
"$(code -b "$JAR" -X POST $API/api/admin/upstreams/save -H 'Content-Type: application/json' \
-d '{"slot":"health1","base":"http://127.0.0.1:18999","auth":"Bearer sk-H1","models":["m1"]}')"
ck "★初始状态 = unknown(没被任何真实请求打过)" unknown "$(up_field_of health1 status)"
# 临时账号(空白名单 = 不限)换一把干净 Key 来打靶机注入的 500
curl -s -b "$JAR" -X POST $API/api/admin/users/create -H 'Content-Type: application/json' \
-d "$(jbody username healthprobe password 123456 display_name hp)" >/dev/null
HKEY=$(curl -s -b "$JAR" $API/api/admin/users | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
print(next((u["api_key"] for u in items if u.get("username") == "healthprobe"), ""))
')
BOOM='{"model":"m1","max_tokens":1,"boom":true,"messages":[{"role":"user","content":"x"}]}'
ck "boom 请求 → 靶机注入的 500 原样透传给客户端" 500 \
"$(code -H "Authorization: Bearer $HKEY" -X POST $GW/health1/v1/chat/completions \
-H 'Content-Type: application/json' -d "$BOOM")"
sleep 0.3
ck "★上游 500 → health1 自动标记为 failed(无需人工探测)" failed "$(up_field_of health1 status)"
ck " 缓存里记录的状态码 = 500" 500 "$(up_field_of health1 status_code)"
ck "★/v1/models 顶层附 x_ipass_upstreams(地址 + 模型 + 状态)" yes \
"$(curl -s -H "Authorization: Bearer $HKEY" $GW/v1/models | has '"x_ipass_upstreams"')"
ck " 其中 health1 同样标 failed" failed "$(xm_status health1)"
ck " 给出 anthropic 形式的 base_url" yes \
"$(curl -s -H "Authorization: Bearer $HKEY" $GW/v1/models | has '"anthropic_base_url"')"
ck "★账号态 /api/v1/upstreams 也带状态与双协议地址" yes \
"$(curl -s -H "X-API-Key: $HKEY" $API/api/v1/upstreams | has '"anthropic_base_url"')"
OKBODY='{"model":"m1","max_tokens":1,"messages":[{"role":"user","content":"ping"}]}'
ck "恢复:正常请求 → 200" 200 \
"$(code -H "Authorization: Bearer $HKEY" -X POST $GW/health1/v1/chat/completions \
-H 'Content-Type: application/json' -d "$OKBODY")"
sleep 0.3
ck "★下次成功转发自动转回 ok(自愈,无需清缓存)" ok "$(up_field_of health1 status)"
# 清理临时数据
curl -s -b "$JAR" -X POST $API/api/admin/upstreams/delete -H 'Content-Type: application/json' \
-d "$(jbody slot health1)" >/dev/null
ck "删除后列表不再出现 health1" miss "$(up_field_of health1 status)"
curl -s -b "$JAR" -X POST $API/api/admin/users/delete -H 'Content-Type: application/json' \
-d "$(jbody username healthprobe)" >/dev/null
echo
echo "【9】对话审计(audit/ 目录:请求体落盘 + meta 索引 + 被拒不入档)"
AUDIT=$RUN/audit
sleep 0.3
audit_n() { find "$AUDIT" -name '*.json' 2>/dev/null | wc -l | tr -d ' '; }
N1=$(audit_n)
ck "转发的请求都落了审计文件" yes "$([ "$N1" -ge 3 ] && echo yes || echo no)"
ck "meta.jsonl 记录账号/上游/状态" yes \
"$(grep -q '"user":"alice"' "$AUDIT"/*/meta.jsonl 2>/dev/null && grep -q '"slot":"openai"' "$AUDIT"/*/meta.jsonl && grep -q '"status":200' "$AUDIT"/*/meta.jsonl && echo yes || echo no)"
# 取 alice 一条成功记录,比对审计文件内容 = 原始请求体
AF=$("$PY" -c "
import json, sys
for l in open(sys.argv[1], encoding='utf-8'):
d = json.loads(l)
if d.get('user') == 'alice' and d.get('status') == 200:
print(d['file']); break
" "$AUDIT"/*/meta.jsonl 2>/dev/null | head -1)
ck "审计文件内容 = 客户端原始请求体" "$BODY" "$(cat "$AUDIT/$AF" 2>/dev/null)"
N2=$(audit_n)
code -H "Authorization: Bearer sk_deadbeef" -X POST $GW/openai/v1/chat/completions \
-H 'Content-Type: application/json' -d "$BODY" >/dev/null
sleep 0.3
ck "被拒请求(401)不入档" 0 "$(( $(audit_n) - N2 ))"
echo "【10】token 计量(body filter 旁路抓 usage → utt:/utms:)"
# 非 stream usage 固定 7/3;带 usage_probe 的流式在 SSE 尾部给 11/22
curl -s -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
-H 'Content-Type: application/json' -d "$BODY" >/dev/null
curl -s -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
-H 'Content-Type: application/json' -d '{"model":"gpt-4o","stream":true,"usage_probe":true}' >/dev/null
sleep 0.3
TOK=$(curl -s -b "$JAR" $API/api/admin/usage)
echo "【多账号路由】粘性绑定 + 额度耗尽自动切换"
# 用 admin(白名单为空=不限;本套件中途会删除 bob,故不复用 bob)
ck "admin 基础鉴权(openai,白名单不限)→ 200" 200 "$(code -H "Authorization: Bearer $ADMIN_KEY" -X POST $GW/openai/v1/chat/completions -H 'Content-Type: application/json' -d '{"model":"gpt-4o","messages":[{"role":"user","content":"hi"}],"max_tokens":8}')"
acct_of() { curl -s -D - -o /dev/null -X POST "$GW/$1/v1/chat/completions" \
-H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \
-d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}' \
| grep -i '^X-Upstream-Acct' | tr -d '\r' | awk '{print $2}'; }
code_of() { curl -s -o /dev/null -w '%{http_code}' -X POST "$GW/$1/v1/chat/completions" \
-H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \
-d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}'; }
ck "sticky 首次请求命中 acct=0" 0 "$(acct_of sticky sticky-model)"
ck "sticky 首次请求 200" 200 "$(code_of sticky sticky-model)"
ck "sticky 粘性:第 2 次仍 acct=0" 0 "$(acct_of sticky sticky-model)"
ck "sticky 粘性:第 3 次仍 acct=0" 0 "$(acct_of sticky sticky-model)"
# 切换:multi slot accounts[0]=bad(命中假上游 UPSTREAM_FAIL_AUTH → 401+marker)→ 自动切 acct=1
mh() { curl -s -D - -o /dev/null -X POST "$GW/$1/v1/chat/completions" \
-H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \
-d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}' \
| grep -iE '^HTTP/|^X-Upstream-Acct' | tr -d '\r'; }
M1=$(mh multi multi-model)
ck "multi 初次请求打到 acct=0(bad)" 0 "$(echo "$M1" | awk '/X-Upstream-Acct/{print $2}')"
ck "multi 初次请求 401(触发冷却)" 401 "$(echo "$M1" | awk '/^HTTP/{print $2}')"
M2=$(mh multi multi-model)
ck "multi 冷却中切到 acct=1" 1 "$(echo "$M2" | awk '/X-Upstream-Acct/{print $2}')"
ck "multi 切到 acct=1 后 200" 200 "$(echo "$M2" | awk '/^HTTP/{print $2}')"
M3=$(mh multi multi-model)
ck "multi 冷却期间保持 acct=1(不回退坏账号)" 1 "$(echo "$M3" | awk '/X-Upstream-Acct/{print $2}')"
# 靶机验证:再发一次确认切换后的账号,读靶机实际收到的 auth(证明下行 Key 真被换成对应账号 Key)
M4=$(mh multi multi-model)
ck "multi 切换后下游 Key 真换成 acct-good" "Bearer acct-good" "$(up_hdr Authorization)"
# ---- 账号级负载统计(管理台「密钥 / 负载」列的数据源)----
# 每次真实转发在 log 阶段落到「本次选中的那个账号」的日桶 uda:<slot>:<idx>:<day>;
# 额度耗尽另记 uex:。用同一份 /api/admin/upstreams 投影断言负载分布是否正确。
UPS=$(curl -s -b "$JAR" $API/api/admin/upstreams)
acct_num() { # $1=slot $2=idx $3=度量路径(如 load.requests)→ 取账号级数值
printf '%s' "$UPS" | "$PY" -c '
import sys, json
slot, idx, path = sys.argv[1], int(sys.argv[2]), sys.argv[3]
d = json.load(sys.stdin)
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
for u in items:
if u.get("slot") == slot:
for a in u.get("accounts", []):
if a.get("idx") == idx:
cur = a
for k in path.split("."):
cur = cur.get(k) if isinstance(cur, dict) else None
print(cur if cur is not None else 0)
sys.exit(0)
print(0)
' "$1" "$2" "$3"
}
ck "★sticky 账号#0 记到今日请求(负载分布生效)" yes "$([ "$(acct_num sticky 0 load.requests)" -ge 3 ] && echo yes || echo no)"
ck "★sticky 账号#1 未被分配(0 请求,粘性=集中在一把 Key)" 0 "$(acct_num sticky 1 load.requests)"
ck "★multi 账号#0 记到 1 次额度耗尽" yes "$([ "$(acct_num multi 0 load.exhausts)" -ge 1 ] && echo yes || echo no)"
ck "★multi 账号#1 记到今日请求(切换后负载落在新 Key)" yes "$([ "$(acct_num multi 1 load.requests)" -ge 3 ] && echo yes || echo no)"
ck "账号级趋势序列含 14 天" 14 \
"$(printf '%s' "$UPS" | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
for u in items:
if u.get("slot") == "sticky":
t = u.get("account_trend") or [{}]
print(len(t[0].get("requests") or []))
break
else:
print(0)
')"
# 日期标签与趋势序列逐位对齐(图表悬停提示用;由后端算,避免前后端各自推导时区)
ck "账号趋势带 14 个日期标签(与序列对齐)" 14 \
"$(printf '%s' "$UPS" | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
for u in items:
if u.get("slot") == "sticky":
print(len(u.get("trend_labels") or []))
break
else:
print(0)
')"
# ---- 趋势(日桶)----
# 口径与请求级用量一致:只统计真正转发出去的请求与上游 usage 里的 token
TR=$(curl -s -b "$JAR" "$API/api/admin/trend?days=7")
ck "趋势接口 → 200" 200 "$(code -b "$JAR" "$API/api/admin/trend?days=7")"
ck "趋势默认含 14 天" 14 "$(printf '%s' "$(curl -s -b "$JAR" $API/api/admin/trend)" | "$PY" -c 'import json,sys;print(len(json.load(sys.stdin)["data"]["labels"]))' 2>/dev/null)"
ck "趋势天数上限 90" 90 "$(printf '%s' "$(curl -s -b "$JAR" "$API/api/admin/trend?days=99999")" | "$PY" -c 'import json,sys;print(json.load(sys.stdin)["data"]["days"])' 2>/dev/null)"
ck "7 天区间 → 7 个标签" 7 "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;print(len(json.load(sys.stdin)["data"]["labels"]))' 2>/dev/null)"
ck "今日请求已进日桶" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if sum(d["total"]["requests"])>0 else "no")' 2>/dev/null)"
ck "今日 token 已进日桶" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if sum(d["total"]["in"])+sum(d["total"]["out"])>0 else "no")' 2>/dev/null)"
ck "趋势含按账号维度" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if any(u["username"]=="alice" for u in d["users"]) else "no")' 2>/dev/null)"
ck "趋势含按 API 组维度" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if any(not s["empty"] for s in d["slots"]) else "no")' 2>/dev/null)"
tok_ok() { "$PY" -c "
import json, sys
d = json.loads(sys.argv[1])['data']
ok_u = ok_s = False
for u in d.get('users', []):
if u['username'] == 'alice':
t = u.get('tokens', {})
ok_u = t.get('in_total', 0) >= 18 and t.get('out_total', 0) >= 25
for m in u.get('models', []):
if m['slot'] == 'openai':
ok_u = ok_u and m.get('tokens', {}).get('in_total', 0) >= 18
for s in d.get('models', []):
if s['slot'] == 'openai':
ts = s.get('tokens', {})
ok_s = ts.get('in_total', 0) >= 18 and ts.get('out_total', 0) >= 25
print('yes' if ok_u and ok_s else 'no')
" "$1"; }
ck "usage API 出现 token 维度(账号/上游明细/上游聚合)" yes "$(tok_ok "$TOK")"
ck "审计 meta 行记录 token" yes "$("$PY" -c "
import json, sys
hit = False
for f in sys.argv[1:]:
for l in open(f, encoding='utf-8'):
d = json.loads(l)
if d.get('user') == 'alice' and d.get('tin', 0) > 0:
hit = True
print('yes' if hit else 'no')
" "$AUDIT"/*/meta.jsonl 2>/dev/null)"
echo
echo "============================================================"
printf " 通过 \033[32m%d\033[0m 失败 \033[31m%d\033[0m\n" "$pass" "$fail"
echo "============================================================"
[ "$fail" -eq 0 ] || exit 1