Files
ipass__ipass/tests/llm/e2e.sh
T
admin 401f9be351 feat(llm): token 计量 + 对话审计日志(10GiB 容量触发式清理)
- llm_usage.rs: body filter 旁路(FFI 注册 top_body_filter,无需额外 C 模块),
  透传 SSE 的同时抓尾部 usage chunk(OpenAI stream_options.include_usage / Anthropic message_delta),
  解析 prompt/completion tokens 写 shm utt:/utms: 键
- llm_audit.rs: log 阶段把请求体临时文件复制进审计目录(audit/ = data 兄弟目录),
  meta.jsonl 记 user/slot/status/token;10 GiB 容量阈值触发式清理,按日期目录从最旧删起
- conf/nginx.conf: LLM location 去掉 proxy_request_buffering off(它管请求体方向、
  与 SSE 无关且与 client_body_in_file_only 互斥),SSE 流式由 proxy_buffering off 保障
- stats/api: 用量接口增加 tokens 维度(today/total per user per slot)
- ipass-agent.py: 容器重建时自动挂审计卷(宿主 audit/ = data 兄弟目录,自动建目录)
- update-agent-face.sh: 控制面自更新脚本(.11 上一行命令完成下载/备份/校验/重启/自检)
- tests: LLM 117/117(+6: 审计 4 + token 2)、网关 90/90,共 207 全绿
2026-09-12 17:59:34 +08:00

465 lines
27 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/bash
# ============================================================================
# LLM 代理 · 本机回归
#
# 一般不用直接跑,走统一入口: bash tests/run.sh llm
# 直接跑需自行保证:假上游(tests/llm/upstream.py)已在 18999 监听,
# nginx 已按 tests/llm/nginx.conf.in 渲染并启动,且 data/ 用的是 fixtures 里
# 那两个固定文件(alice 白名单=openai、bob 白名单为空=不限)。
#
# ★ 核心:不只验网关自己的响应码,而是**读靶机实际收到的请求**——
# 「下游 Key 是否真被换成上游真 Key」只有看靶机才能证明。
#
# 可覆盖:GW / UPSTREAM_LOG / PY
# ============================================================================
set -u
HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
RUN=${RUN:-$(cd "$HERE/.." && pwd)/.run/llm}
GW=${GW:-http://127.0.0.1:18090}
# 管理端口(同一份 nginx / 同一块共享内存)——用量与模型清单从这边读
API=${API:-http://127.0.0.1:18091}
UP_LOG=${UPSTREAM_LOG:-$RUN/upstream.log}
PY=${PY:-python3}
# fixtures 里的固定 Key(见 fixtures/users.json)
ALICE_KEY=sk_alice_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
BOB_KEY=sk_bob_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
pass=0; fail=0
ck() {
if [ "$2" = "$3" ]; then
printf " \033[32m✓\033[0m %s\n" "$1"; pass=$((pass+1))
else
printf " \033[31m✗\033[0m %s 期望=[%s] 实际=[%s]\n" "$1" "$2" "$3"; fail=$((fail+1))
fi
}
code() { curl -s -o /dev/null -w '%{http_code}' "$@"; }
jbody() { local out='{' sep=''; while [ $# -gt 1 ]; do out="$out$sep\"$1\":\"$2\""; sep=','; shift 2; done; printf '%s}' "$out"; }
# 靶机日志:取最后一条记录的某个字段
up_field() {
"$PY" -c "
import json, sys
last = None
try:
for l in open(sys.argv[1], encoding='utf-8'):
l = l.strip()
if l: last = l
except FileNotFoundError:
pass
print(json.loads(last).get(sys.argv[2], '') if last else '')
" "$UP_LOG" "$1" 2>/dev/null
}
up_count() { if [ -f "$UP_LOG" ]; then awk 'NF{n++} END{print n+0}' "$UP_LOG"; else echo 0; fi }
# 靶机日志:取最后一条记录里**某个请求头**的值(头名大小写不敏感)。
# 专门为「出站头策略」服务:要证明某个头没被发出去 / 某个头还活着,
# 只能看上游实际收到了什么。取不到的空值 = 该头确实没发。
up_hdr() {
"$PY" -c "
import json, sys
last = None
try:
for l in open(sys.argv[1], encoding='utf-8'):
l = l.strip()
if l: last = l
except FileNotFoundError:
pass
h = (json.loads(last).get('headers') or {}) if last else {}
print(h.get(sys.argv[2].lower(), ''))
" "$UP_LOG" "$1" 2>/dev/null
}
post() { curl -s -H "Authorization: Bearer $1" -X POST "$GW$2" -H 'Content-Type: application/json' -d "$3"; }
# 从 stdin 的 JSON 里按点分路径取值
jqv() {
"$PY" -c "
import sys, json
try: d = json.load(sys.stdin)
except Exception: print(''); sys.exit()
for k in sys.argv[1].split('.'):
if d is None: break
d = d[int(k)] if k.isdigit() and isinstance(d, list) else (d.get(k) if isinstance(d, dict) else None)
print('' if d is None else d)
" "$1" 2>/dev/null
}
# 在 stdin 里找子串(命中输出 yes,否则 no)
has() { if grep -q "$1"; then echo yes; else echo no; fi; }
# 反向断言:**不**出现该子串才算过
nhas() { if grep -q "$1"; then echo no; else echo yes; fi; }
echo "【1】鉴权(OpenAI 兼容错误体)"
BODY=$(jbody model gpt-4o)
ck "无 Key → 401" 401 "$(code -X POST $GW/openai/v1/chat/completions)"
ck "错误 Key → 401" 401 "$(code -X POST $GW/openai/v1/chat/completions -H 'Authorization: Bearer sk_deadbeef')"
ck "错误体是 OpenAI 形状(含 error.type)" yes \
"$(curl -s -X POST $GW/openai/v1/chat/completions | grep -q '"error"' && echo yes || echo no)"
ck "路径缺 slot → 400" 400 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/)"
ck "未知上游 → 404" 404 "$(code -H "Authorization: Bearer $BOB_KEY" -X POST $GW/nope/v1/chat/completions)"
echo
echo "【2】上游解析与白名单"
ck "alice + openai(在白名单)→ 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "响应体来自上游(含 pong)" yes "$(post "$ALICE_KEY" /openai/v1/chat/completions "$BODY" | grep -q pong && echo yes || echo no)"
ck "alice + other(不在白名单)→ 403" 403 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/other/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "alice + downstream(已停用)→ 404" 404 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/downstream/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "bob + other(空白名单=不限)→ 200" 200 "$(code -H "Authorization: Bearer $BOB_KEY" -X POST $GW/other/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
echo
echo "【3】★ Key 替换 / 路径 / 头(读靶机实际收到的请求)"
N0=$(up_count)
post "$ALICE_KEY" "/openai/v1/chat/completions?a=1" "$BODY" >/dev/null
ck "靶机恰好收到 1 次转发" 1 "$(( $(up_count) - N0 ))"
ck "★Authorization 已换成上游真 Key" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
ck "★下游 Key 未泄漏给上游" yes "$(up_field auth | grep -q 'sk_alice_' && echo leaked || echo yes)"
ck "Host 头 = 上游主机(非下游域名)" "127.0.0.1:18999" "$(up_field host)"
ck "★路径去 slot 且保留 query" "/v1/chat/completions?a=1" "$(up_field path)"
ck "请求体原样透传" "$BODY" "$(up_field body)"
# ★ 出站头策略(对齐 LiteLLM 的两层:默认不放行 + 凭据头显式抹空)。
# 构造最坏情形:客户端把**网关自己的 Key** 塞进每一个凭据头,
# 同时带上两套协议的扩展头,然后看上游究竟收到了什么。
# 注意这里**故意不发 Authorization**,顺带验证「入站只靠 x-api-key 也能过网关」。
curl -s -o /dev/null -X POST "$GW/openai/v1/chat/completions" \
-H "x-api-key: $ALICE_KEY" \
-H "api-key: $ALICE_KEY" \
-H "x-goog-api-key: $ALICE_KEY" \
-H 'anthropic-version: 2023-06-01' \
-H 'anthropic-beta: prompt-caching-2024-07-31' \
-H 'openai-beta: assistants=v2' \
-H 'Content-Type: application/json' -d "$BODY"
ck "★x-api-key 未泄漏给上游(默认摘除清单)" "" "$(up_hdr x-api-key)"
ck "★api-key 未泄漏给上游" "" "$(up_hdr api-key)"
ck "★x-goog-api-key 未泄漏给上游" "" "$(up_hdr x-goog-api-key)"
ck "★Authorization 仍是上游真 Key(客户端塞不进凭据)" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
ck "★Content-Type 已注入(漏了所有 POST 都会废)" "application/json" "$(up_hdr content-type)"
ck " Accept 已注入(SSE 协商要靠它)" "*/*" "$(up_hdr accept)"
ck "★anthropic-version 已注入(Anthropic 协议必填)" "2023-06-01" "$(up_hdr anthropic-version)"
ck "★anthropic-beta 已注入(Claude Code 扩展依赖)" "prompt-caching-2024-07-31" "$(up_hdr anthropic-beta)"
ck " openai-beta 已注入" "assistants=v2" "$(up_hdr openai-beta)"
ck " User-Agent 已透传(厂商侧排障要靠它)" yes \
"$([ -n "$(up_hdr user-agent)" ] && echo yes || echo no)"
echo
echo "【4】上游 base 带路径时的拼接"
post "$BOB_KEY" /other/v1/chat/completions "$BODY" >/dev/null
ck "★base=/llm + path 正确拼接" "/llm/v1/chat/completions" "$(up_field path)"
ck "用的是该上游自己的 Key" "Bearer sk-OTHER-UPSTREAM" "$(up_field auth)"
# ★ 版本段去重(实测踩过的坑:厂商 base_url 自带 /v1,客户端也带 /v1 → /v1/v1 → 上游 404 空响应)
post "$BOB_KEY" /deepseeklike/v1/chat/completions "$BODY" >/dev/null
ck "★base 自带 /v1 + 客户端带 /v1 → 只保留一份" "/v1/chat/completions" "$(up_field path)"
post "$BOB_KEY" /deepseeklike/chat/completions "$BODY" >/dev/null
ck " 客户端不带 /v1 → 照常前缀上 base 的 /v1" "/v1/chat/completions" "$(up_field path)"
post "$BOB_KEY" /deepseeklike/v2/chat/completions "$BODY" >/dev/null
ck " 不同版本段不误判为重复" "/v1/v2/chat/completions" "$(up_field path)"
# ★ 「统一注册」= base 只写到 host,一条记录同时服务两套协议。
# 这是 LiteLLM(/anthropic/* 前缀)/ one-api(根地址 + 路径天然区分)/
# Higress(按路径后缀判协议)三个开源实现的共同做法:
# **协议由客户端路径表达,网关不做任何推断、不需要分发代码**。
# 夹具里 openai 的 base = http://127.0.0.1:18999(纯 host),正好验这条。
post "$ALICE_KEY" "/openai/anthropic/v1/messages" "$BODY" >/dev/null
ck "★Anthropic 路径原样拼在 host 粒度 base 之后" "/anthropic/v1/messages" "$(up_field path)"
ck " 同一条记录也换上游真 Key(不是只有 OpenAI 路径才换)" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
echo
echo "【5】★ 流式(SSE 不被缓冲)"
SSE=$(GW="$GW" KEY="$ALICE_KEY" "$PY" "$HERE/sse_test.py" 2>&1)
ck "Content-Type 是 text/event-stream" "text/event-stream" "$(echo "$SSE" | sed -n 's/^CT=//p')"
ck "收到 3 个 data 块" 3 "$(echo "$SSE" | sed -n 's/^CHUNKS=//p')"
FIRST=$(echo "$SSE" | sed -n 's/^FIRST=//p')
TOTAL=$(echo "$SSE" | sed -n 's/^TOTAL=//p')
ck "★首块 < 0.5s 到达(未被攒包)" yes "$(awk -v f="${FIRST:-9}" 'BEGIN{print (f<0.5)?"yes":"no"}')"
ck "总耗时 ≥ 1.0s(确实分了 3 次推)" yes "$(awk -v t="${TOTAL:-0}" 'BEGIN{print (t>=1.0)?"yes":"no"}')"
echo
echo "【6】★ 限流与每日配额(carol rpm=2、dave daily=2,见 fixtures/users.json)"
CAROL_KEY=sk_carol_cccccccccccccccccccccccccccccccccccccccccccccc
DAVE_KEY=sk_dave_dddddddddddddddddddddddddddddddddddddddddddddddd
cpost() { code -H "Authorization: Bearer $1" -X POST "$GW/other/v1/chat/completions" -H 'Content-Type: application/json' -d "$BODY"; }
ck "carol 第 1 发 → 200" 200 "$(cpost "$CAROL_KEY")"
ck "carol 第 2 发 → 200" 200 "$(cpost "$CAROL_KEY")"
ck "carol 第 3 发 → 429(rpm=2)" 429 "$(cpost "$CAROL_KEY")"
ck "429 是 rate_limit_error / rate_limit_exceeded" yes \
"$(post "$CAROL_KEY" /other/v1/chat/completions "$BODY" | grep -q 'rate_limit' && echo yes || echo no)"
ck "dave 第 1 发 → 200" 200 "$(cpost "$DAVE_KEY")"
ck "dave 第 2 发 → 200" 200 "$(cpost "$DAVE_KEY")"
ck "dave 第 3 发 → 429(daily=2)" 429 "$(cpost "$DAVE_KEY")"
ck "429 是 insufficient_quota" yes \
"$(post "$DAVE_KEY" /other/v1/chat/completions "$BODY" | grep -q 'insufficient_quota' && echo yes || echo no)"
ck "carol 被限后 alice 不受影响(按 Key 隔离)→ 200" 200 \
"$(code -H "Authorization: Bearer $ALICE_KEY" -X POST "$GW/openai/v1/chat/completions" -H 'Content-Type: application/json' -d "$BODY")"
if [ "${HAVE_TLS:-0}" = "1" ]; then
echo
echo "【7】★ HTTPS(18443 自签,模板 tests/llm/tls.conf.in)"
HTTPS_BASE=https://127.0.0.1:18443
ck "https + 错误 Key → 401(TLS 层下鉴权仍生效)" 401 \
"$(code -k -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Authorization: Bearer nope')"
ck "https + alice → 200" 200 \
"$(code -k -H "Authorization: Bearer $ALICE_KEY" -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "https 响应体来自上游" yes \
"$(curl -sk -H "Authorization: Bearer $ALICE_KEY" -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY" | grep -q pong && echo yes || echo no)"
else
echo
echo "【7】HTTPS:跳过(本机 nginx 无 http_ssl_module 或无 openssl)"
fi
echo
echo "【8】★ 模型发现(网关本地应答,不去上游)"
M0=$(up_count)
ck "无 Key → 401" 401 "$(code $GW/v1/models)"
ck "alice GET /v1/models → 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models)"
ck " 形状是 OpenAI 的 object=list" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | has '"object":"list"')"
ck " 含白名单内上游的模型 gpt-4o" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | has 'gpt-4o')"
ck " 不含白名单外上游的模型 claude-3-5-sonnet" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | nhas 'claude-3-5-sonnet')"
ck " 不含已停用上游的模型 should-not-appear" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | nhas 'should-not-appear')"
ck "bob(白名单为空=不限)能看到 other 的模型" yes \
"$(curl -s -H "Authorization: Bearer $BOB_KEY" $GW/v1/models | has 'claude-3-5-sonnet')"
ck "GET /models 同样本地应答 → 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/models)"
ck "★整个模型发现过程 0 次上游调用" 0 "$(( $(up_count) - M0 ))"
# 带 slot 的 /v1/models 是**另一条路**:照常透传给厂商,取"那家厂商的"模型清单
N0=$(up_count)
ck "★/<slot>/v1/models 仍透传上游 → 200" 200 \
"$(code -H "Authorization: Bearer $ALICE_KEY" $GW/openai/v1/models)"
ck " 上游确实收到了 1 次请求" 1 "$(( $(up_count) - N0 ))"
ck " 转发路径未剥错(/v1/models)" "/v1/models" "$(up_field path)"
ck " 响应体来自上游(不是网关本地那份)" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/openai/v1/models | has 'upstream-model')"
echo
echo "【9】★ 账号态 API:可用模型 + 自己的用量"
# base_url 跟随**调用方所用的端口**(88 部署必须带 :88;443 反代后面则不带)——
# 见 rust/src/upstreams.rs 的 base_url 注释,写死任何一种都会在另一种部署里给出错地址。
API_PORT="${API##*:}"
ck "无 Key 查模型 → 401" 401 "$(code $API/api/v1/models)"
ck "无 Key 查用量 → 401" 401 "$(code $API/api/v1/usage)"
ck "alice 查模型 → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/models)"
ck " 模型清单里带 base_url(lm 域名 + 端口 + slot)" yes \
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has "lm.mayishangshu.cn:$API_PORT/openai")"
# ★ base_url 必须**保留上游 base 的路径**:openai 的 base 无路径 → 无;other 的 base 带 /llm → 带 /llm。
# 搞错这一条会让"base 自带 /v1"的上游变成 /v1/v1/... → 上游 404 且响应体为空,极难排查。
# (校验 base_url 字段的**完整值**而不是整份响应的子串——后者会被同段新增的
# anthropic_base_url(…/openai/anthropic)误伤。)
ck "★base_url 不凭空拼 /v1(base 无路径就不带)" yes \
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has "\"base_url\":\"https://lm.mayishangshu.cn:$API_PORT/openai\"")"
ck "★base_url 保留上游 base 的路径 /llm" yes \
"$(curl -s -H "X-API-Key: $BOB_KEY" $API/api/v1/models | has "lm.mayishangshu.cn:$API_PORT/other/llm")"
# ★ 端口自适应:Host 不带端口(走 443 反代)时 base_url 也不该带端口。
# 写死 88 → 443 部署下客户端连到另一个端口的服务;裸域名 → 88 部署下被出口设备抢答
# (实测 HTTP 200 + 一个 HTML 登录页,是最难看出问题的一种失败)。
ck "★Host 无端口 → base_url 不带端口" yes \
"$(curl -s -H "X-API-Key: $ALICE_KEY" -H 'Host: lm.example' $API/api/v1/models | has 'lm.mayishangshu.cn/openai')"
ck "★Host 带端口 → base_url 带同一端口" yes \
"$(curl -s -H "X-API-Key: $ALICE_KEY" -H "Host: lm.example:$API_PORT" $API/api/v1/models | has "lm.mayishangshu.cn:$API_PORT/openai")"
ALICE_USAGE=$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/usage)
ck "alice 查用量 → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/usage)"
ALICE_TOTAL=$(echo "$ALICE_USAGE" | jqv data.total)
ck "★alice 累计用量 > 0(前面真发过请求)" yes \
"$(awk -v t="${ALICE_TOTAL:-0}" 'BEGIN{print (t>0)?"yes":"no"}')"
ck " 今日 = 累计(都是刚发的)" "$ALICE_TOTAL" "$(echo "$ALICE_USAGE" | jqv data.today)"
ck " 分上游明细含 openai" yes "$(echo "$ALICE_USAGE" | has '"slot": *"openai"')"
ck "★被拒的越权/未知上游不进明细(无 other/downstream)" yes \
"$(echo "$ALICE_USAGE" | nhas '"slot": *"other"')"
ck " 回显限额供客户端自我节流" yes "$(echo "$ALICE_USAGE" | has '"limits"')"
# ★ 关键语义:统计的是"真的转发出去"的请求 —— 超限被 429 的那一发不计
CAROL_TOTAL=$(curl -s -H "X-API-Key: $CAROL_KEY" $API/api/v1/usage | jqv data.total)
ck "★carol 发了 3 次、第 3 次被限流 → 用量只记 2" 2 "${CAROL_TOTAL:-x}"
DAVE_TOTAL=$(curl -s -H "X-API-Key: $DAVE_KEY" $API/api/v1/usage | jqv data.total)
ck "★dave 同上(每日配额)→ 用量只记 2" 2 "${DAVE_TOTAL:-x}"
ck "bob 的明细含 other(他真调过)" yes \
"$(curl -s -H "X-API-Key: $BOB_KEY" $API/api/v1/usage | has '"slot": *"other"')"
echo
echo "【10】管理员用量总览 + 删账号清理用量键"
ck "无会话 → 401" 401 "$(code $API/api/admin/usage)"
ck "不是接口路径 → 404" 404 "$(code $API/api/v1/nope)"
JAR=$RUN/cookie.jar
LOGIN=$(curl -s -c "$JAR" -X POST $API/api/admin/login -H 'Content-Type: application/json' \
-d "$(jbody username admin password 123456)")
ck "管理员登录 → 200" 200 "$(echo "$LOGIN" | jqv code)"
USAGE=$(curl -s -b "$JAR" $API/api/admin/usage)
ck "用量总览含 alice" yes "$(echo "$USAGE" | has '"username": *"alice"')"
ck " 总览含按上游分组(openai)" yes "$(echo "$USAGE" | has '"slot": *"openai"')"
ck " 全局合计 > 0" yes "$(echo "$USAGE" | jqv data.total.total | awk '{print ($1>0)?"yes":"no"}')"
ck "★删除前用量榜里有 bob(他真调过 other)" yes "$(echo "$USAGE" | has '"username": *"bob"')"
# ★ 删除账号必须**连带清掉用量键**(`ust:` / `ums:`)——它们是按前缀扫出来出报表的,
# 不清就会让已删账号一直挂在用量榜上(与"删账号必须摘 ukey:"同类的正确性问题)。
curl -s -b "$JAR" -X POST $API/api/admin/users/delete -H 'Content-Type: application/json' \
-d "$(jbody username bob)" >/dev/null
ck "★bob 已被删除(他的 Key 立即失效 → 401)" 401 "$(code -H "X-API-Key: $BOB_KEY" $API/api/v1/usage)"
USAGE2=$(curl -s -b "$JAR" $API/api/admin/usage)
ck "★删除后用量榜不再出现 bob" yes "$(echo "$USAGE2" | nhas 'bob')"
ck " 但 alice 仍在榜单上" yes "$(echo "$USAGE2" | has '"username": *"alice"')"
echo
echo "【11】★ 上游连通性测试(管理台「测试」按钮的后端链路)"
# 管理台把探测体原样发上来,网关只负责「定位上游 + 换真 Key + 转发 + 透传状态」。
# 判据照旧:不只看 HTTP 200,还要看靶机实际收到的 path / auth。
PING='{"model":"ds-chat","max_tokens":1,"stream":false,"messages":[{"role":"user","content":"ping"}]}'
ck "无会话测试 → 401" 401 \
"$(code -X POST "$API/api/admin/upstreams/test?slot=openai" -d '{}')"
ck "★admin 测 openai → 透传上游 200" 200 \
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=openai" -H 'Content-Type: application/json' -d "$PING")"
ck " 靶机收到的 path 不含 slot 段" "/v1/chat/completions" "$(up_field path)"
ck " 换上了 openai 上游的真 Key" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
ck "★base 自带 /v1 的上游(deepseeklike)→ 去重后 200" 200 \
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=deepseeklike" -H 'Content-Type: application/json' -d "$PING")"
ck " 靶机 path 仍是 /v1/chat/completions(无 /v1/v1、无 slot 段)" "/v1/chat/completions" "$(up_field path)"
ck " 换上了 deepseeklike 的真 Key" "Bearer sk-DS-REAL" "$(up_field auth)"
ck "★已停用的上游测不了(shm 已摘除 → 404)" 404 \
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=downstream" -d '{}')"
ck "不存在的 slot → 404 信封" 404 \
"$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=ghost" -d '{}')"
ck "★内部 location 外部直连 → 404(不能绕过 admin 会话触达上游)" 404 \
"$(code -b "$JAR" -X POST "$API/_upstream_test/openai/v1/chat/completions" -d '{}')"
echo
echo "【12】★ 上游健康标记(失效自动缓存 + 模型发现附可用地址/状态)"
# 健康缓存由 log 阶段的**真实转发**写入(uh:<slot>)——「测试」按钮走同一条
# proxy_pass 链路,探测结果也天然更新缓存,无需第二套标记逻辑。
# 语义:上游回了头 = 活着(4xx 是它对请求内容的意见,不是机器死了);
# 5xx / 连不上(502) / 等头超时(504) = 失效;没被任何请求打过 = unknown。
# 缓存只用于展示(列表标红 / 模型发现附状态),转发路径不读它。
up_field_of() { # $1=slot $2=字段 → 取上游列表里该 slot 的字段值
curl -s -b "$JAR" $API/api/admin/upstreams | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
u = next((x for x in items if x.get("slot") == sys.argv[1]), None)
v = u.get(sys.argv[2]) if u else "miss"
print("miss" if v is None else v)
' "$1" "$2"
}
xm_status() { # /v1/models 扩展块里某 slot 的健康状态
curl -s -H "Authorization: Bearer $HKEY" $GW/v1/models | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
u = next((x for x in d.get("x_ipass_upstreams", []) if x.get("slot") == sys.argv[1]), {})
print(u.get("status", "miss"))
' "$1"
}
ck "创建临时上游 health1 → 200" 200 \
"$(code -b "$JAR" -X POST $API/api/admin/upstreams/save -H 'Content-Type: application/json' \
-d '{"slot":"health1","base":"http://127.0.0.1:18999","auth":"Bearer sk-H1","models":["m1"]}')"
ck "★初始状态 = unknown(没被任何真实请求打过)" unknown "$(up_field_of health1 status)"
# 临时账号(空白名单 = 不限)换一把干净 Key 来打靶机注入的 500
curl -s -b "$JAR" -X POST $API/api/admin/users/create -H 'Content-Type: application/json' \
-d "$(jbody username healthprobe password 123456 display_name hp)" >/dev/null
HKEY=$(curl -s -b "$JAR" $API/api/admin/users | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
print(next((u["api_key"] for u in items if u.get("username") == "healthprobe"), ""))
')
BOOM='{"model":"m1","max_tokens":1,"boom":true,"messages":[{"role":"user","content":"x"}]}'
ck "boom 请求 → 靶机注入的 500 原样透传给客户端" 500 \
"$(code -H "Authorization: Bearer $HKEY" -X POST $GW/health1/v1/chat/completions \
-H 'Content-Type: application/json' -d "$BOOM")"
sleep 0.3
ck "★上游 500 → health1 自动标记为 failed(无需人工探测)" failed "$(up_field_of health1 status)"
ck " 缓存里记录的状态码 = 500" 500 "$(up_field_of health1 status_code)"
ck "★/v1/models 顶层附 x_ipass_upstreams(地址 + 模型 + 状态)" yes \
"$(curl -s -H "Authorization: Bearer $HKEY" $GW/v1/models | has '"x_ipass_upstreams"')"
ck " 其中 health1 同样标 failed" failed "$(xm_status health1)"
ck " 给出 anthropic 形式的 base_url" yes \
"$(curl -s -H "Authorization: Bearer $HKEY" $GW/v1/models | has '"anthropic_base_url"')"
ck "★账号态 /api/v1/upstreams 也带状态与双协议地址" yes \
"$(curl -s -H "X-API-Key: $HKEY" $API/api/v1/upstreams | has '"anthropic_base_url"')"
OKBODY='{"model":"m1","max_tokens":1,"messages":[{"role":"user","content":"ping"}]}'
ck "恢复:正常请求 → 200" 200 \
"$(code -H "Authorization: Bearer $HKEY" -X POST $GW/health1/v1/chat/completions \
-H 'Content-Type: application/json' -d "$OKBODY")"
sleep 0.3
ck "★下次成功转发自动转回 ok(自愈,无需清缓存)" ok "$(up_field_of health1 status)"
# 清理临时数据
curl -s -b "$JAR" -X POST $API/api/admin/upstreams/delete -H 'Content-Type: application/json' \
-d "$(jbody slot health1)" >/dev/null
ck "删除后列表不再出现 health1" miss "$(up_field_of health1 status)"
curl -s -b "$JAR" -X POST $API/api/admin/users/delete -H 'Content-Type: application/json' \
-d "$(jbody username healthprobe)" >/dev/null
echo
echo "【9】对话审计(audit/ 目录:请求体落盘 + meta 索引 + 被拒不入档)"
AUDIT=$RUN/audit
sleep 0.3
audit_n() { find "$AUDIT" -name '*.json' 2>/dev/null | wc -l | tr -d ' '; }
N1=$(audit_n)
ck "转发的请求都落了审计文件" yes "$([ "$N1" -ge 3 ] && echo yes || echo no)"
ck "meta.jsonl 记录账号/上游/状态" yes \
"$(grep -q '"user":"alice"' "$AUDIT"/*/meta.jsonl 2>/dev/null && grep -q '"slot":"openai"' "$AUDIT"/*/meta.jsonl && grep -q '"status":200' "$AUDIT"/*/meta.jsonl && echo yes || echo no)"
# 取 alice 一条成功记录,比对审计文件内容 = 原始请求体
AF=$("$PY" -c "
import json, sys
for l in open(sys.argv[1], encoding='utf-8'):
d = json.loads(l)
if d.get('user') == 'alice' and d.get('status') == 200:
print(d['file']); break
" "$AUDIT"/*/meta.jsonl 2>/dev/null | head -1)
ck "审计文件内容 = 客户端原始请求体" "$BODY" "$(cat "$AUDIT/$AF" 2>/dev/null)"
N2=$(audit_n)
code -H "Authorization: Bearer sk_deadbeef" -X POST $GW/openai/v1/chat/completions \
-H 'Content-Type: application/json' -d "$BODY" >/dev/null
sleep 0.3
ck "被拒请求(401)不入档" 0 "$(( $(audit_n) - N2 ))"
echo "【10】token 计量(body filter 旁路抓 usage → utt:/utms:)"
# 非 stream usage 固定 7/3;带 usage_probe 的流式在 SSE 尾部给 11/22
curl -s -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
-H 'Content-Type: application/json' -d "$BODY" >/dev/null
curl -s -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \
-H 'Content-Type: application/json' -d '{"model":"gpt-4o","stream":true,"usage_probe":true}' >/dev/null
sleep 0.3
TOK=$(curl -s -b "$JAR" $API/api/admin/usage)
tok_ok() { "$PY" -c "
import json, sys
d = json.loads(sys.argv[1])['data']
ok_u = ok_s = False
for u in d.get('users', []):
if u['username'] == 'alice':
t = u.get('tokens', {})
ok_u = t.get('in_total', 0) >= 18 and t.get('out_total', 0) >= 25
for m in u.get('models', []):
if m['slot'] == 'openai':
ok_u = ok_u and m.get('tokens', {}).get('in_total', 0) >= 18
for s in d.get('models', []):
if s['slot'] == 'openai':
ts = s.get('tokens', {})
ok_s = ts.get('in_total', 0) >= 18 and ts.get('out_total', 0) >= 25
print('yes' if ok_u and ok_s else 'no')
" "$1"; }
ck "usage API 出现 token 维度(账号/上游明细/上游聚合)" yes "$(tok_ok "$TOK")"
ck "审计 meta 行记录 token" yes "$("$PY" -c "
import json, sys
hit = False
for f in sys.argv[1:]:
for l in open(f, encoding='utf-8'):
d = json.loads(l)
if d.get('user') == 'alice' and d.get('tin', 0) > 0:
hit = True
print('yes' if hit else 'no')
" "$AUDIT"/*/meta.jsonl 2>/dev/null)"
echo
echo "============================================================"
printf " 通过 \033[32m%d\033[0m 失败 \033[31m%d\033[0m\n" "$pass" "$fail"
echo "============================================================"
[ "$fail" -eq 0 ] || exit 1