Files
ssw__ipass/tests/llm/e2e.sh
T
admin e38237dc4a feat(deploy): 新增 .11 一键更新脚本 + 上游多模型/模型发现/用量统计
功能(本轮 5 项需求的落地):
- 上游可挂多模型:llm-upstreams.json 每上游新增 models 清单(仅用于模型发现,不参与路由)
- 模型发现:GET /v1/models(OpenAI {object:list} 形状,网关本地应答)+ /api/v1/models + /api/v1/upstreams
- 用量统计:/api/admin/usage(管理台「用量统计」Tab)+ /api/v1/usage + /me 的 llm_usage 摘要
  口径:只统计真正转发出去的请求,401/403/404/429 不计;与配额计数 st: 刻意分开
- 账号展示名 display_name(纯展示,不进 ukey:)
- 修复 /v1 版本段重复拼接(llm::join_target)→ 上游 base 自带 /v1 时不再 404 空体
- 修复删账号未清用量键(stats::forget)

部署脚本 deploy/update-ipass.sh:
- 拉镜像 → tar 备份数据 → 删旧容器 → 按旧容器复刻端口/挂载/网络起新容器 → 自检
- 参数从 docker inspect 静态配置读取(docker port 对停止态容器返回空,会让端口静默回落)
- 端口按 host:container 去重(inspect 同一映射含 IPv4/IPv6 两项,重复 -p 会 address already in use)
- --bg 用 setsid/start_new_session 真正脱离会话(仅 nohup & 会被父进程组连带杀掉)
- 旧镜像自动打 rollback-<时间> 标签,收尾打印回滚命令

回归:bash tests/run.sh all → 网关 78 + LLM 75 = 153 条断言全绿。
文档:readme §12.4 / §15 / §16、CLAUDE.md 常用操作 + 排障速查已同步。
2026-09-12 10:09:56 +08:00

250 lines
14 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/bash
# ============================================================================
# LLM 代理 · 本机回归
#
# 一般不用直接跑,走统一入口: bash tests/run.sh llm
# 直接跑需自行保证:假上游(tests/llm/upstream.py)已在 18999 监听,
# nginx 已按 tests/llm/nginx.conf.in 渲染并启动,且 data/ 用的是 fixtures 里
# 那两个固定文件(alice 白名单=openai、bob 白名单为空=不限)。
#
# ★ 核心:不只验网关自己的响应码,而是**读靶机实际收到的请求**——
# 「下游 Key 是否真被换成上游真 Key」只有看靶机才能证明。
#
# 可覆盖:GW / UPSTREAM_LOG / PY
# ============================================================================
set -u
HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
RUN=${RUN:-$(cd "$HERE/.." && pwd)/.run/llm}
GW=${GW:-http://127.0.0.1:18090}
# 管理端口(同一份 nginx / 同一块共享内存)——用量与模型清单从这边读
API=${API:-http://127.0.0.1:18091}
UP_LOG=${UPSTREAM_LOG:-$RUN/upstream.log}
PY=${PY:-python3}
# fixtures 里的固定 Key(见 fixtures/users.json)
ALICE_KEY=sk_alice_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
BOB_KEY=sk_bob_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
pass=0; fail=0
ck() {
if [ "$2" = "$3" ]; then
printf " \033[32m✓\033[0m %s\n" "$1"; pass=$((pass+1))
else
printf " \033[31m✗\033[0m %s 期望=[%s] 实际=[%s]\n" "$1" "$2" "$3"; fail=$((fail+1))
fi
}
code() { curl -s -o /dev/null -w '%{http_code}' "$@"; }
jbody() { local out='{' sep=''; while [ $# -gt 1 ]; do out="$out$sep\"$1\":\"$2\""; sep=','; shift 2; done; printf '%s}' "$out"; }
# 靶机日志:取最后一条记录的某个字段
up_field() {
"$PY" -c "
import json, sys
last = None
try:
for l in open(sys.argv[1], encoding='utf-8'):
l = l.strip()
if l: last = l
except FileNotFoundError:
pass
print(json.loads(last).get(sys.argv[2], '') if last else '')
" "$UP_LOG" "$1" 2>/dev/null
}
up_count() { if [ -f "$UP_LOG" ]; then awk 'NF{n++} END{print n+0}' "$UP_LOG"; else echo 0; fi }
post() { curl -s -H "Authorization: Bearer $1" -X POST "$GW$2" -H 'Content-Type: application/json' -d "$3"; }
# 从 stdin 的 JSON 里按点分路径取值
jqv() {
"$PY" -c "
import sys, json
try: d = json.load(sys.stdin)
except Exception: print(''); sys.exit()
for k in sys.argv[1].split('.'):
if d is None: break
d = d[int(k)] if k.isdigit() and isinstance(d, list) else (d.get(k) if isinstance(d, dict) else None)
print('' if d is None else d)
" "$1" 2>/dev/null
}
# 在 stdin 里找子串(命中输出 yes,否则 no)
has() { if grep -q "$1"; then echo yes; else echo no; fi; }
# 反向断言:**不**出现该子串才算过
nhas() { if grep -q "$1"; then echo no; else echo yes; fi; }
echo "【1】鉴权(OpenAI 兼容错误体)"
BODY=$(jbody model gpt-4o)
ck "无 Key → 401" 401 "$(code -X POST $GW/openai/v1/chat/completions)"
ck "错误 Key → 401" 401 "$(code -X POST $GW/openai/v1/chat/completions -H 'Authorization: Bearer sk_deadbeef')"
ck "错误体是 OpenAI 形状(含 error.type)" yes \
"$(curl -s -X POST $GW/openai/v1/chat/completions | grep -q '"error"' && echo yes || echo no)"
ck "路径缺 slot → 400" 400 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/)"
ck "未知上游 → 404" 404 "$(code -H "Authorization: Bearer $BOB_KEY" -X POST $GW/nope/v1/chat/completions)"
echo
echo "【2】上游解析与白名单"
ck "alice + openai(在白名单)→ 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "响应体来自上游(含 pong)" yes "$(post "$ALICE_KEY" /openai/v1/chat/completions "$BODY" | grep -q pong && echo yes || echo no)"
ck "alice + other(不在白名单)→ 403" 403 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/other/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "alice + downstream(已停用)→ 404" 404 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/downstream/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "bob + other(空白名单=不限)→ 200" 200 "$(code -H "Authorization: Bearer $BOB_KEY" -X POST $GW/other/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
echo
echo "【3】★ Key 替换 / 路径 / 头(读靶机实际收到的请求)"
N0=$(up_count)
post "$ALICE_KEY" "/openai/v1/chat/completions?a=1" "$BODY" >/dev/null
ck "靶机恰好收到 1 次转发" 1 "$(( $(up_count) - N0 ))"
ck "★Authorization 已换成上游真 Key" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)"
ck "★下游 Key 未泄漏给上游" yes "$(up_field auth | grep -q 'sk_alice_' && echo leaked || echo yes)"
ck "Host 头 = 上游主机(非下游域名)" "127.0.0.1:18999" "$(up_field host)"
ck "★路径去 slot 且保留 query" "/v1/chat/completions?a=1" "$(up_field path)"
ck "请求体原样透传" "$BODY" "$(up_field body)"
echo
echo "【4】上游 base 带路径时的拼接"
post "$BOB_KEY" /other/v1/chat/completions "$BODY" >/dev/null
ck "★base=/llm + path 正确拼接" "/llm/v1/chat/completions" "$(up_field path)"
ck "用的是该上游自己的 Key" "Bearer sk-OTHER-UPSTREAM" "$(up_field auth)"
# ★ 版本段去重(实测踩过的坑:厂商 base_url 自带 /v1,客户端也带 /v1 → /v1/v1 → 上游 404 空响应)
post "$BOB_KEY" /deepseeklike/v1/chat/completions "$BODY" >/dev/null
ck "★base 自带 /v1 + 客户端带 /v1 → 只保留一份" "/v1/chat/completions" "$(up_field path)"
post "$BOB_KEY" /deepseeklike/chat/completions "$BODY" >/dev/null
ck " 客户端不带 /v1 → 照常前缀上 base 的 /v1" "/v1/chat/completions" "$(up_field path)"
post "$BOB_KEY" /deepseeklike/v2/chat/completions "$BODY" >/dev/null
ck " 不同版本段不误判为重复" "/v1/v2/chat/completions" "$(up_field path)"
echo
echo "【5】★ 流式(SSE 不被缓冲)"
SSE=$(GW="$GW" KEY="$ALICE_KEY" "$PY" "$HERE/sse_test.py" 2>&1)
ck "Content-Type 是 text/event-stream" "text/event-stream" "$(echo "$SSE" | sed -n 's/^CT=//p')"
ck "收到 3 个 data 块" 3 "$(echo "$SSE" | sed -n 's/^CHUNKS=//p')"
FIRST=$(echo "$SSE" | sed -n 's/^FIRST=//p')
TOTAL=$(echo "$SSE" | sed -n 's/^TOTAL=//p')
ck "★首块 < 0.5s 到达(未被攒包)" yes "$(awk -v f="${FIRST:-9}" 'BEGIN{print (f<0.5)?"yes":"no"}')"
ck "总耗时 ≥ 1.0s(确实分了 3 次推)" yes "$(awk -v t="${TOTAL:-0}" 'BEGIN{print (t>=1.0)?"yes":"no"}')"
echo
echo "【6】★ 限流与每日配额(carol rpm=2、dave daily=2,见 fixtures/users.json)"
CAROL_KEY=sk_carol_cccccccccccccccccccccccccccccccccccccccccccccc
DAVE_KEY=sk_dave_dddddddddddddddddddddddddddddddddddddddddddddddd
cpost() { code -H "Authorization: Bearer $1" -X POST "$GW/other/v1/chat/completions" -H 'Content-Type: application/json' -d "$BODY"; }
ck "carol 第 1 发 → 200" 200 "$(cpost "$CAROL_KEY")"
ck "carol 第 2 发 → 200" 200 "$(cpost "$CAROL_KEY")"
ck "carol 第 3 发 → 429(rpm=2)" 429 "$(cpost "$CAROL_KEY")"
ck "429 是 rate_limit_error / rate_limit_exceeded" yes \
"$(post "$CAROL_KEY" /other/v1/chat/completions "$BODY" | grep -q 'rate_limit' && echo yes || echo no)"
ck "dave 第 1 发 → 200" 200 "$(cpost "$DAVE_KEY")"
ck "dave 第 2 发 → 200" 200 "$(cpost "$DAVE_KEY")"
ck "dave 第 3 发 → 429(daily=2)" 429 "$(cpost "$DAVE_KEY")"
ck "429 是 insufficient_quota" yes \
"$(post "$DAVE_KEY" /other/v1/chat/completions "$BODY" | grep -q 'insufficient_quota' && echo yes || echo no)"
ck "carol 被限后 alice 不受影响(按 Key 隔离)→ 200" 200 \
"$(code -H "Authorization: Bearer $ALICE_KEY" -X POST "$GW/openai/v1/chat/completions" -H 'Content-Type: application/json' -d "$BODY")"
if [ "${HAVE_TLS:-0}" = "1" ]; then
echo
echo "【7】★ HTTPS(18443 自签,模板 tests/llm/tls.conf.in)"
HTTPS_BASE=https://127.0.0.1:18443
ck "https + 错误 Key → 401(TLS 层下鉴权仍生效)" 401 \
"$(code -k -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Authorization: Bearer nope')"
ck "https + alice → 200" 200 \
"$(code -k -H "Authorization: Bearer $ALICE_KEY" -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")"
ck "https 响应体来自上游" yes \
"$(curl -sk -H "Authorization: Bearer $ALICE_KEY" -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY" | grep -q pong && echo yes || echo no)"
else
echo
echo "【7】HTTPS:跳过(本机 nginx 无 http_ssl_module 或无 openssl)"
fi
echo
echo "【8】★ 模型发现(网关本地应答,不去上游)"
M0=$(up_count)
ck "无 Key → 401" 401 "$(code $GW/v1/models)"
ck "alice GET /v1/models → 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models)"
ck " 形状是 OpenAI 的 object=list" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | has '"object":"list"')"
ck " 含白名单内上游的模型 gpt-4o" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | has 'gpt-4o')"
ck " 不含白名单外上游的模型 claude-3-5-sonnet" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | nhas 'claude-3-5-sonnet')"
ck " 不含已停用上游的模型 should-not-appear" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | nhas 'should-not-appear')"
ck "bob(白名单为空=不限)能看到 other 的模型" yes \
"$(curl -s -H "Authorization: Bearer $BOB_KEY" $GW/v1/models | has 'claude-3-5-sonnet')"
ck "GET /models 同样本地应答 → 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/models)"
ck "★整个模型发现过程 0 次上游调用" 0 "$(( $(up_count) - M0 ))"
# 带 slot 的 /v1/models 是**另一条路**:照常透传给厂商,取"那家厂商的"模型清单
N0=$(up_count)
ck "★/<slot>/v1/models 仍透传上游 → 200" 200 \
"$(code -H "Authorization: Bearer $ALICE_KEY" $GW/openai/v1/models)"
ck " 上游确实收到了 1 次请求" 1 "$(( $(up_count) - N0 ))"
ck " 转发路径未剥错(/v1/models)" "/v1/models" "$(up_field path)"
ck " 响应体来自上游(不是网关本地那份)" yes \
"$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/openai/v1/models | has 'upstream-model')"
echo
echo "【9】★ 账号态 API:可用模型 + 自己的用量"
ck "无 Key 查模型 → 401" 401 "$(code $API/api/v1/models)"
ck "无 Key 查用量 → 401" 401 "$(code $API/api/v1/usage)"
ck "alice 查模型 → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/models)"
ck " 模型清单里带 base_url(lm 域名 + slot)" yes \
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has 'lm.mayishangshu.cn/openai')"
# ★ base_url 必须**保留上游 base 的路径**:openai 的 base 无路径 → 无;other 的 base 带 /llm → 带 /llm。
# 搞错这一条会让"base 自带 /v1"的上游变成 /v1/v1/... → 上游 404 且响应体为空,极难排查。
ck "★base_url 不凭空拼 /v1(base 无路径就不带)" yes \
"$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | nhas 'lm.mayishangshu.cn/openai/')"
ck "★base_url 保留上游 base 的路径 /llm" yes \
"$(curl -s -H "X-API-Key: $BOB_KEY" $API/api/v1/models | has 'lm.mayishangshu.cn/other/llm')"
ALICE_USAGE=$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/usage)
ck "alice 查用量 → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/usage)"
ALICE_TOTAL=$(echo "$ALICE_USAGE" | jqv data.total)
ck "★alice 累计用量 > 0(前面真发过请求)" yes \
"$(awk -v t="${ALICE_TOTAL:-0}" 'BEGIN{print (t>0)?"yes":"no"}')"
ck " 今日 = 累计(都是刚发的)" "$ALICE_TOTAL" "$(echo "$ALICE_USAGE" | jqv data.today)"
ck " 分上游明细含 openai" yes "$(echo "$ALICE_USAGE" | has '"slot": *"openai"')"
ck "★被拒的越权/未知上游不进明细(无 other/downstream)" yes \
"$(echo "$ALICE_USAGE" | nhas '"slot": *"other"')"
ck " 回显限额供客户端自我节流" yes "$(echo "$ALICE_USAGE" | has '"limits"')"
# ★ 关键语义:统计的是"真的转发出去"的请求 —— 超限被 429 的那一发不计
CAROL_TOTAL=$(curl -s -H "X-API-Key: $CAROL_KEY" $API/api/v1/usage | jqv data.total)
ck "★carol 发了 3 次、第 3 次被限流 → 用量只记 2" 2 "${CAROL_TOTAL:-x}"
DAVE_TOTAL=$(curl -s -H "X-API-Key: $DAVE_KEY" $API/api/v1/usage | jqv data.total)
ck "★dave 同上(每日配额)→ 用量只记 2" 2 "${DAVE_TOTAL:-x}"
ck "bob 的明细含 other(他真调过)" yes \
"$(curl -s -H "X-API-Key: $BOB_KEY" $API/api/v1/usage | has '"slot": *"other"')"
echo
echo "【10】管理员用量总览 + 删账号清理用量键"
ck "无会话 → 401" 401 "$(code $API/api/admin/usage)"
ck "不是接口路径 → 404" 404 "$(code $API/api/v1/nope)"
JAR=$RUN/cookie.jar
LOGIN=$(curl -s -c "$JAR" -X POST $API/api/admin/login -H 'Content-Type: application/json' \
-d "$(jbody username admin password 123456)")
ck "管理员登录 → 200" 200 "$(echo "$LOGIN" | jqv code)"
USAGE=$(curl -s -b "$JAR" $API/api/admin/usage)
ck "用量总览含 alice" yes "$(echo "$USAGE" | has '"username": *"alice"')"
ck " 总览含按上游分组(openai)" yes "$(echo "$USAGE" | has '"slot": *"openai"')"
ck " 全局合计 > 0" yes "$(echo "$USAGE" | jqv data.total.total | awk '{print ($1>0)?"yes":"no"}')"
ck "★删除前用量榜里有 bob(他真调过 other)" yes "$(echo "$USAGE" | has '"username": *"bob"')"
# ★ 删除账号必须**连带清掉用量键**(`ust:` / `ums:`)——它们是按前缀扫出来出报表的,
# 不清就会让已删账号一直挂在用量榜上(与"删账号必须摘 ukey:"同类的正确性问题)。
curl -s -b "$JAR" -X POST $API/api/admin/users/delete -H 'Content-Type: application/json' \
-d "$(jbody username bob)" >/dev/null
ck "★bob 已被删除(他的 Key 立即失效 → 401)" 401 "$(code -H "X-API-Key: $BOB_KEY" $API/api/v1/usage)"
USAGE2=$(curl -s -b "$JAR" $API/api/admin/usage)
ck "★删除后用量榜不再出现 bob" yes "$(echo "$USAGE2" | nhas 'bob')"
ck " 但 alice 仍在榜单上" yes "$(echo "$USAGE2" | has '"username": *"alice"')"
echo
echo "============================================================"
printf " 通过 \033[32m%d\033[0m 失败 \033[31m%d\033[0m\n" "$pass" "$fail"
echo "============================================================"
[ "$fail" -eq 0 ] || exit 1