#!/bin/bash # ============================================================================ # LLM 代理 · 本机回归 # # 一般不用直接跑,走统一入口: bash tests/run.sh llm # 直接跑需自行保证:假上游(tests/llm/upstream.py)已在 18999 监听, # nginx 已按 tests/llm/nginx.conf.in 渲染并启动,且 data/ 用的是 fixtures 里 # 那两个固定文件(alice 白名单=openai、bob 白名单为空=不限)。 # # ★ 核心:不只验网关自己的响应码,而是**读靶机实际收到的请求**—— # 「下游 Key 是否真被换成上游真 Key」只有看靶机才能证明。 # # 可覆盖:GW / UPSTREAM_LOG / PY # ============================================================================ set -u HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) RUN=${RUN:-$(cd "$HERE/.." && pwd)/.run/llm} GW=${GW:-http://127.0.0.1:18090} # 管理端口(同一份 nginx / 同一块共享内存)——用量与模型清单从这边读 API=${API:-http://127.0.0.1:18091} UP_LOG=${UPSTREAM_LOG:-$RUN/upstream.log} PY=${PY:-python3} # fixtures 里的固定 Key(见 fixtures/users.json) ALICE_KEY=sk_alice_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa BOB_KEY=sk_bob_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb pass=0; fail=0 ck() { if [ "$2" = "$3" ]; then printf " \033[32m✓\033[0m %s\n" "$1"; pass=$((pass+1)) else printf " \033[31m✗\033[0m %s 期望=[%s] 实际=[%s]\n" "$1" "$2" "$3"; fail=$((fail+1)) fi } code() { curl -s -o /dev/null -w '%{http_code}' "$@"; } jbody() { local out='{' sep=''; while [ $# -gt 1 ]; do out="$out$sep\"$1\":\"$2\""; sep=','; shift 2; done; printf '%s}' "$out"; } # 靶机日志:取最后一条记录的某个字段 up_field() { "$PY" -c " import json, sys last = None try: for l in open(sys.argv[1], encoding='utf-8'): l = l.strip() if l: last = l except FileNotFoundError: pass print(json.loads(last).get(sys.argv[2], '') if last else '') " "$UP_LOG" "$1" 2>/dev/null } up_count() { if [ -f "$UP_LOG" ]; then awk 'NF{n++} END{print n+0}' "$UP_LOG"; else echo 0; fi } post() { curl -s -H "Authorization: Bearer $1" -X POST "$GW$2" -H 'Content-Type: application/json' -d "$3"; } # 从 stdin 的 JSON 里按点分路径取值 jqv() { "$PY" -c " import sys, json try: d = json.load(sys.stdin) except Exception: print(''); sys.exit() for k in sys.argv[1].split('.'): if d is None: break d = d[int(k)] if k.isdigit() and isinstance(d, list) else (d.get(k) if isinstance(d, dict) else None) print('' if d is None else d) " "$1" 2>/dev/null } # 在 stdin 里找子串(命中输出 yes,否则 no) has() { if grep -q "$1"; then echo yes; else echo no; fi; } # 反向断言:**不**出现该子串才算过 nhas() { if grep -q "$1"; then echo no; else echo yes; fi; } echo "【1】鉴权(OpenAI 兼容错误体)" BODY=$(jbody model gpt-4o) ck "无 Key → 401" 401 "$(code -X POST $GW/openai/v1/chat/completions)" ck "错误 Key → 401" 401 "$(code -X POST $GW/openai/v1/chat/completions -H 'Authorization: Bearer sk_deadbeef')" ck "错误体是 OpenAI 形状(含 error.type)" yes \ "$(curl -s -X POST $GW/openai/v1/chat/completions | grep -q '"error"' && echo yes || echo no)" ck "路径缺 slot → 400" 400 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/)" ck "未知上游 → 404" 404 "$(code -H "Authorization: Bearer $BOB_KEY" -X POST $GW/nope/v1/chat/completions)" echo echo "【2】上游解析与白名单" ck "alice + openai(在白名单)→ 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")" ck "响应体来自上游(含 pong)" yes "$(post "$ALICE_KEY" /openai/v1/chat/completions "$BODY" | grep -q pong && echo yes || echo no)" ck "alice + other(不在白名单)→ 403" 403 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/other/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")" ck "alice + downstream(已停用)→ 404" 404 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/downstream/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")" ck "bob + other(空白名单=不限)→ 200" 200 "$(code -H "Authorization: Bearer $BOB_KEY" -X POST $GW/other/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")" echo echo "【3】★ Key 替换 / 路径 / 头(读靶机实际收到的请求)" N0=$(up_count) post "$ALICE_KEY" "/openai/v1/chat/completions?a=1" "$BODY" >/dev/null ck "靶机恰好收到 1 次转发" 1 "$(( $(up_count) - N0 ))" ck "★Authorization 已换成上游真 Key" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)" ck "★下游 Key 未泄漏给上游" yes "$(up_field auth | grep -q 'sk_alice_' && echo leaked || echo yes)" ck "Host 头 = 上游主机(非下游域名)" "127.0.0.1:18999" "$(up_field host)" ck "★路径去 slot 且保留 query" "/v1/chat/completions?a=1" "$(up_field path)" ck "请求体原样透传" "$BODY" "$(up_field body)" echo echo "【4】上游 base 带路径时的拼接" post "$BOB_KEY" /other/v1/chat/completions "$BODY" >/dev/null ck "★base=/llm + path 正确拼接" "/llm/v1/chat/completions" "$(up_field path)" ck "用的是该上游自己的 Key" "Bearer sk-OTHER-UPSTREAM" "$(up_field auth)" # ★ 版本段去重(实测踩过的坑:厂商 base_url 自带 /v1,客户端也带 /v1 → /v1/v1 → 上游 404 空响应) post "$BOB_KEY" /deepseeklike/v1/chat/completions "$BODY" >/dev/null ck "★base 自带 /v1 + 客户端带 /v1 → 只保留一份" "/v1/chat/completions" "$(up_field path)" post "$BOB_KEY" /deepseeklike/chat/completions "$BODY" >/dev/null ck " 客户端不带 /v1 → 照常前缀上 base 的 /v1" "/v1/chat/completions" "$(up_field path)" post "$BOB_KEY" /deepseeklike/v2/chat/completions "$BODY" >/dev/null ck " 不同版本段不误判为重复" "/v1/v2/chat/completions" "$(up_field path)" echo echo "【5】★ 流式(SSE 不被缓冲)" SSE=$(GW="$GW" KEY="$ALICE_KEY" "$PY" "$HERE/sse_test.py" 2>&1) ck "Content-Type 是 text/event-stream" "text/event-stream" "$(echo "$SSE" | sed -n 's/^CT=//p')" ck "收到 3 个 data 块" 3 "$(echo "$SSE" | sed -n 's/^CHUNKS=//p')" FIRST=$(echo "$SSE" | sed -n 's/^FIRST=//p') TOTAL=$(echo "$SSE" | sed -n 's/^TOTAL=//p') ck "★首块 < 0.5s 到达(未被攒包)" yes "$(awk -v f="${FIRST:-9}" 'BEGIN{print (f<0.5)?"yes":"no"}')" ck "总耗时 ≥ 1.0s(确实分了 3 次推)" yes "$(awk -v t="${TOTAL:-0}" 'BEGIN{print (t>=1.0)?"yes":"no"}')" echo echo "【6】★ 限流与每日配额(carol rpm=2、dave daily=2,见 fixtures/users.json)" CAROL_KEY=sk_carol_cccccccccccccccccccccccccccccccccccccccccccccc DAVE_KEY=sk_dave_dddddddddddddddddddddddddddddddddddddddddddddddd cpost() { code -H "Authorization: Bearer $1" -X POST "$GW/other/v1/chat/completions" -H 'Content-Type: application/json' -d "$BODY"; } ck "carol 第 1 发 → 200" 200 "$(cpost "$CAROL_KEY")" ck "carol 第 2 发 → 200" 200 "$(cpost "$CAROL_KEY")" ck "carol 第 3 发 → 429(rpm=2)" 429 "$(cpost "$CAROL_KEY")" ck "429 是 rate_limit_error / rate_limit_exceeded" yes \ "$(post "$CAROL_KEY" /other/v1/chat/completions "$BODY" | grep -q 'rate_limit' && echo yes || echo no)" ck "dave 第 1 发 → 200" 200 "$(cpost "$DAVE_KEY")" ck "dave 第 2 发 → 200" 200 "$(cpost "$DAVE_KEY")" ck "dave 第 3 发 → 429(daily=2)" 429 "$(cpost "$DAVE_KEY")" ck "429 是 insufficient_quota" yes \ "$(post "$DAVE_KEY" /other/v1/chat/completions "$BODY" | grep -q 'insufficient_quota' && echo yes || echo no)" ck "carol 被限后 alice 不受影响(按 Key 隔离)→ 200" 200 \ "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST "$GW/openai/v1/chat/completions" -H 'Content-Type: application/json' -d "$BODY")" if [ "${HAVE_TLS:-0}" = "1" ]; then echo echo "【7】★ HTTPS(18443 自签,模板 tests/llm/tls.conf.in)" HTTPS_BASE=https://127.0.0.1:18443 ck "https + 错误 Key → 401(TLS 层下鉴权仍生效)" 401 \ "$(code -k -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Authorization: Bearer nope')" ck "https + alice → 200" 200 \ "$(code -k -H "Authorization: Bearer $ALICE_KEY" -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")" ck "https 响应体来自上游" yes \ "$(curl -sk -H "Authorization: Bearer $ALICE_KEY" -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY" | grep -q pong && echo yes || echo no)" else echo echo "【7】HTTPS:跳过(本机 nginx 无 http_ssl_module 或无 openssl)" fi echo echo "【8】★ 模型发现(网关本地应答,不去上游)" M0=$(up_count) ck "无 Key → 401" 401 "$(code $GW/v1/models)" ck "alice GET /v1/models → 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models)" ck " 形状是 OpenAI 的 object=list" yes \ "$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | has '"object":"list"')" ck " 含白名单内上游的模型 gpt-4o" yes \ "$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | has 'gpt-4o')" ck " 不含白名单外上游的模型 claude-3-5-sonnet" yes \ "$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | nhas 'claude-3-5-sonnet')" ck " 不含已停用上游的模型 should-not-appear" yes \ "$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | nhas 'should-not-appear')" ck "bob(白名单为空=不限)能看到 other 的模型" yes \ "$(curl -s -H "Authorization: Bearer $BOB_KEY" $GW/v1/models | has 'claude-3-5-sonnet')" ck "GET /models 同样本地应答 → 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/models)" ck "★整个模型发现过程 0 次上游调用" 0 "$(( $(up_count) - M0 ))" # 带 slot 的 /v1/models 是**另一条路**:照常透传给厂商,取"那家厂商的"模型清单 N0=$(up_count) ck "★//v1/models 仍透传上游 → 200" 200 \ "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/openai/v1/models)" ck " 上游确实收到了 1 次请求" 1 "$(( $(up_count) - N0 ))" ck " 转发路径未剥错(/v1/models)" "/v1/models" "$(up_field path)" ck " 响应体来自上游(不是网关本地那份)" yes \ "$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/openai/v1/models | has 'upstream-model')" echo echo "【9】★ 账号态 API:可用模型 + 自己的用量" ck "无 Key 查模型 → 401" 401 "$(code $API/api/v1/models)" ck "无 Key 查用量 → 401" 401 "$(code $API/api/v1/usage)" ck "alice 查模型 → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/models)" ck " 模型清单里带 base_url(lm 域名 + slot)" yes \ "$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has 'lm.mayishangshu.cn/openai')" # ★ base_url 必须**保留上游 base 的路径**:openai 的 base 无路径 → 无;other 的 base 带 /llm → 带 /llm。 # 搞错这一条会让"base 自带 /v1"的上游变成 /v1/v1/... → 上游 404 且响应体为空,极难排查。 ck "★base_url 不凭空拼 /v1(base 无路径就不带)" yes \ "$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | nhas 'lm.mayishangshu.cn/openai/')" ck "★base_url 保留上游 base 的路径 /llm" yes \ "$(curl -s -H "X-API-Key: $BOB_KEY" $API/api/v1/models | has 'lm.mayishangshu.cn/other/llm')" ALICE_USAGE=$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/usage) ck "alice 查用量 → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/usage)" ALICE_TOTAL=$(echo "$ALICE_USAGE" | jqv data.total) ck "★alice 累计用量 > 0(前面真发过请求)" yes \ "$(awk -v t="${ALICE_TOTAL:-0}" 'BEGIN{print (t>0)?"yes":"no"}')" ck " 今日 = 累计(都是刚发的)" "$ALICE_TOTAL" "$(echo "$ALICE_USAGE" | jqv data.today)" ck " 分上游明细含 openai" yes "$(echo "$ALICE_USAGE" | has '"slot": *"openai"')" ck "★被拒的越权/未知上游不进明细(无 other/downstream)" yes \ "$(echo "$ALICE_USAGE" | nhas '"slot": *"other"')" ck " 回显限额供客户端自我节流" yes "$(echo "$ALICE_USAGE" | has '"limits"')" # ★ 关键语义:统计的是"真的转发出去"的请求 —— 超限被 429 的那一发不计 CAROL_TOTAL=$(curl -s -H "X-API-Key: $CAROL_KEY" $API/api/v1/usage | jqv data.total) ck "★carol 发了 3 次、第 3 次被限流 → 用量只记 2" 2 "${CAROL_TOTAL:-x}" DAVE_TOTAL=$(curl -s -H "X-API-Key: $DAVE_KEY" $API/api/v1/usage | jqv data.total) ck "★dave 同上(每日配额)→ 用量只记 2" 2 "${DAVE_TOTAL:-x}" ck "bob 的明细含 other(他真调过)" yes \ "$(curl -s -H "X-API-Key: $BOB_KEY" $API/api/v1/usage | has '"slot": *"other"')" echo echo "【10】管理员用量总览 + 删账号清理用量键" ck "无会话 → 401" 401 "$(code $API/api/admin/usage)" ck "不是接口路径 → 404" 404 "$(code $API/api/v1/nope)" JAR=$RUN/cookie.jar LOGIN=$(curl -s -c "$JAR" -X POST $API/api/admin/login -H 'Content-Type: application/json' \ -d "$(jbody username admin password 123456)") ck "管理员登录 → 200" 200 "$(echo "$LOGIN" | jqv code)" USAGE=$(curl -s -b "$JAR" $API/api/admin/usage) ck "用量总览含 alice" yes "$(echo "$USAGE" | has '"username": *"alice"')" ck " 总览含按上游分组(openai)" yes "$(echo "$USAGE" | has '"slot": *"openai"')" ck " 全局合计 > 0" yes "$(echo "$USAGE" | jqv data.total.total | awk '{print ($1>0)?"yes":"no"}')" ck "★删除前用量榜里有 bob(他真调过 other)" yes "$(echo "$USAGE" | has '"username": *"bob"')" # ★ 删除账号必须**连带清掉用量键**(`ust:` / `ums:`)——它们是按前缀扫出来出报表的, # 不清就会让已删账号一直挂在用量榜上(与"删账号必须摘 ukey:"同类的正确性问题)。 curl -s -b "$JAR" -X POST $API/api/admin/users/delete -H 'Content-Type: application/json' \ -d "$(jbody username bob)" >/dev/null ck "★bob 已被删除(他的 Key 立即失效 → 401)" 401 "$(code -H "X-API-Key: $BOB_KEY" $API/api/v1/usage)" USAGE2=$(curl -s -b "$JAR" $API/api/admin/usage) ck "★删除后用量榜不再出现 bob" yes "$(echo "$USAGE2" | nhas 'bob')" ck " 但 alice 仍在榜单上" yes "$(echo "$USAGE2" | has '"username": *"alice"')" echo echo "============================================================" printf " 通过 \033[32m%d\033[0m 失败 \033[31m%d\033[0m\n" "$pass" "$fail" echo "============================================================" [ "$fail" -eq 0 ] || exit 1