#!/bin/bash # ============================================================================ # LLM 代理 · 本机回归 # # 一般不用直接跑,走统一入口: bash tests/run.sh llm # 直接跑需自行保证:假上游(tests/llm/upstream.py)已在 18999 监听, # nginx 已按 tests/llm/nginx.conf.in 渲染并启动,且 data/ 用的是 fixtures 里 # 那两个固定文件(alice 白名单=openai、bob 白名单为空=不限)。 # # ★ 核心:不只验网关自己的响应码,而是**读靶机实际收到的请求**—— # 「下游 Key 是否真被换成上游真 Key」只有看靶机才能证明。 # # 可覆盖:GW / UPSTREAM_LOG / PY # ============================================================================ set -u HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) RUN=${RUN:-$(cd "$HERE/.." && pwd)/.run/llm} GW=${GW:-http://127.0.0.1:18090} # 管理端口(同一份 nginx / 同一块共享内存)——用量与模型清单从这边读 API=${API:-http://127.0.0.1:18091} UP_LOG=${UPSTREAM_LOG:-$RUN/upstream.log} PY=${PY:-python3} # fixtures 里的固定 Key(见 fixtures/users.json) ALICE_KEY=sk_alice_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa BOB_KEY=sk_bob_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb # 多账号路由验证用 admin:白名单为空=不限、无配额限制、且本套件中途会删除 bob 故不复用 ADMIN_KEY=sk_admin_0000000000000000000000000000000000000000000000ab pass=0; fail=0 ck() { if [ "$2" = "$3" ]; then printf " \033[32m✓\033[0m %s\n" "$1"; pass=$((pass+1)) else printf " \033[31m✗\033[0m %s 期望=[%s] 实际=[%s]\n" "$1" "$2" "$3"; fail=$((fail+1)) fi } code() { curl -s -o /dev/null -w '%{http_code}' "$@"; } jbody() { local out='{' sep=''; while [ $# -gt 1 ]; do out="$out$sep\"$1\":\"$2\""; sep=','; shift 2; done; printf '%s}' "$out"; } # 靶机日志:取最后一条记录的某个字段 up_field() { "$PY" -c " import json, sys last = None try: for l in open(sys.argv[1], encoding='utf-8'): l = l.strip() if l: last = l except FileNotFoundError: pass print(json.loads(last).get(sys.argv[2], '') if last else '') " "$UP_LOG" "$1" 2>/dev/null } up_count() { if [ -f "$UP_LOG" ]; then awk 'NF{n++} END{print n+0}' "$UP_LOG"; else echo 0; fi } # 靶机日志:取最后一条记录里**某个请求头**的值(头名大小写不敏感)。 # 专门为「出站头策略」服务:要证明某个头没被发出去 / 某个头还活着, # 只能看上游实际收到了什么。取不到的空值 = 该头确实没发。 up_hdr() { "$PY" -c " import json, sys last = None try: for l in open(sys.argv[1], encoding='utf-8'): l = l.strip() if l: last = l except FileNotFoundError: pass h = (json.loads(last).get('headers') or {}) if last else {} print(h.get(sys.argv[2].lower(), '')) " "$UP_LOG" "$1" 2>/dev/null } post() { curl -s -H "Authorization: Bearer $1" -X POST "$GW$2" -H 'Content-Type: application/json' -d "$3"; } # 从 stdin 的 JSON 里按点分路径取值 jqv() { "$PY" -c " import sys, json try: d = json.load(sys.stdin) except Exception: print(''); sys.exit() for k in sys.argv[1].split('.'): if d is None: break d = d[int(k)] if k.isdigit() and isinstance(d, list) else (d.get(k) if isinstance(d, dict) else None) print('' if d is None else d) " "$1" 2>/dev/null } # 在 stdin 里找子串(命中输出 yes,否则 no) has() { if grep -q "$1"; then echo yes; else echo no; fi; } # 反向断言:**不**出现该子串才算过 nhas() { if grep -q "$1"; then echo no; else echo yes; fi; } echo "【1】鉴权(OpenAI 兼容错误体)" BODY=$(jbody model gpt-4o) ck "无 Key → 401" 401 "$(code -X POST $GW/openai/v1/chat/completions)" ck "错误 Key → 401" 401 "$(code -X POST $GW/openai/v1/chat/completions -H 'Authorization: Bearer sk_deadbeef')" ck "错误体是 OpenAI 形状(含 error.type)" yes \ "$(curl -s -X POST $GW/openai/v1/chat/completions | grep -q '"error"' && echo yes || echo no)" ck "路径缺 slot → 400" 400 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/)" ck "未知上游 → 404" 404 "$(code -H "Authorization: Bearer $BOB_KEY" -X POST $GW/nope/v1/chat/completions)" echo echo "【2】上游解析与白名单" ck "alice + openai(在白名单)→ 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")" ck "响应体来自上游(含 pong)" yes "$(post "$ALICE_KEY" /openai/v1/chat/completions "$BODY" | grep -q pong && echo yes || echo no)" ck "alice + other(不在白名单)→ 403" 403 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/other/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")" ck "alice + downstream(已停用)→ 404" 404 "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/downstream/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")" ck "bob + other(空白名单=不限)→ 200" 200 "$(code -H "Authorization: Bearer $BOB_KEY" -X POST $GW/other/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")" echo echo "【3】★ Key 替换 / 路径 / 头(读靶机实际收到的请求)" N0=$(up_count) post "$ALICE_KEY" "/openai/v1/chat/completions?a=1" "$BODY" >/dev/null ck "靶机恰好收到 1 次转发" 1 "$(( $(up_count) - N0 ))" ck "★Authorization 已换成上游真 Key" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)" ck "★下游 Key 未泄漏给上游" yes "$(up_field auth | grep -q 'sk_alice_' && echo leaked || echo yes)" ck "Host 头 = 上游主机(非下游域名)" "127.0.0.1:18999" "$(up_field host)" ck "★路径去 slot 且保留 query" "/v1/chat/completions?a=1" "$(up_field path)" ck "请求体原样透传" "$BODY" "$(up_field body)" # ★ 出站头策略(对齐 LiteLLM 的两层:默认不放行 + 凭据头显式抹空)。 # 构造最坏情形:客户端把**网关自己的 Key** 塞进每一个凭据头, # 同时带上两套协议的扩展头,然后看上游究竟收到了什么。 # 注意这里**故意不发 Authorization**,顺带验证「入站只靠 x-api-key 也能过网关」。 curl -s -o /dev/null -X POST "$GW/openai/v1/chat/completions" \ -H "x-api-key: $ALICE_KEY" \ -H "api-key: $ALICE_KEY" \ -H "x-goog-api-key: $ALICE_KEY" \ -H 'anthropic-version: 2023-06-01' \ -H 'anthropic-beta: prompt-caching-2024-07-31' \ -H 'openai-beta: assistants=v2' \ -H 'Content-Type: application/json' -d "$BODY" ck "★x-api-key 未泄漏给上游(默认摘除清单)" "" "$(up_hdr x-api-key)" ck "★api-key 未泄漏给上游" "" "$(up_hdr api-key)" ck "★x-goog-api-key 未泄漏给上游" "" "$(up_hdr x-goog-api-key)" ck "★Authorization 仍是上游真 Key(客户端塞不进凭据)" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)" ck "★Content-Type 已注入(漏了所有 POST 都会废)" "application/json" "$(up_hdr content-type)" ck " Accept 已注入(SSE 协商要靠它)" "*/*" "$(up_hdr accept)" ck "★anthropic-version 已注入(Anthropic 协议必填)" "2023-06-01" "$(up_hdr anthropic-version)" ck "★anthropic-beta 已注入(Claude Code 扩展依赖)" "prompt-caching-2024-07-31" "$(up_hdr anthropic-beta)" ck " openai-beta 已注入" "assistants=v2" "$(up_hdr openai-beta)" ck " User-Agent 已透传(厂商侧排障要靠它)" yes \ "$([ -n "$(up_hdr user-agent)" ] && echo yes || echo no)" echo echo "【4】上游 base 带路径时的拼接" post "$BOB_KEY" /other/v1/chat/completions "$BODY" >/dev/null ck "★base=/llm + path 正确拼接" "/llm/v1/chat/completions" "$(up_field path)" ck "用的是该上游自己的 Key" "Bearer sk-OTHER-UPSTREAM" "$(up_field auth)" # ★ 版本段去重(实测踩过的坑:厂商 base_url 自带 /v1,客户端也带 /v1 → /v1/v1 → 上游 404 空响应) post "$BOB_KEY" /deepseeklike/v1/chat/completions "$BODY" >/dev/null ck "★base 自带 /v1 + 客户端带 /v1 → 只保留一份" "/v1/chat/completions" "$(up_field path)" post "$BOB_KEY" /deepseeklike/chat/completions "$BODY" >/dev/null ck " 客户端不带 /v1 → 照常前缀上 base 的 /v1" "/v1/chat/completions" "$(up_field path)" post "$BOB_KEY" /deepseeklike/v2/chat/completions "$BODY" >/dev/null ck " 不同版本段不误判为重复" "/v1/v2/chat/completions" "$(up_field path)" # ★ 「统一注册」= base 只写到 host,一条记录同时服务两套协议。 # 这是 LiteLLM(/anthropic/* 前缀)/ one-api(根地址 + 路径天然区分)/ # Higress(按路径后缀判协议)三个开源实现的共同做法: # **协议由客户端路径表达,网关不做任何推断、不需要分发代码**。 # 夹具里 openai 的 base = http://127.0.0.1:18999(纯 host),正好验这条。 post "$ALICE_KEY" "/openai/anthropic/v1/messages" "$BODY" >/dev/null ck "★Anthropic 路径原样拼在 host 粒度 base 之后" "/anthropic/v1/messages" "$(up_field path)" ck " 同一条记录也换上游真 Key(不是只有 OpenAI 路径才换)" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)" echo echo "【5】★ 流式(SSE 不被缓冲)" SSE=$(GW="$GW" KEY="$ALICE_KEY" "$PY" "$HERE/sse_test.py" 2>&1) ck "Content-Type 是 text/event-stream" "text/event-stream" "$(echo "$SSE" | sed -n 's/^CT=//p')" ck "收到 3 个 data 块" 3 "$(echo "$SSE" | sed -n 's/^CHUNKS=//p')" FIRST=$(echo "$SSE" | sed -n 's/^FIRST=//p') TOTAL=$(echo "$SSE" | sed -n 's/^TOTAL=//p') ck "★首块 < 0.5s 到达(未被攒包)" yes "$(awk -v f="${FIRST:-9}" 'BEGIN{print (f<0.5)?"yes":"no"}')" ck "总耗时 ≥ 1.0s(确实分了 3 次推)" yes "$(awk -v t="${TOTAL:-0}" 'BEGIN{print (t>=1.0)?"yes":"no"}')" echo echo "【6】★ 限流与每日配额(carol rpm=2、dave daily=2,见 fixtures/users.json)" CAROL_KEY=sk_carol_cccccccccccccccccccccccccccccccccccccccccccccc DAVE_KEY=sk_dave_dddddddddddddddddddddddddddddddddddddddddddddddd cpost() { code -H "Authorization: Bearer $1" -X POST "$GW/other/v1/chat/completions" -H 'Content-Type: application/json' -d "$BODY"; } ck "carol 第 1 发 → 200" 200 "$(cpost "$CAROL_KEY")" ck "carol 第 2 发 → 200" 200 "$(cpost "$CAROL_KEY")" ck "carol 第 3 发 → 429(rpm=2)" 429 "$(cpost "$CAROL_KEY")" ck "429 是 rate_limit_error / rate_limit_exceeded" yes \ "$(post "$CAROL_KEY" /other/v1/chat/completions "$BODY" | grep -q 'rate_limit' && echo yes || echo no)" ck "dave 第 1 发 → 200" 200 "$(cpost "$DAVE_KEY")" ck "dave 第 2 发 → 200" 200 "$(cpost "$DAVE_KEY")" ck "dave 第 3 发 → 429(daily=2)" 429 "$(cpost "$DAVE_KEY")" ck "429 是 insufficient_quota" yes \ "$(post "$DAVE_KEY" /other/v1/chat/completions "$BODY" | grep -q 'insufficient_quota' && echo yes || echo no)" ck "carol 被限后 alice 不受影响(按 Key 隔离)→ 200" 200 \ "$(code -H "Authorization: Bearer $ALICE_KEY" -X POST "$GW/openai/v1/chat/completions" -H 'Content-Type: application/json' -d "$BODY")" if [ "${HAVE_TLS:-0}" = "1" ]; then echo echo "【7】★ HTTPS(18443 自签,模板 tests/llm/tls.conf.in)" HTTPS_BASE=https://127.0.0.1:18443 ck "https + 错误 Key → 401(TLS 层下鉴权仍生效)" 401 \ "$(code -k -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Authorization: Bearer nope')" ck "https + alice → 200" 200 \ "$(code -k -H "Authorization: Bearer $ALICE_KEY" -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY")" ck "https 响应体来自上游" yes \ "$(curl -sk -H "Authorization: Bearer $ALICE_KEY" -X POST $HTTPS_BASE/openai/v1/chat/completions -H 'Content-Type: application/json' -d "$BODY" | grep -q pong && echo yes || echo no)" else echo echo "【7】HTTPS:跳过(本机 nginx 无 http_ssl_module 或无 openssl)" fi echo echo "【8】★ 模型发现(网关本地应答,不去上游)" M0=$(up_count) ck "无 Key → 401" 401 "$(code $GW/v1/models)" ck "alice GET /v1/models → 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models)" ck " 形状是 OpenAI 的 object=list" yes \ "$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | has '"object":"list"')" ck " 含白名单内上游的模型 gpt-4o" yes \ "$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | has 'gpt-4o')" ck " 不含白名单外上游的模型 claude-3-5-sonnet" yes \ "$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | nhas 'claude-3-5-sonnet')" ck " 不含已停用上游的模型 should-not-appear" yes \ "$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/v1/models | nhas 'should-not-appear')" ck "bob(白名单为空=不限)能看到 other 的模型" yes \ "$(curl -s -H "Authorization: Bearer $BOB_KEY" $GW/v1/models | has 'claude-3-5-sonnet')" ck "GET /models 同样本地应答 → 200" 200 "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/models)" ck "★整个模型发现过程 0 次上游调用" 0 "$(( $(up_count) - M0 ))" # 带 slot 的 /v1/models 是**另一条路**:照常透传给厂商,取"那家厂商的"模型清单 N0=$(up_count) ck "★//v1/models 仍透传上游 → 200" 200 \ "$(code -H "Authorization: Bearer $ALICE_KEY" $GW/openai/v1/models)" ck " 上游确实收到了 1 次请求" 1 "$(( $(up_count) - N0 ))" ck " 转发路径未剥错(/v1/models)" "/v1/models" "$(up_field path)" ck " 响应体来自上游(不是网关本地那份)" yes \ "$(curl -s -H "Authorization: Bearer $ALICE_KEY" $GW/openai/v1/models | has 'upstream-model')" echo echo "【9】★ 账号态 API:可用模型 + 自己的用量" # base_url 跟随**调用方所用的端口**(88 部署必须带 :88;443 反代后面则不带)—— # 见 rust/src/upstreams.rs 的 base_url 注释,写死任何一种都会在另一种部署里给出错地址。 API_PORT="${API##*:}" ck "无 Key 查模型 → 401" 401 "$(code $API/api/v1/models)" ck "无 Key 查用量 → 401" 401 "$(code $API/api/v1/usage)" ck "alice 查模型 → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/models)" ck " 模型清单里带 base_url(lm 域名 + 端口 + slot)" yes \ "$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has "lm.mayishangshu.cn:$API_PORT/openai")" # ★ base_url 必须**保留上游 base 的路径**:openai 的 base 无路径 → 无;other 的 base 带 /llm → 带 /llm。 # 搞错这一条会让"base 自带 /v1"的上游变成 /v1/v1/... → 上游 404 且响应体为空,极难排查。 # (校验 base_url 字段的**完整值**而不是整份响应的子串——后者会被同段新增的 # anthropic_base_url(…/openai/anthropic)误伤。) ck "★base_url 不凭空拼 /v1(base 无路径就不带)" yes \ "$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/models | has "\"base_url\":\"https://lm.mayishangshu.cn:$API_PORT/openai\"")" ck "★base_url 保留上游 base 的路径 /llm" yes \ "$(curl -s -H "X-API-Key: $BOB_KEY" $API/api/v1/models | has "lm.mayishangshu.cn:$API_PORT/other/llm")" # ★ 端口自适应:Host 不带端口(走 443 反代)时 base_url 也不该带端口。 # 写死 88 → 443 部署下客户端连到另一个端口的服务;裸域名 → 88 部署下被出口设备抢答 # (实测 HTTP 200 + 一个 HTML 登录页,是最难看出问题的一种失败)。 ck "★Host 无端口 → base_url 不带端口" yes \ "$(curl -s -H "X-API-Key: $ALICE_KEY" -H 'Host: lm.example' $API/api/v1/models | has 'lm.mayishangshu.cn/openai')" ck "★Host 带端口 → base_url 带同一端口" yes \ "$(curl -s -H "X-API-Key: $ALICE_KEY" -H "Host: lm.example:$API_PORT" $API/api/v1/models | has "lm.mayishangshu.cn:$API_PORT/openai")" ALICE_USAGE=$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/usage) ck "alice 查用量 → 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" $API/api/v1/usage)" ALICE_TOTAL=$(echo "$ALICE_USAGE" | jqv data.total) ck "★alice 累计用量 > 0(前面真发过请求)" yes \ "$(awk -v t="${ALICE_TOTAL:-0}" 'BEGIN{print (t>0)?"yes":"no"}')" ck " 今日 = 累计(都是刚发的)" "$ALICE_TOTAL" "$(echo "$ALICE_USAGE" | jqv data.today)" ck " 分上游明细含 openai" yes "$(echo "$ALICE_USAGE" | has '"slot": *"openai"')" ck "★被拒的越权/未知上游不进明细(无 other/downstream)" yes \ "$(echo "$ALICE_USAGE" | nhas '"slot": *"other"')" ck " 回显限额供客户端自我节流" yes "$(echo "$ALICE_USAGE" | has '"limits"')" # ★ 关键语义:统计的是"真的转发出去"的请求 —— 超限被 429 的那一发不计 CAROL_TOTAL=$(curl -s -H "X-API-Key: $CAROL_KEY" $API/api/v1/usage | jqv data.total) ck "★carol 发了 3 次、第 3 次被限流 → 用量只记 2" 2 "${CAROL_TOTAL:-x}" DAVE_TOTAL=$(curl -s -H "X-API-Key: $DAVE_KEY" $API/api/v1/usage | jqv data.total) ck "★dave 同上(每日配额)→ 用量只记 2" 2 "${DAVE_TOTAL:-x}" ck "bob 的明细含 other(他真调过)" yes \ "$(curl -s -H "X-API-Key: $BOB_KEY" $API/api/v1/usage | has '"slot": *"other"')" echo echo "【10】管理员用量总览 + 删账号清理用量键" ck "无会话 → 401" 401 "$(code $API/api/admin/usage)" ck "不是接口路径 → 404" 404 "$(code $API/api/v1/nope)" JAR=$RUN/cookie.jar LOGIN=$(curl -s -c "$JAR" -X POST $API/api/admin/login -H 'Content-Type: application/json' \ -d "$(jbody username admin password 123456)") ck "管理员登录 → 200" 200 "$(echo "$LOGIN" | jqv code)" USAGE=$(curl -s -b "$JAR" $API/api/admin/usage) ck "用量总览含 alice" yes "$(echo "$USAGE" | has '"username": *"alice"')" ck " 总览含按上游分组(openai)" yes "$(echo "$USAGE" | has '"slot": *"openai"')" ck " 全局合计 > 0" yes "$(echo "$USAGE" | jqv data.total.total | awk '{print ($1>0)?"yes":"no"}')" ck "★删除前用量榜里有 bob(他真调过 other)" yes "$(echo "$USAGE" | has '"username": *"bob"')" # ★ 删除账号必须**连带清掉用量键**(`ust:` / `ums:`)——它们是按前缀扫出来出报表的, # 不清就会让已删账号一直挂在用量榜上(与"删账号必须摘 ukey:"同类的正确性问题)。 curl -s -b "$JAR" -X POST $API/api/admin/users/delete -H 'Content-Type: application/json' \ -d "$(jbody username bob)" >/dev/null ck "★bob 已被删除(他的 Key 立即失效 → 401)" 401 "$(code -H "X-API-Key: $BOB_KEY" $API/api/v1/usage)" USAGE2=$(curl -s -b "$JAR" $API/api/admin/usage) ck "★删除后用量榜不再出现 bob" yes "$(echo "$USAGE2" | nhas 'bob')" ck " 但 alice 仍在榜单上" yes "$(echo "$USAGE2" | has '"username": *"alice"')" echo echo "【11】★ 上游连通性测试(管理台「测试」按钮的后端链路)" # 管理台把探测体原样发上来,网关只负责「定位上游 + 换真 Key + 转发 + 透传状态」。 # 判据照旧:不只看 HTTP 200,还要看靶机实际收到的 path / auth。 PING='{"model":"ds-chat","max_tokens":1,"stream":false,"messages":[{"role":"user","content":"ping"}]}' ck "无会话测试 → 401" 401 \ "$(code -X POST "$API/api/admin/upstreams/test?slot=openai" -d '{}')" ck "★admin 测 openai → 透传上游 200" 200 \ "$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=openai" -H 'Content-Type: application/json' -d "$PING")" ck " 靶机收到的 path 不含 slot 段" "/v1/chat/completions" "$(up_field path)" ck " 换上了 openai 上游的真 Key" "Bearer sk-UPSTREAM-REAL-SECRET" "$(up_field auth)" ck "★base 自带 /v1 的上游(deepseeklike)→ 去重后 200" 200 \ "$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=deepseeklike" -H 'Content-Type: application/json' -d "$PING")" ck " 靶机 path 仍是 /v1/chat/completions(无 /v1/v1、无 slot 段)" "/v1/chat/completions" "$(up_field path)" ck " 换上了 deepseeklike 的真 Key" "Bearer sk-DS-REAL" "$(up_field auth)" ck "★已停用的上游测不了(shm 已摘除 → 404)" 404 \ "$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=downstream" -d '{}')" ck "不存在的 slot → 404 信封" 404 \ "$(code -b "$JAR" -X POST "$API/api/admin/upstreams/test?slot=ghost" -d '{}')" ck "★内部 location 外部直连 → 404(不能绕过 admin 会话触达上游)" 404 \ "$(code -b "$JAR" -X POST "$API/_upstream_test/openai/v1/chat/completions" -d '{}')" echo echo "【11.5】★ API 代理(/apix/ → 外部供应商)" # 与 LLM 代理同一把 sk_ Key,但上游形态不同:供应商只有一个收口,资产由 # **请求体里的 asset_id** 决定,所以请求体必须由网关拼——判据是看靶机 # 收到的 path / 头 / body 三件事,只看网关自己的 200 什么都证明不了。 ck "无 Key → 401" 401 "$(code "$GW/apix/search?q=hello")" ck "坏 Key → 401" 401 "$(code -H 'X-API-Key: sk_nope' "$GW/apix/search?q=hello")" ck "不存在的 slug → 404" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/ghost?q=1")" ck "★已停用的 slug → 404(shm 里已摘除)" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/offline?q=1")" ck "★正常调用 → 透传上游 200" 200 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/search?q=hello")" ck " 靶机 path = 供应商的收口" "/api/v1/ai/request" "$(up_field path)" ck " ★GET 被改写成 POST(proxy_method + proxy_set_body)" "POST" "$(up_field method)" ck " ★换上了供应商真 Key" "ak_test_supplier_key" "$(up_hdr x-api-key)" ck " ★调用体由网关拼:asset_id 来自配置" yes \ "$(up_field body | grep -q '"asset_id":"searxng-search"' && echo yes || echo no)" ck " ★query 参数进 call_data" yes \ "$(up_field body | grep -q '"q":"hello"' && echo yes || echo no)" ck " ★下行 Key 没被带出去" yes "$(up_field body | nhas 'sk_alice')" ck " 下游 Authorization 未透传" "" "$(up_hdr authorization)" # x-api-key 头**会**有值——那正是换上的供应商真 Key。要证的是它不再以下游的 # sk_ 开头:客户端用自己的 Key 换来供应商凭据,而不是凭据被原样透传。 AXH=$(up_hdr x-api-key) ck " ★下游 x-api-key 已被换掉(不再是 sk_ 开头)" no \ "$(printf '%s' "$AXH" | grep -q '^sk_' && echo yes || echo no)" # 管理台:清单 / 凭据掩码 / 停启用写穿 ck "管理台列表含 search" yes "$(curl -s -b "$JAR" $API/api/admin/apix | has '"slug": *"search"')" ck " ★条目 Key 只回显掩码(不回显真值)" no \ "$(curl -s -b "$JAR" $API/api/admin/apix | has 'ak_test_supplier_key')" ck " 条目 base 回显" "http://127.0.0.1:18999" \ "$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c ' import sys, json d = json.load(sys.stdin)["data"] it = next((x for x in d["items"] if x["slug"] == "search"), None) print(it.get("base") if it else "miss")')" ck " 供应商名回显" "vendor-a" \ "$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c ' import sys, json d = json.load(sys.stdin)["data"] it = next((x for x in d["items"] if x["slug"] == "search"), None) print(it.get("supplier") if it else "miss")')" ck " 已停用条目 enabled=false" "False" "$(curl -s -b "$JAR" $API/api/admin/apix | "$PY" -c ' import sys, json d = json.load(sys.stdin)["data"] it = next((x for x in d["items"] if x["slug"] == "offline"), None) print(str(it.get("enabled")) if it else "miss")')" ck "账号态接口只列已启用的(offline 不出现)" "1" \ "$(curl -s -H "X-API-Key: $ALICE_KEY" $API/api/v1/apix | jqv data.count)" ck "非法 slug → 400" 400 "$(code -b "$JAR" -X POST $API/api/admin/apix/save \ -H 'Content-Type: application/json' -d "$(jbody slug 'Bad Slug!' asset_id x)")" ck "缺 asset_id → 400" 400 "$(code -b "$JAR" -X POST $API/api/admin/apix/save \ -H 'Content-Type: application/json' -d "$(jbody slug med asset_id '')")" ck "新建代理 med → 200" 200 "$(code -b "$JAR" -X POST $API/api/admin/apix/save \ -H 'Content-Type: application/json' -d "$(jbody slug med asset_id asset_26476ef1-4687-45 name 疾病百科 params kw base http://127.0.0.1:18999 key ak_test_supplier_key supplier vendor-a)")" ck " ★新代理立即可用(写穿 shm)" 200 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/med?kw=感冒")" ck " 靶机 body 带新 asset_id 与新参数" yes \ "$(up_field body | grep -q '"asset_id":"asset_26476ef1-4687-45"' && echo yes || echo no)" ck " 参数名用的是 med 声明的 kw" yes "$(up_field body | grep -q '"kw":"' && echo yes || echo no)" curl -s -b "$JAR" -X POST $API/api/admin/apix/save -H 'Content-Type: application/json' \ -d "$(jbody slug search asset_id searxng-search enabled false)" >/dev/null ck "★停用后立即 404(写穿 shm,无回源兜底)" 404 \ "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/search?q=1")" curl -s -b "$JAR" -X POST $API/api/admin/apix/save -H 'Content-Type: application/json' \ -d "$(jbody slug search asset_id searxng-search enabled true)" >/dev/null ck " 重新启用后又能调通" 200 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/search?q=1")" curl -s -b "$JAR" -X POST $API/api/admin/apix/delete -H 'Content-Type: application/json' \ -d "$(jbody slug med)" >/dev/null ck "删除后再调 → 404" 404 "$(code -H "X-API-Key: $ALICE_KEY" "$GW/apix/med?kw=x")" # 探测:走供应商的**免费**预估接口,不执行资产、不扣费 ck "★探测(免费预估)→ 透传 200" 200 "$(code -b "$JAR" -X POST \ "$API/api/admin/apix/test?slug=search" -H 'Content-Type: application/json' -d '{}')" ck " 探测打的是 /api/v1/ai/estimate" "/api/v1/ai/estimate" "$(up_field path)" ck " 探测也换了供应商真 Key" "ak_test_supplier_key" "$(up_hdr x-api-key)" ck "★内部探测口外部直连 → 404" 404 \ "$(code -b "$JAR" -X POST "$API/_apix_probe/search" -d '{}')" ck " 不存在的 slug 探测 → 404" 404 "$(code -b "$JAR" -X POST \ "$API/api/admin/apix/test?slug=ghost" -H 'Content-Type: application/json' -d '{}')" ck "★用量统计按 api/ 归组(与 LLM 上游天然分开)" yes \ "$(curl -s -b "$JAR" $API/api/admin/usage | has '"slot": *"api/search"')" ck "★趋势含 api/ 维度" yes "$(curl -s -b "$JAR" "$API/api/admin/trend?days=7" | has '"slot": *"api/search"')" echo echo "【12】★ 上游健康标记(失效自动缓存 + 模型发现附可用地址/状态)" # 健康缓存由 log 阶段的**真实转发**写入(uh:)——「测试」按钮走同一条 # proxy_pass 链路,探测结果也天然更新缓存,无需第二套标记逻辑。 # 语义:上游回了头 = 活着(4xx 是它对请求内容的意见,不是机器死了); # 5xx / 连不上(502) / 等头超时(504) = 失效;没被任何请求打过 = unknown。 # 缓存只用于展示(列表标红 / 模型发现附状态),转发路径不读它。 up_field_of() { # $1=slot $2=字段 → 取上游列表里该 slot 的字段值 curl -s -b "$JAR" $API/api/admin/upstreams | "$PY" -c ' import sys, json d = json.load(sys.stdin) items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or []) u = next((x for x in items if x.get("slot") == sys.argv[1]), None) v = u.get(sys.argv[2]) if u else "miss" print("miss" if v is None else v) ' "$1" "$2" } xm_status() { # /v1/models 扩展块里某 slot 的健康状态 curl -s -H "Authorization: Bearer $HKEY" $GW/v1/models | "$PY" -c ' import sys, json d = json.load(sys.stdin) u = next((x for x in d.get("x_ipass_upstreams", []) if x.get("slot") == sys.argv[1]), {}) print(u.get("status", "miss")) ' "$1" } ck "创建临时上游 health1 → 200" 200 \ "$(code -b "$JAR" -X POST $API/api/admin/upstreams/save -H 'Content-Type: application/json' \ -d '{"slot":"health1","base":"http://127.0.0.1:18999","auth":"Bearer sk-H1","models":["m1"]}')" ck "★初始状态 = unknown(没被任何真实请求打过)" unknown "$(up_field_of health1 status)" # 临时账号(空白名单 = 不限)换一把干净 Key 来打靶机注入的 500 curl -s -b "$JAR" -X POST $API/api/admin/users/create -H 'Content-Type: application/json' \ -d "$(jbody username healthprobe password 123456 display_name hp)" >/dev/null HKEY=$(curl -s -b "$JAR" $API/api/admin/users | "$PY" -c ' import sys, json d = json.load(sys.stdin) items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or []) print(next((u["api_key"] for u in items if u.get("username") == "healthprobe"), "")) ') BOOM='{"model":"m1","max_tokens":1,"boom":true,"messages":[{"role":"user","content":"x"}]}' ck "boom 请求 → 靶机注入的 500 原样透传给客户端" 500 \ "$(code -H "Authorization: Bearer $HKEY" -X POST $GW/health1/v1/chat/completions \ -H 'Content-Type: application/json' -d "$BOOM")" sleep 0.3 ck "★上游 500 → health1 自动标记为 failed(无需人工探测)" failed "$(up_field_of health1 status)" ck " 缓存里记录的状态码 = 500" 500 "$(up_field_of health1 status_code)" ck "★/v1/models 顶层附 x_ipass_upstreams(地址 + 模型 + 状态)" yes \ "$(curl -s -H "Authorization: Bearer $HKEY" $GW/v1/models | has '"x_ipass_upstreams"')" ck " 其中 health1 同样标 failed" failed "$(xm_status health1)" ck " 给出 anthropic 形式的 base_url" yes \ "$(curl -s -H "Authorization: Bearer $HKEY" $GW/v1/models | has '"anthropic_base_url"')" ck "★账号态 /api/v1/upstreams 也带状态与双协议地址" yes \ "$(curl -s -H "X-API-Key: $HKEY" $API/api/v1/upstreams | has '"anthropic_base_url"')" OKBODY='{"model":"m1","max_tokens":1,"messages":[{"role":"user","content":"ping"}]}' ck "恢复:正常请求 → 200" 200 \ "$(code -H "Authorization: Bearer $HKEY" -X POST $GW/health1/v1/chat/completions \ -H 'Content-Type: application/json' -d "$OKBODY")" sleep 0.3 ck "★下次成功转发自动转回 ok(自愈,无需清缓存)" ok "$(up_field_of health1 status)" # 清理临时数据 curl -s -b "$JAR" -X POST $API/api/admin/upstreams/delete -H 'Content-Type: application/json' \ -d "$(jbody slot health1)" >/dev/null ck "删除后列表不再出现 health1" miss "$(up_field_of health1 status)" curl -s -b "$JAR" -X POST $API/api/admin/users/delete -H 'Content-Type: application/json' \ -d "$(jbody username healthprobe)" >/dev/null echo echo "【9】对话审计(audit/ 目录:请求体落盘 + meta 索引 + 被拒不入档)" AUDIT=$RUN/audit sleep 0.3 audit_n() { find "$AUDIT" -name '*.json' 2>/dev/null | wc -l | tr -d ' '; } N1=$(audit_n) ck "转发的请求都落了审计文件" yes "$([ "$N1" -ge 3 ] && echo yes || echo no)" ck "meta.jsonl 记录账号/上游/状态" yes \ "$(grep -q '"user":"alice"' "$AUDIT"/*/meta.jsonl 2>/dev/null && grep -q '"slot":"openai"' "$AUDIT"/*/meta.jsonl && grep -q '"status":200' "$AUDIT"/*/meta.jsonl && echo yes || echo no)" # 取 alice 一条成功记录,比对审计文件内容 = 原始请求体 AF=$("$PY" -c " import json, sys for l in open(sys.argv[1], encoding='utf-8'): d = json.loads(l) if d.get('user') == 'alice' and d.get('status') == 200: print(d['file']); break " "$AUDIT"/*/meta.jsonl 2>/dev/null | head -1) ck "审计文件内容 = 客户端原始请求体" "$BODY" "$(cat "$AUDIT/$AF" 2>/dev/null)" N2=$(audit_n) code -H "Authorization: Bearer sk_deadbeef" -X POST $GW/openai/v1/chat/completions \ -H 'Content-Type: application/json' -d "$BODY" >/dev/null sleep 0.3 ck "被拒请求(401)不入档" 0 "$(( $(audit_n) - N2 ))" echo "【10】token 计量(body filter 旁路抓 usage → utt:/utms:)" # 非 stream usage 固定 7/3;带 usage_probe 的流式在 SSE 尾部给 11/22 curl -s -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \ -H 'Content-Type: application/json' -d "$BODY" >/dev/null curl -s -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/completions \ -H 'Content-Type: application/json' -d '{"model":"gpt-4o","stream":true,"usage_probe":true}' >/dev/null sleep 0.3 TOK=$(curl -s -b "$JAR" $API/api/admin/usage) echo "【多账号路由】粘性绑定 + 额度耗尽自动切换" # 用 admin(白名单为空=不限;本套件中途会删除 bob,故不复用 bob) ck "admin 基础鉴权(openai,白名单不限)→ 200" 200 "$(code -H "Authorization: Bearer $ADMIN_KEY" -X POST $GW/openai/v1/chat/completions -H 'Content-Type: application/json' -d '{"model":"gpt-4o","messages":[{"role":"user","content":"hi"}],"max_tokens":8}')" acct_of() { curl -s -D - -o /dev/null -X POST "$GW/$1/v1/chat/completions" \ -H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \ -d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}' \ | grep -i '^X-Upstream-Acct' | tr -d '\r' | awk '{print $2}'; } code_of() { curl -s -o /dev/null -w '%{http_code}' -X POST "$GW/$1/v1/chat/completions" \ -H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \ -d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}'; } ck "sticky 首次请求命中 acct=0" 0 "$(acct_of sticky sticky-model)" ck "sticky 首次请求 200" 200 "$(code_of sticky sticky-model)" ck "sticky 粘性:第 2 次仍 acct=0" 0 "$(acct_of sticky sticky-model)" ck "sticky 粘性:第 3 次仍 acct=0" 0 "$(acct_of sticky sticky-model)" # 切换:multi slot accounts[0]=bad(命中假上游 UPSTREAM_FAIL_AUTH → 401+marker)→ 自动切 acct=1 mh() { curl -s -D - -o /dev/null -X POST "$GW/$1/v1/chat/completions" \ -H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \ -d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}' \ | grep -iE '^HTTP/|^X-Upstream-Acct' | tr -d '\r'; } M1=$(mh multi multi-model) ck "multi 初次请求打到 acct=0(bad)" 0 "$(echo "$M1" | awk '/X-Upstream-Acct/{print $2}')" ck "multi 初次请求 401(触发冷却)" 401 "$(echo "$M1" | awk '/^HTTP/{print $2}')" M2=$(mh multi multi-model) ck "multi 冷却中切到 acct=1" 1 "$(echo "$M2" | awk '/X-Upstream-Acct/{print $2}')" ck "multi 切到 acct=1 后 200" 200 "$(echo "$M2" | awk '/^HTTP/{print $2}')" M3=$(mh multi multi-model) ck "multi 冷却期间保持 acct=1(不回退坏账号)" 1 "$(echo "$M3" | awk '/X-Upstream-Acct/{print $2}')" # 靶机验证:再发一次确认切换后的账号,读靶机实际收到的 auth(证明下行 Key 真被换成对应账号 Key) M4=$(mh multi multi-model) ck "multi 切换后下游 Key 真换成 acct-good" "Bearer acct-good" "$(up_hdr Authorization)" # ---- 账号级负载统计(管理台「密钥 / 负载」列的数据源)---- # 每次真实转发在 log 阶段落到「本次选中的那个账号」的日桶 uda:::; # 额度耗尽另记 uex:。用同一份 /api/admin/upstreams 投影断言负载分布是否正确。 UPS=$(curl -s -b "$JAR" $API/api/admin/upstreams) acct_num() { # $1=slot $2=idx $3=度量路径(如 load.requests)→ 取账号级数值 printf '%s' "$UPS" | "$PY" -c ' import sys, json slot, idx, path = sys.argv[1], int(sys.argv[2]), sys.argv[3] d = json.load(sys.stdin) items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or []) for u in items: if u.get("slot") == slot: for a in u.get("accounts", []): if a.get("idx") == idx: cur = a for k in path.split("."): cur = cur.get(k) if isinstance(cur, dict) else None print(cur if cur is not None else 0) sys.exit(0) print(0) ' "$1" "$2" "$3" } ck "★sticky 账号#0 记到今日请求(负载分布生效)" yes "$([ "$(acct_num sticky 0 load.requests)" -ge 3 ] && echo yes || echo no)" ck "★sticky 账号#1 未被分配(0 请求,粘性=集中在一把 Key)" 0 "$(acct_num sticky 1 load.requests)" ck "★multi 账号#0 记到 1 次额度耗尽" yes "$([ "$(acct_num multi 0 load.exhausts)" -ge 1 ] && echo yes || echo no)" ck "★multi 账号#1 记到今日请求(切换后负载落在新 Key)" yes "$([ "$(acct_num multi 1 load.requests)" -ge 3 ] && echo yes || echo no)" ck "账号级趋势序列含 14 天" 14 \ "$(printf '%s' "$UPS" | "$PY" -c ' import sys, json d = json.load(sys.stdin) items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or []) for u in items: if u.get("slot") == "sticky": t = u.get("account_trend") or [{}] print(len(t[0].get("requests") or [])) break else: print(0) ')" # 日期标签与趋势序列逐位对齐(图表悬停提示用;由后端算,避免前后端各自推导时区) ck "账号趋势带 14 个日期标签(与序列对齐)" 14 \ "$(printf '%s' "$UPS" | "$PY" -c ' import sys, json d = json.load(sys.stdin) items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or []) for u in items: if u.get("slot") == "sticky": print(len(u.get("trend_labels") or [])) break else: print(0) ')" # ---- 趋势(日桶)---- # 口径与请求级用量一致:只统计真正转发出去的请求与上游 usage 里的 token TR=$(curl -s -b "$JAR" "$API/api/admin/trend?days=7") ck "趋势接口 → 200" 200 "$(code -b "$JAR" "$API/api/admin/trend?days=7")" ck "趋势默认含 14 天" 14 "$(printf '%s' "$(curl -s -b "$JAR" $API/api/admin/trend)" | "$PY" -c 'import json,sys;print(len(json.load(sys.stdin)["data"]["labels"]))' 2>/dev/null)" ck "趋势天数上限 90" 90 "$(printf '%s' "$(curl -s -b "$JAR" "$API/api/admin/trend?days=99999")" | "$PY" -c 'import json,sys;print(json.load(sys.stdin)["data"]["days"])' 2>/dev/null)" ck "7 天区间 → 7 个标签" 7 "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;print(len(json.load(sys.stdin)["data"]["labels"]))' 2>/dev/null)" ck "今日请求已进日桶" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if sum(d["total"]["requests"])>0 else "no")' 2>/dev/null)" ck "今日 token 已进日桶" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if sum(d["total"]["in"])+sum(d["total"]["out"])>0 else "no")' 2>/dev/null)" ck "趋势含按账号维度" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if any(u["username"]=="alice" for u in d["users"]) else "no")' 2>/dev/null)" ck "趋势含按 API 组维度" yes "$(printf '%s' "$TR" | "$PY" -c 'import json,sys;d=json.load(sys.stdin)["data"];print("yes" if any(not s["empty"] for s in d["slots"]) else "no")' 2>/dev/null)" tok_ok() { "$PY" -c " import json, sys d = json.loads(sys.argv[1])['data'] ok_u = ok_s = False for u in d.get('users', []): if u['username'] == 'alice': t = u.get('tokens', {}) ok_u = t.get('in_total', 0) >= 18 and t.get('out_total', 0) >= 25 for m in u.get('models', []): if m['slot'] == 'openai': ok_u = ok_u and m.get('tokens', {}).get('in_total', 0) >= 18 for s in d.get('models', []): if s['slot'] == 'openai': ts = s.get('tokens', {}) ok_s = ts.get('in_total', 0) >= 18 and ts.get('out_total', 0) >= 25 print('yes' if ok_u and ok_s else 'no') " "$1"; } ck "usage API 出现 token 维度(账号/上游明细/上游聚合)" yes "$(tok_ok "$TOK")" ck "审计 meta 行记录 token" yes "$("$PY" -c " import json, sys hit = False for f in sys.argv[1:]: for l in open(f, encoding='utf-8'): d = json.loads(l) if d.get('user') == 'alice' and d.get('tin', 0) > 0: hit = True print('yes' if hit else 'no') " "$AUDIT"/*/meta.jsonl 2>/dev/null)" echo echo "============================================================" printf " 通过 \033[32m%d\033[0m 失败 \033[31m%d\033[0m\n" "$pass" "$fail" echo "============================================================" [ "$fail" -eq 0 ] || exit 1