一个 slot 挂多把上游账号 Key,解决厂商「单账号额度/并发受限」与 「同一用户缓存命中」两个矛盾诉求。 多账号路由 (plugin-llm) - models: Upstream.accounts[] (auth/note/resets) + 上游级 exhaust_status/exhaust_markers;顶层 auth 保留兼容(accounts 为空时合成单账号) - upstreams::pick_account:粘性绑定 ukb:<key> 优先 → 无绑定时 FNV-1a(下游 Key) % 可用数 冷分散 → 绑定账号冷却则重新分散 → 全冷却取最早恢复 * 分散避开厂商单账号 agent 并发上限;落点固定保住 prompt/token 缓存 - cool_until_until 天锚定冷却:boundary = day0 + ceil(已过/窗口)*窗口, 封顶次日 0 点(MiniMax 5h 窗口、24=5*4+4 → 末段 20:00-24:00 只有 4h 随天重置) - 耗尽双检测 (log 阶段):$upstream_status 命中 exhaust_status,或 usage_filter 流式扫到 marker → $llm_acct 定位本次账号 → 写冷却 uac: - shm: up:/upa:/uac:/ukb:/upe: 五类前缀;$llm_acct 变量 (NOCACHEABLE) 账号级负载统计(问题发现与跟踪) - 新增 uda:<slot>:<idx>:<day>(请求/token)与 uex:(耗尽次数) 日桶, 复用内核 usage::bump_day —— 无数据库、无新进程(项目红线) - 计数点只在 log 阶段一次;pick_account 会被 $llm_target/$llm_auth/$llm_acct 三个变量各触发一次,放进那里会三倍计数 - view() 投影带每账号 load 与 account_trend + trend_labels(日期由后端算) - 管理台「上游」页密钥列:各 Key 今日请求/token/耗尽 + 负载比例条 + 14 天迷你趋势(自绘 SVG,零第三方依赖)+ 冷却/耗尽徽标 + 上游合计 测试与验证 - 回归 279 条全绿(网关 102 + LLM 177),新增 6 条多账号/负载断言 - 假上游新增 UPSTREAM_FAIL_AUTH 模拟账号额度耗尽;靶机验证 Key 真被替换 - 前端以真实 admin.css + upstreams.js 渲染路径做无头浏览器断言: 比例条与负载成比例(412 vs 96 → 100%/23%)、每账号 14 条块、 悬停峰值日期与数据逐位吻合、单账号/无账号兜底正常、无横向溢出、零 JS 错误 文档 - 新增 docs/llm/MULTI_ACCOUNT_ROUTING.md(数据模型/路由/冷却数学/shm 布局/负载统计) - readme §11 数据模型 + §15 上游配置 + §15 用量统计 + §12.6 tag 表 + §16 变更记录 - 修正 docs/llm/*.md 中插件化前的过时路径(rust/src/* → crates/plugin-llm/src/*)
222 lines
7.8 KiB
Bash
Executable File
222 lines
7.8 KiB
Bash
Executable File
#!/bin/bash
|
||
# ============================================================================
|
||
# 本机回归 · 统一入口
|
||
#
|
||
# bash tests/run.sh # 跑全部(域名网关 + LLM 代理)
|
||
# bash tests/run.sh gateway # 只跑域名网关(90 条断言)
|
||
# bash tests/run.sh llm # 只跑 LLM 代理
|
||
#
|
||
# 前置:本机 nginx 已编译好(默认 /Users/xs/Desktop/workspace/iPass/build/ngxbuild/nginx-1.30.4/objs/nginx),
|
||
# Rust 模块由本脚本自动 cargo build。
|
||
# 可覆盖:NGINX_BIN / NGINX_BUILD_DIR / PY
|
||
#
|
||
# ⚠️ 全程使用 tests/.run/ 下的独立 data 目录,**绝不碰生产 data/**。
|
||
# nginx.conf 由 .in 模板渲染(占位符 @MODULE@ / @REPO@ / @RUN@),
|
||
# 所以 nginx 跑在哪个路径、模块编到哪都不写死。
|
||
# ============================================================================
|
||
set -uo pipefail
|
||
|
||
HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||
REPO=$(cd "$HERE/.." && pwd)
|
||
|
||
NGINX_BIN=${NGINX_BIN:-/Users/xs/Desktop/workspace/iPass/build/ngxbuild/nginx-1.30.4/objs/nginx}
|
||
NGINX_BUILD_DIR=${NGINX_BUILD_DIR:-/Users/xs/Desktop/workspace/iPass/build/ngxbuild/nginx-1.30.4/objs}
|
||
MODULE="$REPO/rust/target/debug/libipass_gateway.dylib"
|
||
|
||
# python3:优先隔离的托管运行时,其次系统的
|
||
PY=${PY:-}
|
||
if [ -z "$PY" ]; then
|
||
for c in /Users/xs/.workbuddy/binaries/python/versions/3.13.12/bin/python3 python3; do
|
||
if command -v "$c" >/dev/null 2>&1; then PY=$c; break; fi
|
||
done
|
||
fi
|
||
[ -n "$PY" ] || { echo "错误:找不到 python3" >&2; exit 1; }
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 工具
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# sed 替换串里的 & 和 | 需要转义,否则路径里的这些字符会炸
|
||
esc() { printf '%s' "$1" | sed -e 's/[&|]/\\&/g'; }
|
||
|
||
# render <模板> <输出> <RUN 目录>
|
||
render() {
|
||
sed -e "s|@MODULE@|$(esc "$MODULE")|g" \
|
||
-e "s|@REPO@|$(esc "$REPO")|g" \
|
||
-e "s|@RUN@|$(esc "$3")|g" "$1" > "$2"
|
||
}
|
||
|
||
ngx_stop() {
|
||
if [ -f "$1/nginx.conf" ]; then
|
||
"$NGINX_BIN" -p "$1" -c "$1/nginx.conf" -s stop >/dev/null 2>&1
|
||
fi
|
||
sleep 1
|
||
return 0
|
||
}
|
||
ngx_start() { "$NGINX_BIN" -p "$1" -c "$1/nginx.conf"; }
|
||
|
||
# wait_tcp <host> <port>
|
||
wait_tcp() {
|
||
local i
|
||
for i in $(seq 1 60); do
|
||
"$PY" -c 'import socket, sys
|
||
s = socket.socket(); s.settimeout(0.2)
|
||
try:
|
||
s.connect((sys.argv[1], int(sys.argv[2]))); sys.exit(0)
|
||
except Exception:
|
||
sys.exit(1)' "$1" "$2" && return 0
|
||
sleep 0.1
|
||
done
|
||
return 1
|
||
}
|
||
|
||
build() {
|
||
echo "── 编译 Rust 模块 ──"
|
||
( cd "$REPO/rust" && NGINX_BUILD_DIR="$NGINX_BUILD_DIR" cargo build ) || return 1
|
||
}
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 套件:域名网关
|
||
# ---------------------------------------------------------------------------
|
||
suite_gateway() {
|
||
local RUN="$HERE/.run/gateway"
|
||
mkdir -p "$RUN/data" "$RUN/logs"
|
||
render "$HERE/gateway/nginx.conf.in" "$RUN/nginx.conf" "$RUN"
|
||
|
||
ngx_stop "$RUN"
|
||
# 夹具覆盖式播种:保证每次跑起点一致(合成数据,非生产 data)。
|
||
# 用 cp 而非 rm —— 批量删除会被安全钩子拦截,导致脏数据残留、断言失败。
|
||
cp "$HERE/gateway/fixtures/users.json" "$RUN/data/users.json"
|
||
cp "$HERE/gateway/fixtures/domains.json" "$RUN/data/domains.json"
|
||
: > "$RUN/cookie.jar"
|
||
: > "$RUN/logs/error.log"
|
||
|
||
echo "── 启动网关(18080 转发 / 18081 靶机 / 18082 管理)──"
|
||
if ! ngx_start "$RUN"; then
|
||
echo "nginx 启动失败,见 $RUN/logs/error.log" >&2; return 1
|
||
fi
|
||
wait_tcp 127.0.0.1 18082 || { echo "管理端口未就绪" >&2; ngx_stop "$RUN"; return 1; }
|
||
|
||
echo
|
||
ADMIN="http://localhost:18082" GW="http://127.0.0.1:18080" \
|
||
JAR="$RUN/cookie.jar" RUN="$RUN" PY="$PY" \
|
||
bash "$HERE/gateway/e2e.sh"
|
||
local rc=$?
|
||
|
||
ngx_stop "$RUN"
|
||
return $rc
|
||
}
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 套件:LLM 代理
|
||
# ---------------------------------------------------------------------------
|
||
suite_llm() {
|
||
local RUN="$HERE/.run/llm"
|
||
local up_pid=""
|
||
mkdir -p "$RUN/data" "$RUN/logs"
|
||
|
||
# 夹具覆盖式播种,保证每次跑的起点一致(这两个文件是合成数据,不是生产 data)
|
||
cp "$HERE/llm/fixtures/users.json" "$RUN/data/users.json"
|
||
cp "$HERE/llm/fixtures/llm-upstreams.json" "$RUN/data/llm-upstreams.json"
|
||
cp "$HERE/llm/fixtures/api-proxies.json" "$RUN/data/api-proxies.json"
|
||
rm -f "$RUN/upstream.log"
|
||
render "$HERE/llm/nginx.conf.in" "$RUN/nginx.conf" "$RUN"
|
||
|
||
# TLS 夹具:本机 nginx 带 http_ssl_module 且有 openssl 时,生成自签证书 +
|
||
# 渲染 443 server(@RUN@/tls/llm.conf,nginx.conf.in 已 include 该目录)。
|
||
# 任一条件不满足就留空目录 → include 空匹配,HTTPS 组断言自动跳过。
|
||
HAVE_TLS=0
|
||
mkdir -p "$RUN/tls"
|
||
rm -f "$RUN"/tls/*.pem "$RUN"/tls/*.conf
|
||
if command -v openssl >/dev/null 2>&1 \
|
||
&& "$NGINX_BIN" -V 2>&1 | grep -q http_ssl_module; then
|
||
if openssl req -x509 -newkey rsa:2048 -nodes -days 30 \
|
||
-subj "/CN=llm.local" -addext "subjectAltName=DNS:llm.local" \
|
||
-keyout "$RUN/tls/key.pem" -out "$RUN/tls/cert.pem" >/dev/null 2>&1; then
|
||
render "$HERE/llm/tls.conf.in" "$RUN/tls/llm.conf" "$RUN"
|
||
HAVE_TLS=1
|
||
fi
|
||
fi
|
||
|
||
ngx_stop "$RUN"
|
||
: > "$RUN/logs/error.log"
|
||
|
||
echo "── 启动假上游(18999)与 LLM 网关(18090)──"
|
||
# 清掉**本项目自己**残留的靶机进程(如上次调试用 run_in_background 起的)。
|
||
# 否则新靶机 bind 失败、旧靶机继续服务但日志写到别的目录——
|
||
# 表现就是「请求全 200 但 UP_LOG 全空」,25 条断言集体假失败(2026-09-12 实测)。
|
||
pkill -f "tests/llm/upstream.py" 2>/dev/null
|
||
for i in $(seq 1 20); do
|
||
"$PY" -c 'import socket, sys
|
||
s = socket.socket(); s.settimeout(0.2)
|
||
try:
|
||
s.connect(("127.0.0.1", 18999)); sys.exit(0)
|
||
except Exception:
|
||
sys.exit(1)' && sleep 0.2 || break
|
||
done
|
||
|
||
UPSTREAM_LOG="$RUN/upstream.log" UPSTREAM_PORT=18999 UPSTREAM_FAIL_AUTH="acct-bad" "$PY" "$HERE/llm/upstream.py" \
|
||
>"$RUN/logs/upstream.out" 2>&1 &
|
||
up_pid=$!
|
||
|
||
# wait_tcp 可能被残留进程「骗绿」——必须确认**本进程**还活着(bind 失败会立刻退出)
|
||
if ! kill -0 "$up_pid" 2>/dev/null; then
|
||
echo "假上游启动即退出(多半是 18999 被非本项目的进程占用),见 $RUN/logs/upstream.out" >&2
|
||
return 1
|
||
fi
|
||
if ! wait_tcp 127.0.0.1 18999; then
|
||
echo "假上游未就绪,见 $RUN/logs/upstream.out" >&2
|
||
kill "$up_pid" 2>/dev/null; return 1
|
||
fi
|
||
if ! ngx_start "$RUN"; then
|
||
echo "nginx 启动失败,见 $RUN/logs/error.log" >&2
|
||
kill "$up_pid" 2>/dev/null; return 1
|
||
fi
|
||
if ! wait_tcp 127.0.0.1 18090; then
|
||
echo "LLM 端口未就绪" >&2
|
||
ngx_stop "$RUN"; kill "$up_pid" 2>/dev/null; return 1
|
||
fi
|
||
wait_tcp 127.0.0.1 18091 || { echo "管理端口未就绪" >&2; ngx_stop "$RUN"; kill "$up_pid" 2>/dev/null; return 1; }
|
||
|
||
echo
|
||
GW="http://127.0.0.1:18090" API="http://127.0.0.1:18091" \
|
||
UPSTREAM_LOG="$RUN/upstream.log" RUN="$RUN" PY="$PY" HAVE_TLS="$HAVE_TLS" \
|
||
bash "$HERE/llm/e2e.sh"
|
||
local rc=$?
|
||
|
||
ngx_stop "$RUN"
|
||
kill "$up_pid" 2>/dev/null
|
||
wait "$up_pid" 2>/dev/null
|
||
return $rc
|
||
}
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# main
|
||
# ---------------------------------------------------------------------------
|
||
WHICH=${1:-all}
|
||
rc=0
|
||
rc2=0
|
||
|
||
build || { echo "编译失败,先修 Rust 再跑回归" >&2; exit 1; }
|
||
echo
|
||
|
||
case "$WHICH" in
|
||
gateway) suite_gateway; rc=$? ;;
|
||
llm) suite_llm; rc=$? ;;
|
||
all|"")
|
||
suite_gateway; rc=$?
|
||
echo
|
||
suite_llm; rc2=$?
|
||
[ "$rc2" -ne 0 ] && rc=$rc2
|
||
;;
|
||
*) echo "用法: bash tests/run.sh [gateway|llm|all]" >&2; exit 2 ;;
|
||
esac
|
||
|
||
echo
|
||
if [ "$rc" -eq 0 ]; then
|
||
echo "全部通过"
|
||
else
|
||
echo "有断言失败(退出码 ${rc:-0})"
|
||
fi
|
||
exit $rc
|