Files
ipass__ipass/tests/run.sh
T
admin bfa3cf8ff8 feat(llm): 上游多账号路由 + 账号级负载统计 (1.0.14)
一个 slot 挂多把上游账号 Key,解决厂商「单账号额度/并发受限」与
「同一用户缓存命中」两个矛盾诉求。

多账号路由 (plugin-llm)
- models: Upstream.accounts[] (auth/note/resets) + 上游级
  exhaust_status/exhaust_markers;顶层 auth 保留兼容(accounts 为空时合成单账号)
- upstreams::pick_account:粘性绑定 ukb:<key> 优先 → 无绑定时
  FNV-1a(下游 Key) % 可用数 冷分散 → 绑定账号冷却则重新分散 → 全冷却取最早恢复
  * 分散避开厂商单账号 agent 并发上限;落点固定保住 prompt/token 缓存
- cool_until_until 天锚定冷却:boundary = day0 + ceil(已过/窗口)*窗口,
  封顶次日 0 点(MiniMax 5h 窗口、24=5*4+4 → 末段 20:00-24:00 只有 4h 随天重置)
- 耗尽双检测 (log 阶段):$upstream_status 命中 exhaust_status,或
  usage_filter 流式扫到 marker → $llm_acct 定位本次账号 → 写冷却 uac:
- shm: up:/upa:/uac:/ukb:/upe: 五类前缀;$llm_acct 变量 (NOCACHEABLE)

账号级负载统计(问题发现与跟踪)
- 新增 uda:<slot>:<idx>:<day>(请求/token)与 uex:(耗尽次数) 日桶,
  复用内核 usage::bump_day —— 无数据库、无新进程(项目红线)
- 计数点只在 log 阶段一次;pick_account 会被 $llm_target/$llm_auth/$llm_acct
  三个变量各触发一次,放进那里会三倍计数
- view() 投影带每账号 load 与 account_trend + trend_labels(日期由后端算)
- 管理台「上游」页密钥列:各 Key 今日请求/token/耗尽 + 负载比例条 +
  14 天迷你趋势(自绘 SVG,零第三方依赖)+ 冷却/耗尽徽标 + 上游合计

测试与验证
- 回归 279 条全绿(网关 102 + LLM 177),新增 6 条多账号/负载断言
- 假上游新增 UPSTREAM_FAIL_AUTH 模拟账号额度耗尽;靶机验证 Key 真被替换
- 前端以真实 admin.css + upstreams.js 渲染路径做无头浏览器断言:
  比例条与负载成比例(412 vs 96 → 100%/23%)、每账号 14 条块、
  悬停峰值日期与数据逐位吻合、单账号/无账号兜底正常、无横向溢出、零 JS 错误

文档
- 新增 docs/llm/MULTI_ACCOUNT_ROUTING.md(数据模型/路由/冷却数学/shm 布局/负载统计)
- readme §11 数据模型 + §15 上游配置 + §15 用量统计 + §12.6 tag 表 + §16 变更记录
- 修正 docs/llm/*.md 中插件化前的过时路径(rust/src/* → crates/plugin-llm/src/*)
2026-09-14 11:57:08 +08:00

222 lines
7.8 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/bash
# ============================================================================
# 本机回归 · 统一入口
#
# bash tests/run.sh # 跑全部(域名网关 + LLM 代理)
# bash tests/run.sh gateway # 只跑域名网关(90 条断言)
# bash tests/run.sh llm # 只跑 LLM 代理
#
# 前置:本机 nginx 已编译好(默认 /Users/xs/Desktop/workspace/iPass/build/ngxbuild/nginx-1.30.4/objs/nginx),
# Rust 模块由本脚本自动 cargo build。
# 可覆盖:NGINX_BIN / NGINX_BUILD_DIR / PY
#
# ⚠️ 全程使用 tests/.run/ 下的独立 data 目录,**绝不碰生产 data/**。
# nginx.conf 由 .in 模板渲染(占位符 @MODULE@ / @REPO@ / @RUN@),
# 所以 nginx 跑在哪个路径、模块编到哪都不写死。
# ============================================================================
set -uo pipefail
HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
REPO=$(cd "$HERE/.." && pwd)
NGINX_BIN=${NGINX_BIN:-/Users/xs/Desktop/workspace/iPass/build/ngxbuild/nginx-1.30.4/objs/nginx}
NGINX_BUILD_DIR=${NGINX_BUILD_DIR:-/Users/xs/Desktop/workspace/iPass/build/ngxbuild/nginx-1.30.4/objs}
MODULE="$REPO/rust/target/debug/libipass_gateway.dylib"
# python3:优先隔离的托管运行时,其次系统的
PY=${PY:-}
if [ -z "$PY" ]; then
for c in /Users/xs/.workbuddy/binaries/python/versions/3.13.12/bin/python3 python3; do
if command -v "$c" >/dev/null 2>&1; then PY=$c; break; fi
done
fi
[ -n "$PY" ] || { echo "错误:找不到 python3" >&2; exit 1; }
# ---------------------------------------------------------------------------
# 工具
# ---------------------------------------------------------------------------
# sed 替换串里的 & 和 | 需要转义,否则路径里的这些字符会炸
esc() { printf '%s' "$1" | sed -e 's/[&|]/\\&/g'; }
# render <模板> <输出> <RUN 目录>
render() {
sed -e "s|@MODULE@|$(esc "$MODULE")|g" \
-e "s|@REPO@|$(esc "$REPO")|g" \
-e "s|@RUN@|$(esc "$3")|g" "$1" > "$2"
}
ngx_stop() {
if [ -f "$1/nginx.conf" ]; then
"$NGINX_BIN" -p "$1" -c "$1/nginx.conf" -s stop >/dev/null 2>&1
fi
sleep 1
return 0
}
ngx_start() { "$NGINX_BIN" -p "$1" -c "$1/nginx.conf"; }
# wait_tcp <host> <port>
wait_tcp() {
local i
for i in $(seq 1 60); do
"$PY" -c 'import socket, sys
s = socket.socket(); s.settimeout(0.2)
try:
s.connect((sys.argv[1], int(sys.argv[2]))); sys.exit(0)
except Exception:
sys.exit(1)' "$1" "$2" && return 0
sleep 0.1
done
return 1
}
build() {
echo "── 编译 Rust 模块 ──"
( cd "$REPO/rust" && NGINX_BUILD_DIR="$NGINX_BUILD_DIR" cargo build ) || return 1
}
# ---------------------------------------------------------------------------
# 套件:域名网关
# ---------------------------------------------------------------------------
suite_gateway() {
local RUN="$HERE/.run/gateway"
mkdir -p "$RUN/data" "$RUN/logs"
render "$HERE/gateway/nginx.conf.in" "$RUN/nginx.conf" "$RUN"
ngx_stop "$RUN"
# 夹具覆盖式播种:保证每次跑起点一致(合成数据,非生产 data)。
# 用 cp 而非 rm —— 批量删除会被安全钩子拦截,导致脏数据残留、断言失败。
cp "$HERE/gateway/fixtures/users.json" "$RUN/data/users.json"
cp "$HERE/gateway/fixtures/domains.json" "$RUN/data/domains.json"
: > "$RUN/cookie.jar"
: > "$RUN/logs/error.log"
echo "── 启动网关(18080 转发 / 18081 靶机 / 18082 管理)──"
if ! ngx_start "$RUN"; then
echo "nginx 启动失败,见 $RUN/logs/error.log" >&2; return 1
fi
wait_tcp 127.0.0.1 18082 || { echo "管理端口未就绪" >&2; ngx_stop "$RUN"; return 1; }
echo
ADMIN="http://localhost:18082" GW="http://127.0.0.1:18080" \
JAR="$RUN/cookie.jar" RUN="$RUN" PY="$PY" \
bash "$HERE/gateway/e2e.sh"
local rc=$?
ngx_stop "$RUN"
return $rc
}
# ---------------------------------------------------------------------------
# 套件:LLM 代理
# ---------------------------------------------------------------------------
suite_llm() {
local RUN="$HERE/.run/llm"
local up_pid=""
mkdir -p "$RUN/data" "$RUN/logs"
# 夹具覆盖式播种,保证每次跑的起点一致(这两个文件是合成数据,不是生产 data)
cp "$HERE/llm/fixtures/users.json" "$RUN/data/users.json"
cp "$HERE/llm/fixtures/llm-upstreams.json" "$RUN/data/llm-upstreams.json"
cp "$HERE/llm/fixtures/api-proxies.json" "$RUN/data/api-proxies.json"
rm -f "$RUN/upstream.log"
render "$HERE/llm/nginx.conf.in" "$RUN/nginx.conf" "$RUN"
# TLS 夹具:本机 nginx 带 http_ssl_module 且有 openssl 时,生成自签证书 +
# 渲染 443 server(@RUN@/tls/llm.conf,nginx.conf.in 已 include 该目录)。
# 任一条件不满足就留空目录 → include 空匹配,HTTPS 组断言自动跳过。
HAVE_TLS=0
mkdir -p "$RUN/tls"
rm -f "$RUN"/tls/*.pem "$RUN"/tls/*.conf
if command -v openssl >/dev/null 2>&1 \
&& "$NGINX_BIN" -V 2>&1 | grep -q http_ssl_module; then
if openssl req -x509 -newkey rsa:2048 -nodes -days 30 \
-subj "/CN=llm.local" -addext "subjectAltName=DNS:llm.local" \
-keyout "$RUN/tls/key.pem" -out "$RUN/tls/cert.pem" >/dev/null 2>&1; then
render "$HERE/llm/tls.conf.in" "$RUN/tls/llm.conf" "$RUN"
HAVE_TLS=1
fi
fi
ngx_stop "$RUN"
: > "$RUN/logs/error.log"
echo "── 启动假上游(18999)与 LLM 网关(18090)──"
# 清掉**本项目自己**残留的靶机进程(如上次调试用 run_in_background 起的)。
# 否则新靶机 bind 失败、旧靶机继续服务但日志写到别的目录——
# 表现就是「请求全 200 但 UP_LOG 全空」,25 条断言集体假失败(2026-09-12 实测)。
pkill -f "tests/llm/upstream.py" 2>/dev/null
for i in $(seq 1 20); do
"$PY" -c 'import socket, sys
s = socket.socket(); s.settimeout(0.2)
try:
s.connect(("127.0.0.1", 18999)); sys.exit(0)
except Exception:
sys.exit(1)' && sleep 0.2 || break
done
UPSTREAM_LOG="$RUN/upstream.log" UPSTREAM_PORT=18999 UPSTREAM_FAIL_AUTH="acct-bad" "$PY" "$HERE/llm/upstream.py" \
>"$RUN/logs/upstream.out" 2>&1 &
up_pid=$!
# wait_tcp 可能被残留进程「骗绿」——必须确认**本进程**还活着(bind 失败会立刻退出)
if ! kill -0 "$up_pid" 2>/dev/null; then
echo "假上游启动即退出(多半是 18999 被非本项目的进程占用),见 $RUN/logs/upstream.out" >&2
return 1
fi
if ! wait_tcp 127.0.0.1 18999; then
echo "假上游未就绪,见 $RUN/logs/upstream.out" >&2
kill "$up_pid" 2>/dev/null; return 1
fi
if ! ngx_start "$RUN"; then
echo "nginx 启动失败,见 $RUN/logs/error.log" >&2
kill "$up_pid" 2>/dev/null; return 1
fi
if ! wait_tcp 127.0.0.1 18090; then
echo "LLM 端口未就绪" >&2
ngx_stop "$RUN"; kill "$up_pid" 2>/dev/null; return 1
fi
wait_tcp 127.0.0.1 18091 || { echo "管理端口未就绪" >&2; ngx_stop "$RUN"; kill "$up_pid" 2>/dev/null; return 1; }
echo
GW="http://127.0.0.1:18090" API="http://127.0.0.1:18091" \
UPSTREAM_LOG="$RUN/upstream.log" RUN="$RUN" PY="$PY" HAVE_TLS="$HAVE_TLS" \
bash "$HERE/llm/e2e.sh"
local rc=$?
ngx_stop "$RUN"
kill "$up_pid" 2>/dev/null
wait "$up_pid" 2>/dev/null
return $rc
}
# ---------------------------------------------------------------------------
# main
# ---------------------------------------------------------------------------
WHICH=${1:-all}
rc=0
rc2=0
build || { echo "编译失败,先修 Rust 再跑回归" >&2; exit 1; }
echo
case "$WHICH" in
gateway) suite_gateway; rc=$? ;;
llm) suite_llm; rc=$? ;;
all|"")
suite_gateway; rc=$?
echo
suite_llm; rc2=$?
[ "$rc2" -ne 0 ] && rc=$rc2
;;
*) echo "用法: bash tests/run.sh [gateway|llm|all]" >&2; exit 2 ;;
esac
echo
if [ "$rc" -eq 0 ]; then
echo "全部通过"
else
echo "有断言失败(退出码 ${rc:-0})"
fi
exit $rc