Files
admin 0a9047840d feat(llm): 上游账号冷却改指数退避 + 真实请求探测闭环 (1.0.44)
天锚定重置边界计算(resets)删掉:锚点算错过两次(1.0.14 / 1.0.24),
线上表现为「厂商 15:00 已重置、网关锁到 18:00」。resets 只存不路由。

新口径:uac: = <backoff>:<next_probe>:<hard_cap>
- 到期放一个「无绑定/绑定失效」的真实请求过去探测,2xx → clear_cool 回队;
- 仍耗尽 → 退避翻倍(首轮 ipass_probe_min,默认 60s,30min 硬顶无条件回队自愈);
- 绑定流量永不被劫持当探测(保 prompt 缓存亲和)。
- pick_account 必须纯决策:四个 $llm_* 变量各求值一次,副作用只在 log 阶段记一次。

fix: 探测窗口判定改严格小于 —— 秒级时钟下 >= 会让「标记于 45.95s、next_probe=46」
的账号在 46.05s 即被当探测目标,1s 退避实际只等 0.05s,冷却形同虚设(回归暴露)。

回归 370 条全绿(网关 108 + LLM 262,新增「探测恢复闭环」剧本)。
2026-09-17 07:19:35 +08:00

237 lines
8.8 KiB
Bash
Executable File
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/bash
# ============================================================================
# 本机回归 · 统一入口
#
# bash tests/run.sh # 跑全部(域名网关 + LLM 代理)
# bash tests/run.sh gateway # 只跑域名网关(102 条断言)
# bash tests/run.sh llm # 只跑 LLM 代理
#
# 前置:本机 nginx 已编译好(默认 /Users/xs/Desktop/workspace/iPass/build/ngxbuild/nginx-1.30.4/objs/nginx),
# Rust 模块由本脚本自动 cargo build。
# 可覆盖:NGINX_BIN / NGINX_BUILD_DIR / PY
#
# ⚠️ 全程使用 tests/.run/ 下的独立 data 目录,**绝不碰生产 data/**。
# nginx.conf 由 .in 模板渲染(占位符 @MODULE@ / @REPO@ / @RUN@),
# 所以 nginx 跑在哪个路径、模块编到哪都不写死。
# ============================================================================
set -uo pipefail
HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
REPO=$(cd "$HERE/.." && pwd)
NGINX_BIN=${NGINX_BIN:-/Users/xs/Desktop/workspace/iPass/build/ngxbuild/nginx-1.30.4/objs/nginx}
NGINX_BUILD_DIR=${NGINX_BUILD_DIR:-/Users/xs/Desktop/workspace/iPass/build/ngxbuild/nginx-1.30.4/objs}
MODULE="$REPO/rust/target/debug/libipass_gateway.dylib"
# python3:优先隔离的托管运行时,其次系统的
PY=${PY:-}
if [ -z "$PY" ]; then
for c in /Users/xs/.workbuddy/binaries/python/versions/3.13.12/bin/python3 python3; do
if command -v "$c" >/dev/null 2>&1; then PY=$c; break; fi
done
fi
[ -n "$PY" ] || { echo "错误:找不到 python3" >&2; exit 1; }
# ---------------------------------------------------------------------------
# 工具
# ---------------------------------------------------------------------------
# sed 替换串里的 & 和 | 需要转义,否则路径里的这些字符会炸
esc() { printf '%s' "$1" | sed -e 's/[&|]/\\&/g'; }
# render <模板> <输出> <RUN 目录>
render() {
sed -e "s|@MODULE@|$(esc "$MODULE")|g" \
-e "s|@REPO@|$(esc "$REPO")|g" \
-e "s|@RUN@|$(esc "$3")|g" "$1" > "$2"
}
ngx_stop() {
if [ -f "$1/nginx.conf" ]; then
"$NGINX_BIN" -p "$1" -c "$1/nginx.conf" -s stop >/dev/null 2>&1
fi
sleep 1
return 0
}
ngx_start() { "$NGINX_BIN" -p "$1" -c "$1/nginx.conf"; }
# wait_tcp <host> <port>
wait_tcp() {
local i
for i in $(seq 1 60); do
"$PY" -c 'import socket, sys
s = socket.socket(); s.settimeout(0.2)
try:
s.connect((sys.argv[1], int(sys.argv[2]))); sys.exit(0)
except Exception:
sys.exit(1)' "$1" "$2" && return 0
sleep 0.1
done
return 1
}
build() {
echo "── 编译 Rust 模块 ──"
( cd "$REPO/rust" && NGINX_BUILD_DIR="$NGINX_BUILD_DIR" cargo build ) || return 1
}
# ---------------------------------------------------------------------------
# 套件:域名网关
# ---------------------------------------------------------------------------
suite_gateway() {
local RUN="$HERE/.run/gateway"
mkdir -p "$RUN/data" "$RUN/logs"
render "$HERE/gateway/nginx.conf.in" "$RUN/nginx.conf" "$RUN"
ngx_stop "$RUN"
# 夹具覆盖式播种:保证每次跑起点一致(合成数据,非生产 data)。
# 用 cp 而非 rm —— 批量删除会被安全钩子拦截,导致脏数据残留、断言失败。
cp "$HERE/gateway/fixtures/users.json" "$RUN/data/users.json"
cp "$HERE/gateway/fixtures/domains.json" "$RUN/data/domains.json"
: > "$RUN/cookie.jar"
: > "$RUN/logs/error.log"
echo "── 启动网关(18080 转发 / 18081 靶机 / 18082 管理)──"
if ! ngx_start "$RUN"; then
echo "nginx 启动失败,见 $RUN/logs/error.log" >&2; return 1
fi
wait_tcp 127.0.0.1 18082 || { echo "管理端口未就绪" >&2; ngx_stop "$RUN"; return 1; }
echo
ADMIN="http://localhost:18082" GW="http://127.0.0.1:18080" \
JAR="$RUN/cookie.jar" RUN="$RUN" PY="$PY" \
bash "$HERE/gateway/e2e.sh"
local rc=$?
ngx_stop "$RUN"
return $rc
}
# ---------------------------------------------------------------------------
# 套件:LLM 代理
# ---------------------------------------------------------------------------
suite_llm() {
local RUN="$HERE/.run/llm"
local up_pid=""
mkdir -p "$RUN/data" "$RUN/logs"
# 夹具覆盖式播种,保证每次跑的起点一致(这两个文件是合成数据,不是生产 data)
cp "$HERE/llm/fixtures/users.json" "$RUN/data/users.json"
cp "$HERE/llm/fixtures/llm-upstreams.json" "$RUN/data/llm-upstreams.json"
cp "$HERE/llm/fixtures/api-proxies.json" "$RUN/data/api-proxies.json"
rm -f "$RUN/upstream.log"
# 把上一轮的统计快照**挪开**(而不是删除),保证本轮从 0 起算。
#
# 为什么不删:本机装了 safe-delete 钩子 —— 同一个 agent turn 里累计删除超过
# 50 个文件后,后续所有删除都被静默拦掉(可见日志 SAFE_DELETE_BULK_CONFIRM_REQUIRED)。
# `rm -rf` 与 python 的 shutil.rmtree / os.remove 都撞在这上面。
# 症状很有欺骗性:清理「没报错但没生效」,于是上一轮的 ust:/ud:/uh: 被启动时的
# restore() 恢复进本轮 —— carol 用量期望 2 变成 6、「上游健康初始状态」期望
# unknown 变成 ok(看起来像统计功能有 bug,实际是测试间的状态泄漏)。
# `mv` 是 rename、不是删除,在沙箱下稳定生效;留档也便于比对。
if [ -s "$RUN/data/llm-stats.json" ]; then
mkdir -p "$RUN/stale"
mv "$RUN/data/llm-stats.json" "$RUN/stale/llm-stats.$(date +%s).json"
fi
render "$HERE/llm/nginx.conf.in" "$RUN/nginx.conf" "$RUN"
# TLS 夹具:本机 nginx 带 http_ssl_module 且有 openssl 时,生成自签证书 +
# 渲染 443 server(@RUN@/tls/llm.conf,nginx.conf.in 已 include 该目录)。
# 任一条件不满足就留空目录 → include 空匹配,HTTPS 组断言自动跳过。
HAVE_TLS=0
mkdir -p "$RUN/tls"
rm -f "$RUN"/tls/*.pem "$RUN"/tls/*.conf
if command -v openssl >/dev/null 2>&1 \
&& "$NGINX_BIN" -V 2>&1 | grep -q http_ssl_module; then
if openssl req -x509 -newkey rsa:2048 -nodes -days 30 \
-subj "/CN=llm.local" -addext "subjectAltName=DNS:llm.local" \
-keyout "$RUN/tls/key.pem" -out "$RUN/tls/cert.pem" >/dev/null 2>&1; then
render "$HERE/llm/tls.conf.in" "$RUN/tls/llm.conf" "$RUN"
HAVE_TLS=1
fi
fi
ngx_stop "$RUN"
: > "$RUN/logs/error.log"
echo "── 启动假上游(18999)与 LLM 网关(18090)──"
# 清掉**本项目自己**残留的靶机进程(如上次调试用 run_in_background 起的)。
# 否则新靶机 bind 失败、旧靶机继续服务但日志写到别的目录——
# 表现就是「请求全 200 但 UP_LOG 全空」,25 条断言集体假失败(2026-09-12 实测)。
pkill -f "tests/llm/upstream.py" 2>/dev/null
for i in $(seq 1 20); do
"$PY" -c 'import socket, sys
s = socket.socket(); s.settimeout(0.2)
try:
s.connect(("127.0.0.1", 18999)); sys.exit(0)
except Exception:
sys.exit(1)' && sleep 0.2 || break
done
UPSTREAM_LOG="$RUN/upstream.log" UPSTREAM_PORT=18999 UPSTREAM_FAIL_AUTH="acct-bad" \
UPSTREAM_FAIL_FIRST_AUTH="probe-flip" UPSTREAM_FAIL_ALWAYS_AUTH="probe-dead" "$PY" "$HERE/llm/upstream.py" \
>"$RUN/logs/upstream.out" 2>&1 &
up_pid=$!
# wait_tcp 可能被残留进程「骗绿」——必须确认**本进程**还活着(bind 失败会立刻退出)
if ! kill -0 "$up_pid" 2>/dev/null; then
echo "假上游启动即退出(多半是 18999 被非本项目的进程占用),见 $RUN/logs/upstream.out" >&2
return 1
fi
if ! wait_tcp 127.0.0.1 18999; then
echo "假上游未就绪,见 $RUN/logs/upstream.out" >&2
kill "$up_pid" 2>/dev/null; return 1
fi
if ! ngx_start "$RUN"; then
echo "nginx 启动失败,见 $RUN/logs/error.log" >&2
kill "$up_pid" 2>/dev/null; return 1
fi
if ! wait_tcp 127.0.0.1 18090; then
echo "LLM 端口未就绪" >&2
ngx_stop "$RUN"; kill "$up_pid" 2>/dev/null; return 1
fi
wait_tcp 127.0.0.1 18091 || { echo "管理端口未就绪" >&2; ngx_stop "$RUN"; kill "$up_pid" 2>/dev/null; return 1; }
echo
GW="http://127.0.0.1:18090" API="http://127.0.0.1:18091" \
UPSTREAM_LOG="$RUN/upstream.log" RUN="$RUN" PY="$PY" HAVE_TLS="$HAVE_TLS" \
NGINX_BIN="$NGINX_BIN" \
bash "$HERE/llm/e2e.sh"
local rc=$?
ngx_stop "$RUN"
kill "$up_pid" 2>/dev/null
wait "$up_pid" 2>/dev/null
return $rc
}
# ---------------------------------------------------------------------------
# main
# ---------------------------------------------------------------------------
WHICH=${1:-all}
rc=0
rc2=0
build || { echo "编译失败,先修 Rust 再跑回归" >&2; exit 1; }
echo
case "$WHICH" in
gateway) suite_gateway; rc=$? ;;
llm) suite_llm; rc=$? ;;
all|"")
suite_gateway; rc=$?
echo
suite_llm; rc2=$?
[ "$rc2" -ne 0 ] && rc=$rc2
;;
*) echo "用法: bash tests/run.sh [gateway|llm|all]" >&2; exit 2 ;;
esac
echo
if [ "$rc" -eq 0 ]; then
echo "全部通过"
else
echo "有断言失败(退出码 ${rc:-0})"
fi
exit $rc