feat(llm): 上游多账号路由 + 账号级负载统计 (1.0.14)

一个 slot 挂多把上游账号 Key,解决厂商「单账号额度/并发受限」与
「同一用户缓存命中」两个矛盾诉求。

多账号路由 (plugin-llm)
- models: Upstream.accounts[] (auth/note/resets) + 上游级
  exhaust_status/exhaust_markers;顶层 auth 保留兼容(accounts 为空时合成单账号)
- upstreams::pick_account:粘性绑定 ukb:<key> 优先 → 无绑定时
  FNV-1a(下游 Key) % 可用数 冷分散 → 绑定账号冷却则重新分散 → 全冷却取最早恢复
  * 分散避开厂商单账号 agent 并发上限;落点固定保住 prompt/token 缓存
- cool_until_until 天锚定冷却:boundary = day0 + ceil(已过/窗口)*窗口,
  封顶次日 0 点(MiniMax 5h 窗口、24=5*4+4 → 末段 20:00-24:00 只有 4h 随天重置)
- 耗尽双检测 (log 阶段):$upstream_status 命中 exhaust_status,或
  usage_filter 流式扫到 marker → $llm_acct 定位本次账号 → 写冷却 uac:
- shm: up:/upa:/uac:/ukb:/upe: 五类前缀;$llm_acct 变量 (NOCACHEABLE)

账号级负载统计(问题发现与跟踪)
- 新增 uda:<slot>:<idx>:<day>(请求/token)与 uex:(耗尽次数) 日桶,
  复用内核 usage::bump_day —— 无数据库、无新进程(项目红线)
- 计数点只在 log 阶段一次;pick_account 会被 $llm_target/$llm_auth/$llm_acct
  三个变量各触发一次,放进那里会三倍计数
- view() 投影带每账号 load 与 account_trend + trend_labels(日期由后端算)
- 管理台「上游」页密钥列:各 Key 今日请求/token/耗尽 + 负载比例条 +
  14 天迷你趋势(自绘 SVG,零第三方依赖)+ 冷却/耗尽徽标 + 上游合计

测试与验证
- 回归 279 条全绿(网关 102 + LLM 177),新增 6 条多账号/负载断言
- 假上游新增 UPSTREAM_FAIL_AUTH 模拟账号额度耗尽;靶机验证 Key 真被替换
- 前端以真实 admin.css + upstreams.js 渲染路径做无头浏览器断言:
  比例条与负载成比例(412 vs 96 → 100%/23%)、每账号 14 条块、
  悬停峰值日期与数据逐位吻合、单账号/无账号兜底正常、无横向溢出、零 JS 错误

文档
- 新增 docs/llm/MULTI_ACCOUNT_ROUTING.md(数据模型/路由/冷却数学/shm 布局/负载统计)
- readme §11 数据模型 + §15 上游配置 + §15 用量统计 + §12.6 tag 表 + §16 变更记录
- 修正 docs/llm/*.md 中插件化前的过时路径(rust/src/* → crates/plugin-llm/src/*)
This commit is contained in:
admin committed 2026-09-14 11:57:08 +08:00
1 parent db99ce1521
commit bfa3cf8ff8
21 files changed
+1506 -85

No files matched your search

+88
View File
@@ -26,6 +26,8 @@ PY=${PY:-python3}
# fixtures 里的固定 Key(见 fixtures/users.json)
ALICE_KEY=sk_alice_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
BOB_KEY=sk_bob_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
# 多账号路由验证用 admin:白名单为空=不限、无配额限制、且本套件中途会删除 bob 故不复用
ADMIN_KEY=sk_admin_0000000000000000000000000000000000000000000000ab
pass=0; fail=0
ck() {
@@ -513,6 +515,92 @@ curl -s -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/complet
sleep 0.3
TOK=$(curl -s -b "$JAR" $API/api/admin/usage)
echo "【多账号路由】粘性绑定 + 额度耗尽自动切换"
# 用 admin(白名单为空=不限;本套件中途会删除 bob,故不复用 bob)
ck "admin 基础鉴权(openai,白名单不限)→ 200" 200 "$(code -H "Authorization: Bearer $ADMIN_KEY" -X POST $GW/openai/v1/chat/completions -H 'Content-Type: application/json' -d '{"model":"gpt-4o","messages":[{"role":"user","content":"hi"}],"max_tokens":8}')"
acct_of() { curl -s -D - -o /dev/null -X POST "$GW/$1/v1/chat/completions" \
-H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \
-d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}' \
| grep -i '^X-Upstream-Acct' | tr -d '\r' | awk '{print $2}'; }
code_of() { curl -s -o /dev/null -w '%{http_code}' -X POST "$GW/$1/v1/chat/completions" \
-H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \
-d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}'; }
ck "sticky 首次请求命中 acct=0" 0 "$(acct_of sticky sticky-model)"
ck "sticky 首次请求 200" 200 "$(code_of sticky sticky-model)"
ck "sticky 粘性:第 2 次仍 acct=0" 0 "$(acct_of sticky sticky-model)"
ck "sticky 粘性:第 3 次仍 acct=0" 0 "$(acct_of sticky sticky-model)"
# 切换:multi slot accounts[0]=bad(命中假上游 UPSTREAM_FAIL_AUTH → 401+marker)→ 自动切 acct=1
mh() { curl -s -D - -o /dev/null -X POST "$GW/$1/v1/chat/completions" \
-H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \
-d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}' \
| grep -iE '^HTTP/|^X-Upstream-Acct' | tr -d '\r'; }
M1=$(mh multi multi-model)
ck "multi 初次请求打到 acct=0(bad)" 0 "$(echo "$M1" | awk '/X-Upstream-Acct/{print $2}')"
ck "multi 初次请求 401(触发冷却)" 401 "$(echo "$M1" | awk '/^HTTP/{print $2}')"
M2=$(mh multi multi-model)
ck "multi 冷却中切到 acct=1" 1 "$(echo "$M2" | awk '/X-Upstream-Acct/{print $2}')"
ck "multi 切到 acct=1 后 200" 200 "$(echo "$M2" | awk '/^HTTP/{print $2}')"
M3=$(mh multi multi-model)
ck "multi 冷却期间保持 acct=1(不回退坏账号)" 1 "$(echo "$M3" | awk '/X-Upstream-Acct/{print $2}')"
# 靶机验证:再发一次确认切换后的账号,读靶机实际收到的 auth(证明下行 Key 真被换成对应账号 Key)
M4=$(mh multi multi-model)
ck "multi 切换后下游 Key 真换成 acct-good" "Bearer acct-good" "$(up_hdr Authorization)"
# ---- 账号级负载统计(管理台「密钥 / 负载」列的数据源)----
# 每次真实转发在 log 阶段落到「本次选中的那个账号」的日桶 uda:<slot>:<idx>:<day>;
# 额度耗尽另记 uex:。用同一份 /api/admin/upstreams 投影断言负载分布是否正确。
UPS=$(curl -s -b "$JAR" $API/api/admin/upstreams)
acct_num() { # $1=slot $2=idx $3=度量路径(如 load.requests)→ 取账号级数值
printf '%s' "$UPS" | "$PY" -c '
import sys, json
slot, idx, path = sys.argv[1], int(sys.argv[2]), sys.argv[3]
d = json.load(sys.stdin)
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
for u in items:
if u.get("slot") == slot:
for a in u.get("accounts", []):
if a.get("idx") == idx:
cur = a
for k in path.split("."):
cur = cur.get(k) if isinstance(cur, dict) else None
print(cur if cur is not None else 0)
sys.exit(0)
print(0)
' "$1" "$2" "$3"
}
ck "★sticky 账号#0 记到今日请求(负载分布生效)" yes "$([ "$(acct_num sticky 0 load.requests)" -ge 3 ] && echo yes || echo no)"
ck "★sticky 账号#1 未被分配(0 请求,粘性=集中在一把 Key)" 0 "$(acct_num sticky 1 load.requests)"
ck "★multi 账号#0 记到 1 次额度耗尽" yes "$([ "$(acct_num multi 0 load.exhausts)" -ge 1 ] && echo yes || echo no)"
ck "★multi 账号#1 记到今日请求(切换后负载落在新 Key)" yes "$([ "$(acct_num multi 1 load.requests)" -ge 3 ] && echo yes || echo no)"
ck "账号级趋势序列含 14 天" 14 \
"$(printf '%s' "$UPS" | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
for u in items:
if u.get("slot") == "sticky":
t = u.get("account_trend") or [{}]
print(len(t[0].get("requests") or []))
break
else:
print(0)
')"
# 日期标签与趋势序列逐位对齐(图表悬停提示用;由后端算,避免前后端各自推导时区)
ck "账号趋势带 14 个日期标签(与序列对齐)" 14 \
"$(printf '%s' "$UPS" | "$PY" -c '
import sys, json
d = json.load(sys.stdin)
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
for u in items:
if u.get("slot") == "sticky":
print(len(u.get("trend_labels") or []))
break
else:
print(0)
')"
# ---- 趋势(日桶)----
# 口径与请求级用量一致:只统计真正转发出去的请求与上游 usage 里的 token
TR=$(curl -s -b "$JAR" "$API/api/admin/trend?days=7")
+25
View File
@@ -30,5 +30,30 @@
"enabled": true,
"note": "base 自带 /v1(厂商文档的常见写法),用于验证版本段不重复拼接",
"created_at": 1757500000
},
"multi": {
"base": "http://127.0.0.1:18999",
"models": ["multi-model"],
"enabled": true,
"accounts": [
{"auth": "Bearer acct-bad", "note": "bad", "resets": [18000, 604800]},
{"auth": "Bearer acct-good", "note": "good", "resets": [18000, 604800]}
],
"exhaust_status": [401, 402, 429, 403],
"exhaust_markers": ["\"status_code\":1004"],
"note": "一坏一好,验证额度耗尽自动切换(bad 命中假上游 UPSTREAM_FAIL_AUTH)",
"created_at": 1757500000
},
"sticky": {
"base": "http://127.0.0.1:18999",
"models": ["sticky-model"],
"enabled": true,
"accounts": [
{"auth": "Bearer g-a", "note": "a", "resets": [18000]},
{"auth": "Bearer g-b", "note": "b", "resets": [18000]}
],
"exhaust_status": [401],
"note": "两账号均好,验证同一下游 key 粘性绑定到首个账号",
"created_at": 1757500000
}
}
+4
View File
@@ -72,6 +72,10 @@ http {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Connection "";
# 调试可见:暴露本次请求选中的上游账号下标(多账号路由验证用)。
# 仅测试配置暴露;生产是否暴露由部署决定(不影响安全,只泄露账号序号)。
add_header X-Upstream-Acct $llm_acct always;
# SSE 流式两件套(响应方向;请求方向不再 no_buffering,理由见
# 生产 conf/nginx.conf——与对话审计的 client_body_in_file_only 互斥)
proxy_buffering off;
+17
View File
@@ -16,6 +16,9 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
LOG = os.environ.get("UPSTREAM_LOG", "/tmp/upstream.log")
PORT = int(os.environ.get("UPSTREAM_PORT", "18999"))
# 故障注入(多账号路由测试):收到的 Authorization 含此子串时模拟「上游账号额度耗尽」,
# 回 401 + 响应体标记(与 MiniMax 的 status_code:1004 同构),验证网关自动切换。
FAIL_AUTH = os.environ.get("UPSTREAM_FAIL_AUTH", "")
class H(BaseHTTPRequestHandler):
@@ -61,6 +64,20 @@ class H(BaseHTTPRequestHandler):
body = self.rfile.read(n)
self._record(body)
# 故障注入(多账号路由测试):收到的 Authorization 含 UPSTREAM_FAIL_AUTH 子串时,
# 模拟「上游账号额度耗尽」——回 401 + 响应体标记(与 MiniMax 的 status_code:1004 同构),
# 验证网关「状态码 + body marker 双重检测 → 标记冷却 → 下次自动切到下一个账号」。
if FAIL_AUTH and FAIL_AUTH in self.headers.get("Authorization", "-"):
payload = json.dumps(
{"base_resp": {"status_code": 1004, "status_msg": "exhausted"}}
).encode()
self.send_response(401)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
return
# 故障注入:body 带 `"boom":true` 时回 500,用来验证网关的
# 上游健康标记(log 阶段把 5xx 写进 uh: 缓存 → 列表标红)
if b'"boom"' in body:
+1 -1
View File
@@ -155,7 +155,7 @@ except Exception:
sys.exit(1)' && sleep 0.2 || break
done
UPSTREAM_LOG="$RUN/upstream.log" UPSTREAM_PORT=18999 "$PY" "$HERE/llm/upstream.py" \
UPSTREAM_LOG="$RUN/upstream.log" UPSTREAM_PORT=18999 UPSTREAM_FAIL_AUTH="acct-bad" "$PY" "$HERE/llm/upstream.py" \
>"$RUN/logs/upstream.out" 2>&1 &
up_pid=$!