feat(llm): 上游多账号路由 + 账号级负载统计 (1.0.14)
一个 slot 挂多把上游账号 Key,解决厂商「单账号额度/并发受限」与 「同一用户缓存命中」两个矛盾诉求。 多账号路由 (plugin-llm) - models: Upstream.accounts[] (auth/note/resets) + 上游级 exhaust_status/exhaust_markers;顶层 auth 保留兼容(accounts 为空时合成单账号) - upstreams::pick_account:粘性绑定 ukb:<key> 优先 → 无绑定时 FNV-1a(下游 Key) % 可用数 冷分散 → 绑定账号冷却则重新分散 → 全冷却取最早恢复 * 分散避开厂商单账号 agent 并发上限;落点固定保住 prompt/token 缓存 - cool_until_until 天锚定冷却:boundary = day0 + ceil(已过/窗口)*窗口, 封顶次日 0 点(MiniMax 5h 窗口、24=5*4+4 → 末段 20:00-24:00 只有 4h 随天重置) - 耗尽双检测 (log 阶段):$upstream_status 命中 exhaust_status,或 usage_filter 流式扫到 marker → $llm_acct 定位本次账号 → 写冷却 uac: - shm: up:/upa:/uac:/ukb:/upe: 五类前缀;$llm_acct 变量 (NOCACHEABLE) 账号级负载统计(问题发现与跟踪) - 新增 uda:<slot>:<idx>:<day>(请求/token)与 uex:(耗尽次数) 日桶, 复用内核 usage::bump_day —— 无数据库、无新进程(项目红线) - 计数点只在 log 阶段一次;pick_account 会被 $llm_target/$llm_auth/$llm_acct 三个变量各触发一次,放进那里会三倍计数 - view() 投影带每账号 load 与 account_trend + trend_labels(日期由后端算) - 管理台「上游」页密钥列:各 Key 今日请求/token/耗尽 + 负载比例条 + 14 天迷你趋势(自绘 SVG,零第三方依赖)+ 冷却/耗尽徽标 + 上游合计 测试与验证 - 回归 279 条全绿(网关 102 + LLM 177),新增 6 条多账号/负载断言 - 假上游新增 UPSTREAM_FAIL_AUTH 模拟账号额度耗尽;靶机验证 Key 真被替换 - 前端以真实 admin.css + upstreams.js 渲染路径做无头浏览器断言: 比例条与负载成比例(412 vs 96 → 100%/23%)、每账号 14 条块、 悬停峰值日期与数据逐位吻合、单账号/无账号兜底正常、无横向溢出、零 JS 错误 文档 - 新增 docs/llm/MULTI_ACCOUNT_ROUTING.md(数据模型/路由/冷却数学/shm 布局/负载统计) - readme §11 数据模型 + §15 上游配置 + §15 用量统计 + §12.6 tag 表 + §16 变更记录 - 修正 docs/llm/*.md 中插件化前的过时路径(rust/src/* → crates/plugin-llm/src/*)
This commit is contained in:
1 parent
db99ce1521
commit
bfa3cf8ff8
21 files changed
+1506
-85
No files matched your search
@@ -26,6 +26,8 @@ PY=${PY:-python3}
|
||||
# fixtures 里的固定 Key(见 fixtures/users.json)
|
||||
ALICE_KEY=sk_alice_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
BOB_KEY=sk_bob_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
|
||||
# 多账号路由验证用 admin:白名单为空=不限、无配额限制、且本套件中途会删除 bob 故不复用
|
||||
ADMIN_KEY=sk_admin_0000000000000000000000000000000000000000000000ab
|
||||
|
||||
pass=0; fail=0
|
||||
ck() {
|
||||
@@ -513,6 +515,92 @@ curl -s -H "Authorization: Bearer $ALICE_KEY" -X POST $GW/openai/v1/chat/complet
|
||||
sleep 0.3
|
||||
TOK=$(curl -s -b "$JAR" $API/api/admin/usage)
|
||||
|
||||
echo "【多账号路由】粘性绑定 + 额度耗尽自动切换"
|
||||
# 用 admin(白名单为空=不限;本套件中途会删除 bob,故不复用 bob)
|
||||
ck "admin 基础鉴权(openai,白名单不限)→ 200" 200 "$(code -H "Authorization: Bearer $ADMIN_KEY" -X POST $GW/openai/v1/chat/completions -H 'Content-Type: application/json' -d '{"model":"gpt-4o","messages":[{"role":"user","content":"hi"}],"max_tokens":8}')"
|
||||
acct_of() { curl -s -D - -o /dev/null -X POST "$GW/$1/v1/chat/completions" \
|
||||
-H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \
|
||||
-d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}' \
|
||||
| grep -i '^X-Upstream-Acct' | tr -d '\r' | awk '{print $2}'; }
|
||||
code_of() { curl -s -o /dev/null -w '%{http_code}' -X POST "$GW/$1/v1/chat/completions" \
|
||||
-H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \
|
||||
-d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}'; }
|
||||
|
||||
ck "sticky 首次请求命中 acct=0" 0 "$(acct_of sticky sticky-model)"
|
||||
ck "sticky 首次请求 200" 200 "$(code_of sticky sticky-model)"
|
||||
ck "sticky 粘性:第 2 次仍 acct=0" 0 "$(acct_of sticky sticky-model)"
|
||||
ck "sticky 粘性:第 3 次仍 acct=0" 0 "$(acct_of sticky sticky-model)"
|
||||
|
||||
# 切换:multi slot accounts[0]=bad(命中假上游 UPSTREAM_FAIL_AUTH → 401+marker)→ 自动切 acct=1
|
||||
mh() { curl -s -D - -o /dev/null -X POST "$GW/$1/v1/chat/completions" \
|
||||
-H "Authorization: Bearer $ADMIN_KEY" -H 'Content-Type: application/json' \
|
||||
-d '{"model":"'"$2"'","messages":[{"role":"user","content":"hi"}],"max_tokens":8}' \
|
||||
| grep -iE '^HTTP/|^X-Upstream-Acct' | tr -d '\r'; }
|
||||
M1=$(mh multi multi-model)
|
||||
ck "multi 初次请求打到 acct=0(bad)" 0 "$(echo "$M1" | awk '/X-Upstream-Acct/{print $2}')"
|
||||
ck "multi 初次请求 401(触发冷却)" 401 "$(echo "$M1" | awk '/^HTTP/{print $2}')"
|
||||
M2=$(mh multi multi-model)
|
||||
ck "multi 冷却中切到 acct=1" 1 "$(echo "$M2" | awk '/X-Upstream-Acct/{print $2}')"
|
||||
ck "multi 切到 acct=1 后 200" 200 "$(echo "$M2" | awk '/^HTTP/{print $2}')"
|
||||
M3=$(mh multi multi-model)
|
||||
ck "multi 冷却期间保持 acct=1(不回退坏账号)" 1 "$(echo "$M3" | awk '/X-Upstream-Acct/{print $2}')"
|
||||
# 靶机验证:再发一次确认切换后的账号,读靶机实际收到的 auth(证明下行 Key 真被换成对应账号 Key)
|
||||
M4=$(mh multi multi-model)
|
||||
ck "multi 切换后下游 Key 真换成 acct-good" "Bearer acct-good" "$(up_hdr Authorization)"
|
||||
|
||||
# ---- 账号级负载统计(管理台「密钥 / 负载」列的数据源)----
|
||||
# 每次真实转发在 log 阶段落到「本次选中的那个账号」的日桶 uda:<slot>:<idx>:<day>;
|
||||
# 额度耗尽另记 uex:。用同一份 /api/admin/upstreams 投影断言负载分布是否正确。
|
||||
UPS=$(curl -s -b "$JAR" $API/api/admin/upstreams)
|
||||
acct_num() { # $1=slot $2=idx $3=度量路径(如 load.requests)→ 取账号级数值
|
||||
printf '%s' "$UPS" | "$PY" -c '
|
||||
import sys, json
|
||||
slot, idx, path = sys.argv[1], int(sys.argv[2]), sys.argv[3]
|
||||
d = json.load(sys.stdin)
|
||||
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
|
||||
for u in items:
|
||||
if u.get("slot") == slot:
|
||||
for a in u.get("accounts", []):
|
||||
if a.get("idx") == idx:
|
||||
cur = a
|
||||
for k in path.split("."):
|
||||
cur = cur.get(k) if isinstance(cur, dict) else None
|
||||
print(cur if cur is not None else 0)
|
||||
sys.exit(0)
|
||||
print(0)
|
||||
' "$1" "$2" "$3"
|
||||
}
|
||||
ck "★sticky 账号#0 记到今日请求(负载分布生效)" yes "$([ "$(acct_num sticky 0 load.requests)" -ge 3 ] && echo yes || echo no)"
|
||||
ck "★sticky 账号#1 未被分配(0 请求,粘性=集中在一把 Key)" 0 "$(acct_num sticky 1 load.requests)"
|
||||
ck "★multi 账号#0 记到 1 次额度耗尽" yes "$([ "$(acct_num multi 0 load.exhausts)" -ge 1 ] && echo yes || echo no)"
|
||||
ck "★multi 账号#1 记到今日请求(切换后负载落在新 Key)" yes "$([ "$(acct_num multi 1 load.requests)" -ge 3 ] && echo yes || echo no)"
|
||||
ck "账号级趋势序列含 14 天" 14 \
|
||||
"$(printf '%s' "$UPS" | "$PY" -c '
|
||||
import sys, json
|
||||
d = json.load(sys.stdin)
|
||||
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
|
||||
for u in items:
|
||||
if u.get("slot") == "sticky":
|
||||
t = u.get("account_trend") or [{}]
|
||||
print(len(t[0].get("requests") or []))
|
||||
break
|
||||
else:
|
||||
print(0)
|
||||
')"
|
||||
# 日期标签与趋势序列逐位对齐(图表悬停提示用;由后端算,避免前后端各自推导时区)
|
||||
ck "账号趋势带 14 个日期标签(与序列对齐)" 14 \
|
||||
"$(printf '%s' "$UPS" | "$PY" -c '
|
||||
import sys, json
|
||||
d = json.load(sys.stdin)
|
||||
items = d["data"]["items"] if isinstance(d.get("data"), dict) else (d.get("data") or [])
|
||||
for u in items:
|
||||
if u.get("slot") == "sticky":
|
||||
print(len(u.get("trend_labels") or []))
|
||||
break
|
||||
else:
|
||||
print(0)
|
||||
')"
|
||||
|
||||
# ---- 趋势(日桶)----
|
||||
# 口径与请求级用量一致:只统计真正转发出去的请求与上游 usage 里的 token
|
||||
TR=$(curl -s -b "$JAR" "$API/api/admin/trend?days=7")
|
||||
|
||||
@@ -30,5 +30,30 @@
|
||||
"enabled": true,
|
||||
"note": "base 自带 /v1(厂商文档的常见写法),用于验证版本段不重复拼接",
|
||||
"created_at": 1757500000
|
||||
},
|
||||
"multi": {
|
||||
"base": "http://127.0.0.1:18999",
|
||||
"models": ["multi-model"],
|
||||
"enabled": true,
|
||||
"accounts": [
|
||||
{"auth": "Bearer acct-bad", "note": "bad", "resets": [18000, 604800]},
|
||||
{"auth": "Bearer acct-good", "note": "good", "resets": [18000, 604800]}
|
||||
],
|
||||
"exhaust_status": [401, 402, 429, 403],
|
||||
"exhaust_markers": ["\"status_code\":1004"],
|
||||
"note": "一坏一好,验证额度耗尽自动切换(bad 命中假上游 UPSTREAM_FAIL_AUTH)",
|
||||
"created_at": 1757500000
|
||||
},
|
||||
"sticky": {
|
||||
"base": "http://127.0.0.1:18999",
|
||||
"models": ["sticky-model"],
|
||||
"enabled": true,
|
||||
"accounts": [
|
||||
{"auth": "Bearer g-a", "note": "a", "resets": [18000]},
|
||||
{"auth": "Bearer g-b", "note": "b", "resets": [18000]}
|
||||
],
|
||||
"exhaust_status": [401],
|
||||
"note": "两账号均好,验证同一下游 key 粘性绑定到首个账号",
|
||||
"created_at": 1757500000
|
||||
}
|
||||
}
|
||||
@@ -72,6 +72,10 @@ http {
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header Connection "";
|
||||
|
||||
# 调试可见:暴露本次请求选中的上游账号下标(多账号路由验证用)。
|
||||
# 仅测试配置暴露;生产是否暴露由部署决定(不影响安全,只泄露账号序号)。
|
||||
add_header X-Upstream-Acct $llm_acct always;
|
||||
|
||||
# SSE 流式两件套(响应方向;请求方向不再 no_buffering,理由见
|
||||
# 生产 conf/nginx.conf——与对话审计的 client_body_in_file_only 互斥)
|
||||
proxy_buffering off;
|
||||
|
||||
@@ -16,6 +16,9 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
LOG = os.environ.get("UPSTREAM_LOG", "/tmp/upstream.log")
|
||||
PORT = int(os.environ.get("UPSTREAM_PORT", "18999"))
|
||||
# 故障注入(多账号路由测试):收到的 Authorization 含此子串时模拟「上游账号额度耗尽」,
|
||||
# 回 401 + 响应体标记(与 MiniMax 的 status_code:1004 同构),验证网关自动切换。
|
||||
FAIL_AUTH = os.environ.get("UPSTREAM_FAIL_AUTH", "")
|
||||
|
||||
|
||||
class H(BaseHTTPRequestHandler):
|
||||
@@ -61,6 +64,20 @@ class H(BaseHTTPRequestHandler):
|
||||
body = self.rfile.read(n)
|
||||
self._record(body)
|
||||
|
||||
# 故障注入(多账号路由测试):收到的 Authorization 含 UPSTREAM_FAIL_AUTH 子串时,
|
||||
# 模拟「上游账号额度耗尽」——回 401 + 响应体标记(与 MiniMax 的 status_code:1004 同构),
|
||||
# 验证网关「状态码 + body marker 双重检测 → 标记冷却 → 下次自动切到下一个账号」。
|
||||
if FAIL_AUTH and FAIL_AUTH in self.headers.get("Authorization", "-"):
|
||||
payload = json.dumps(
|
||||
{"base_resp": {"status_code": 1004, "status_msg": "exhausted"}}
|
||||
).encode()
|
||||
self.send_response(401)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
return
|
||||
|
||||
# 故障注入:body 带 `"boom":true` 时回 500,用来验证网关的
|
||||
# 上游健康标记(log 阶段把 5xx 写进 uh: 缓存 → 列表标红)
|
||||
if b'"boom"' in body:
|
||||
|
||||
+1
-1
@@ -155,7 +155,7 @@ except Exception:
|
||||
sys.exit(1)' && sleep 0.2 || break
|
||||
done
|
||||
|
||||
UPSTREAM_LOG="$RUN/upstream.log" UPSTREAM_PORT=18999 "$PY" "$HERE/llm/upstream.py" \
|
||||
UPSTREAM_LOG="$RUN/upstream.log" UPSTREAM_PORT=18999 UPSTREAM_FAIL_AUTH="acct-bad" "$PY" "$HERE/llm/upstream.py" \
|
||||
>"$RUN/logs/upstream.out" 2>&1 &
|
||||
up_pid=$!
|
||||
|
||||
|
||||
Reference in new issue
Block a user