Files
admin 0a9047840d feat(llm): 上游账号冷却改指数退避 + 真实请求探测闭环 (1.0.44)
天锚定重置边界计算(resets)删掉:锚点算错过两次(1.0.14 / 1.0.24),
线上表现为「厂商 15:00 已重置、网关锁到 18:00」。resets 只存不路由。

新口径:uac: = <backoff>:<next_probe>:<hard_cap>
- 到期放一个「无绑定/绑定失效」的真实请求过去探测,2xx → clear_cool 回队;
- 仍耗尽 → 退避翻倍(首轮 ipass_probe_min,默认 60s,30min 硬顶无条件回队自愈);
- 绑定流量永不被劫持当探测(保 prompt 缓存亲和)。
- pick_account 必须纯决策:四个 $llm_* 变量各求值一次,副作用只在 log 阶段记一次。

fix: 探测窗口判定改严格小于 —— 秒级时钟下 >= 会让「标记于 45.95s、next_probe=46」
的账号在 46.05s 即被当探测目标,1s 退避实际只等 0.05s,冷却形同虚设(回归暴露)。

回归 370 条全绿(网关 108 + LLM 262,新增「探测恢复闭环」剧本)。
2026-09-17 07:19:35 +08:00

143 lines
6.6 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""假上游:记录它**实际收到**的 path / auth / host / body。
这是验证「下游 Key 是否真被换成上游真 Key」的唯一可靠手段 ——
只看网关自己的响应是看不出来的(网关返回 200 不代表上游收到了正确的头)。
UPSTREAM_PORT=18999 UPSTREAM_LOG=/path/to/upstream.log python3 upstream.py
带 `"stream": true` 的请求体会回 SSE,分 3 块、每块间隔 0.4s,
用来验证 nginx 侧 `proxy_buffering off` 是否生效(缓冲住的话会攒完一次性返回)。
"""
import json
import os
import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
LOG = os.environ.get("UPSTREAM_LOG", "/tmp/upstream.log")
PORT = int(os.environ.get("UPSTREAM_PORT", "18999"))
# 故障注入(多账号路由测试):收到的 Authorization 含此子串时模拟「上游账号额度耗尽」,
# 回 401 + 响应体标记(与 MiniMax 的 status_code:1004 同构),验证网关自动切换。
FAIL_AUTH = os.environ.get("UPSTREAM_FAIL_AUTH", "")
# 探测闭环测试:FAIL_FIRST 对某 auth **只失败第一次**(之后正常)——用来验证
# 「冷却 → 退避到期 → 真实请求探测成功 → 自动回队」;FAIL_ALWAYS 永远失败——
# 验证「探测失败退避翻倍、冷却期持续跳过、代价被封顶」。
FAIL_FIRST_AUTH = os.environ.get("UPSTREAM_FAIL_FIRST_AUTH", "")
FAIL_ALWAYS_AUTH = os.environ.get("UPSTREAM_FAIL_ALWAYS_AUTH", "")
_first_seen: set = set()
_first_lock = threading.Lock()
class H(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def _record(self, body: bytes):
# 记录**全部**收到的头,而不是手挑几个字段。
# 教训:手挑字段会漏掉"没被替换的那个头"——第一版只记 Authorization,
# 于是 x-api-key 泄漏在整轮回归里全绿通过,完全看不见。
rec = {
"path": self.path,
"method": self.command,
"headers": {k.lower(): v for k, v in self.headers.items()},
# 下面几个是长期被断言的字段,保留以免旧断言失效
"auth": self.headers.get("Authorization", "-"),
"x_api_key": self.headers.get("x-api-key", "-"),
"anthropic_version": self.headers.get("anthropic-version", "-"),
"host": self.headers.get("Host", "-"),
"body": body.decode("utf-8", "replace"),
}
with open(LOG, "a") as f:
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
def do_GET(self):
"""`GET /<slot>/v1/models` 必须原样透传到上游——这里验的就是这条。
网关自己的 `GET /v1/models`(无 slot 段)在本地应答、不会打到这里,
两条路互不干扰。
"""
self._record(b"")
payload = json.dumps({
"object": "list",
"data": [{"id": "upstream-model", "object": "model"}],
}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
def do_POST(self):
n = int(self.headers.get("Content-Length") or 0)
body = self.rfile.read(n)
self._record(body)
# 故障注入(多账号路由测试):收到的 Authorization 含 UPSTREAM_FAIL_AUTH 子串时,
# 模拟「上游账号额度耗尽」——回 401 + 响应体标记(与 MiniMax 的 status_code:1004 同构),
# 验证网关「状态码 + body marker 双重检测 → 标记冷却 → 下次自动切到下一个账号」。
auth_hdr = self.headers.get("Authorization", "-")
exhaust_hit = bool(FAIL_AUTH) and FAIL_AUTH in auth_hdr
if not exhaust_hit and FAIL_ALWAYS_AUTH and FAIL_ALWAYS_AUTH in auth_hdr:
exhaust_hit = True
if not exhaust_hit and FAIL_FIRST_AUTH and FAIL_FIRST_AUTH in auth_hdr:
with _first_lock:
if auth_hdr not in _first_seen:
_first_seen.add(auth_hdr)
exhaust_hit = True
if exhaust_hit:
payload = json.dumps(
{"base_resp": {"status_code": 1004, "status_msg": "exhausted"}}
).encode()
self.send_response(401)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
return
# 故障注入:body 带 `"boom":true` 时回 500,用来验证网关的
# 上游健康标记(log 阶段把 5xx 写进 uh: 缓存 → 列表标红)
if b'"boom"' in body:
payload = json.dumps({"error": {"message": "injected failure", "type": "boom"}}).encode()
self.send_response(500)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
return
if b'"stream"' in body and b"true" in body:
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.send_header("Connection", "close")
self.end_headers()
for i in range(3):
self.wfile.write(('data: {"i":%d}\n\n' % i).encode())
self.wfile.flush()
time.sleep(0.4)
# token 计量探针:body 带 "usage_probe" 才在流尾追加 usage 块。
# 不带标记时保持 3 块不变——sse_test.py 的 CHUNKS=3 契约不能破。
if b'"usage_probe"' in body:
self.wfile.write(b'data: {"choices":[],"usage":{"prompt_tokens":11,"completion_tokens":22}}\n\n')
self.wfile.write(b'data: [DONE]\n\n')
self.wfile.flush()
self.close_connection = True
else:
payload = json.dumps({
"id": "chatcmpl-x",
"choices": [{"message": {"role": "assistant", "content": "pong"}}],
"usage": {"prompt_tokens": 7, "completion_tokens": 3},
}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
def log_message(self, *a):
pass
ThreadingHTTPServer(("127.0.0.1", PORT), H).serve_forever()