天锚定重置边界计算(resets)删掉:锚点算错过两次(1.0.14 / 1.0.24), 线上表现为「厂商 15:00 已重置、网关锁到 18:00」。resets 只存不路由。 新口径:uac: = <backoff>:<next_probe>:<hard_cap> - 到期放一个「无绑定/绑定失效」的真实请求过去探测,2xx → clear_cool 回队; - 仍耗尽 → 退避翻倍(首轮 ipass_probe_min,默认 60s,30min 硬顶无条件回队自愈); - 绑定流量永不被劫持当探测(保 prompt 缓存亲和)。 - pick_account 必须纯决策:四个 $llm_* 变量各求值一次,副作用只在 log 阶段记一次。 fix: 探测窗口判定改严格小于 —— 秒级时钟下 >= 会让「标记于 45.95s、next_probe=46」 的账号在 46.05s 即被当探测目标,1s 退避实际只等 0.05s,冷却形同虚设(回归暴露)。 回归 370 条全绿(网关 108 + LLM 262,新增「探测恢复闭环」剧本)。
143 lines
6.6 KiB
Python
143 lines
6.6 KiB
Python
"""假上游:记录它**实际收到**的 path / auth / host / body。
|
||
|
||
这是验证「下游 Key 是否真被换成上游真 Key」的唯一可靠手段 ——
|
||
只看网关自己的响应是看不出来的(网关返回 200 不代表上游收到了正确的头)。
|
||
|
||
UPSTREAM_PORT=18999 UPSTREAM_LOG=/path/to/upstream.log python3 upstream.py
|
||
|
||
带 `"stream": true` 的请求体会回 SSE,分 3 块、每块间隔 0.4s,
|
||
用来验证 nginx 侧 `proxy_buffering off` 是否生效(缓冲住的话会攒完一次性返回)。
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import threading
|
||
import time
|
||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||
|
||
LOG = os.environ.get("UPSTREAM_LOG", "/tmp/upstream.log")
|
||
PORT = int(os.environ.get("UPSTREAM_PORT", "18999"))
|
||
# 故障注入(多账号路由测试):收到的 Authorization 含此子串时模拟「上游账号额度耗尽」,
|
||
# 回 401 + 响应体标记(与 MiniMax 的 status_code:1004 同构),验证网关自动切换。
|
||
FAIL_AUTH = os.environ.get("UPSTREAM_FAIL_AUTH", "")
|
||
# 探测闭环测试:FAIL_FIRST 对某 auth **只失败第一次**(之后正常)——用来验证
|
||
# 「冷却 → 退避到期 → 真实请求探测成功 → 自动回队」;FAIL_ALWAYS 永远失败——
|
||
# 验证「探测失败退避翻倍、冷却期持续跳过、代价被封顶」。
|
||
FAIL_FIRST_AUTH = os.environ.get("UPSTREAM_FAIL_FIRST_AUTH", "")
|
||
FAIL_ALWAYS_AUTH = os.environ.get("UPSTREAM_FAIL_ALWAYS_AUTH", "")
|
||
_first_seen: set = set()
|
||
_first_lock = threading.Lock()
|
||
|
||
|
||
class H(BaseHTTPRequestHandler):
|
||
protocol_version = "HTTP/1.1"
|
||
|
||
def _record(self, body: bytes):
|
||
# 记录**全部**收到的头,而不是手挑几个字段。
|
||
# 教训:手挑字段会漏掉"没被替换的那个头"——第一版只记 Authorization,
|
||
# 于是 x-api-key 泄漏在整轮回归里全绿通过,完全看不见。
|
||
rec = {
|
||
"path": self.path,
|
||
"method": self.command,
|
||
"headers": {k.lower(): v for k, v in self.headers.items()},
|
||
# 下面几个是长期被断言的字段,保留以免旧断言失效
|
||
"auth": self.headers.get("Authorization", "-"),
|
||
"x_api_key": self.headers.get("x-api-key", "-"),
|
||
"anthropic_version": self.headers.get("anthropic-version", "-"),
|
||
"host": self.headers.get("Host", "-"),
|
||
"body": body.decode("utf-8", "replace"),
|
||
}
|
||
with open(LOG, "a") as f:
|
||
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||
|
||
def do_GET(self):
|
||
"""`GET /<slot>/v1/models` 必须原样透传到上游——这里验的就是这条。
|
||
|
||
网关自己的 `GET /v1/models`(无 slot 段)在本地应答、不会打到这里,
|
||
两条路互不干扰。
|
||
"""
|
||
self._record(b"")
|
||
payload = json.dumps({
|
||
"object": "list",
|
||
"data": [{"id": "upstream-model", "object": "model"}],
|
||
}).encode()
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "application/json")
|
||
self.send_header("Content-Length", str(len(payload)))
|
||
self.end_headers()
|
||
self.wfile.write(payload)
|
||
|
||
def do_POST(self):
|
||
n = int(self.headers.get("Content-Length") or 0)
|
||
body = self.rfile.read(n)
|
||
self._record(body)
|
||
|
||
# 故障注入(多账号路由测试):收到的 Authorization 含 UPSTREAM_FAIL_AUTH 子串时,
|
||
# 模拟「上游账号额度耗尽」——回 401 + 响应体标记(与 MiniMax 的 status_code:1004 同构),
|
||
# 验证网关「状态码 + body marker 双重检测 → 标记冷却 → 下次自动切到下一个账号」。
|
||
auth_hdr = self.headers.get("Authorization", "-")
|
||
exhaust_hit = bool(FAIL_AUTH) and FAIL_AUTH in auth_hdr
|
||
if not exhaust_hit and FAIL_ALWAYS_AUTH and FAIL_ALWAYS_AUTH in auth_hdr:
|
||
exhaust_hit = True
|
||
if not exhaust_hit and FAIL_FIRST_AUTH and FAIL_FIRST_AUTH in auth_hdr:
|
||
with _first_lock:
|
||
if auth_hdr not in _first_seen:
|
||
_first_seen.add(auth_hdr)
|
||
exhaust_hit = True
|
||
if exhaust_hit:
|
||
payload = json.dumps(
|
||
{"base_resp": {"status_code": 1004, "status_msg": "exhausted"}}
|
||
).encode()
|
||
self.send_response(401)
|
||
self.send_header("Content-Type", "application/json")
|
||
self.send_header("Content-Length", str(len(payload)))
|
||
self.end_headers()
|
||
self.wfile.write(payload)
|
||
return
|
||
|
||
# 故障注入:body 带 `"boom":true` 时回 500,用来验证网关的
|
||
# 上游健康标记(log 阶段把 5xx 写进 uh: 缓存 → 列表标红)
|
||
if b'"boom"' in body:
|
||
payload = json.dumps({"error": {"message": "injected failure", "type": "boom"}}).encode()
|
||
self.send_response(500)
|
||
self.send_header("Content-Type", "application/json")
|
||
self.send_header("Content-Length", str(len(payload)))
|
||
self.end_headers()
|
||
self.wfile.write(payload)
|
||
return
|
||
|
||
if b'"stream"' in body and b"true" in body:
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "text/event-stream")
|
||
self.send_header("Cache-Control", "no-cache")
|
||
self.send_header("Connection", "close")
|
||
self.end_headers()
|
||
for i in range(3):
|
||
self.wfile.write(('data: {"i":%d}\n\n' % i).encode())
|
||
self.wfile.flush()
|
||
time.sleep(0.4)
|
||
# token 计量探针:body 带 "usage_probe" 才在流尾追加 usage 块。
|
||
# 不带标记时保持 3 块不变——sse_test.py 的 CHUNKS=3 契约不能破。
|
||
if b'"usage_probe"' in body:
|
||
self.wfile.write(b'data: {"choices":[],"usage":{"prompt_tokens":11,"completion_tokens":22}}\n\n')
|
||
self.wfile.write(b'data: [DONE]\n\n')
|
||
self.wfile.flush()
|
||
self.close_connection = True
|
||
else:
|
||
payload = json.dumps({
|
||
"id": "chatcmpl-x",
|
||
"choices": [{"message": {"role": "assistant", "content": "pong"}}],
|
||
"usage": {"prompt_tokens": 7, "completion_tokens": 3},
|
||
}).encode()
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "application/json")
|
||
self.send_header("Content-Length", str(len(payload)))
|
||
self.end_headers()
|
||
self.wfile.write(payload)
|
||
|
||
def log_message(self, *a):
|
||
pass
|
||
|
||
|
||
ThreadingHTTPServer(("127.0.0.1", PORT), H).serve_forever()
|