- llm_usage.rs: body filter 旁路(FFI 注册 top_body_filter,无需额外 C 模块), 透传 SSE 的同时抓尾部 usage chunk(OpenAI stream_options.include_usage / Anthropic message_delta), 解析 prompt/completion tokens 写 shm utt:/utms: 键 - llm_audit.rs: log 阶段把请求体临时文件复制进审计目录(audit/ = data 兄弟目录), meta.jsonl 记 user/slot/status/token;10 GiB 容量阈值触发式清理,按日期目录从最旧删起 - conf/nginx.conf: LLM location 去掉 proxy_request_buffering off(它管请求体方向、 与 SSE 无关且与 client_body_in_file_only 互斥),SSE 流式由 proxy_buffering off 保障 - stats/api: 用量接口增加 tokens 维度(today/total per user per slot) - ipass-agent.py: 容器重建时自动挂审计卷(宿主 audit/ = data 兄弟目录,自动建目录) - update-agent-face.sh: 控制面自更新脚本(.11 上一行命令完成下载/备份/校验/重启/自检) - tests: LLM 117/117(+6: 审计 4 + token 2)、网关 90/90,共 207 全绿
109 lines
4.6 KiB
Python
109 lines
4.6 KiB
Python
"""假上游:记录它**实际收到**的 path / auth / host / body。
|
||
|
||
这是验证「下游 Key 是否真被换成上游真 Key」的唯一可靠手段 ——
|
||
只看网关自己的响应是看不出来的(网关返回 200 不代表上游收到了正确的头)。
|
||
|
||
UPSTREAM_PORT=18999 UPSTREAM_LOG=/path/to/upstream.log python3 upstream.py
|
||
|
||
带 `"stream": true` 的请求体会回 SSE,分 3 块、每块间隔 0.4s,
|
||
用来验证 nginx 侧 `proxy_buffering off` 是否生效(缓冲住的话会攒完一次性返回)。
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import time
|
||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||
|
||
LOG = os.environ.get("UPSTREAM_LOG", "/tmp/upstream.log")
|
||
PORT = int(os.environ.get("UPSTREAM_PORT", "18999"))
|
||
|
||
|
||
class H(BaseHTTPRequestHandler):
|
||
protocol_version = "HTTP/1.1"
|
||
|
||
def _record(self, body: bytes):
|
||
# 记录**全部**收到的头,而不是手挑几个字段。
|
||
# 教训:手挑字段会漏掉"没被替换的那个头"——第一版只记 Authorization,
|
||
# 于是 x-api-key 泄漏在整轮回归里全绿通过,完全看不见。
|
||
rec = {
|
||
"path": self.path,
|
||
"method": self.command,
|
||
"headers": {k.lower(): v for k, v in self.headers.items()},
|
||
# 下面几个是长期被断言的字段,保留以免旧断言失效
|
||
"auth": self.headers.get("Authorization", "-"),
|
||
"x_api_key": self.headers.get("x-api-key", "-"),
|
||
"anthropic_version": self.headers.get("anthropic-version", "-"),
|
||
"host": self.headers.get("Host", "-"),
|
||
"body": body.decode("utf-8", "replace"),
|
||
}
|
||
with open(LOG, "a") as f:
|
||
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||
|
||
def do_GET(self):
|
||
"""`GET /<slot>/v1/models` 必须原样透传到上游——这里验的就是这条。
|
||
|
||
网关自己的 `GET /v1/models`(无 slot 段)在本地应答、不会打到这里,
|
||
两条路互不干扰。
|
||
"""
|
||
self._record(b"")
|
||
payload = json.dumps({
|
||
"object": "list",
|
||
"data": [{"id": "upstream-model", "object": "model"}],
|
||
}).encode()
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "application/json")
|
||
self.send_header("Content-Length", str(len(payload)))
|
||
self.end_headers()
|
||
self.wfile.write(payload)
|
||
|
||
def do_POST(self):
|
||
n = int(self.headers.get("Content-Length") or 0)
|
||
body = self.rfile.read(n)
|
||
self._record(body)
|
||
|
||
# 故障注入:body 带 `"boom":true` 时回 500,用来验证网关的
|
||
# 上游健康标记(log 阶段把 5xx 写进 uh: 缓存 → 列表标红)
|
||
if b'"boom"' in body:
|
||
payload = json.dumps({"error": {"message": "injected failure", "type": "boom"}}).encode()
|
||
self.send_response(500)
|
||
self.send_header("Content-Type", "application/json")
|
||
self.send_header("Content-Length", str(len(payload)))
|
||
self.end_headers()
|
||
self.wfile.write(payload)
|
||
return
|
||
|
||
if b'"stream"' in body and b"true" in body:
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "text/event-stream")
|
||
self.send_header("Cache-Control", "no-cache")
|
||
self.send_header("Connection", "close")
|
||
self.end_headers()
|
||
for i in range(3):
|
||
self.wfile.write(('data: {"i":%d}\n\n' % i).encode())
|
||
self.wfile.flush()
|
||
time.sleep(0.4)
|
||
# token 计量探针:body 带 "usage_probe" 才在流尾追加 usage 块。
|
||
# 不带标记时保持 3 块不变——sse_test.py 的 CHUNKS=3 契约不能破。
|
||
if b'"usage_probe"' in body:
|
||
self.wfile.write(b'data: {"choices":[],"usage":{"prompt_tokens":11,"completion_tokens":22}}\n\n')
|
||
self.wfile.write(b'data: [DONE]\n\n')
|
||
self.wfile.flush()
|
||
self.close_connection = True
|
||
else:
|
||
payload = json.dumps({
|
||
"id": "chatcmpl-x",
|
||
"choices": [{"message": {"role": "assistant", "content": "pong"}}],
|
||
"usage": {"prompt_tokens": 7, "completion_tokens": 3},
|
||
}).encode()
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "application/json")
|
||
self.send_header("Content-Length", str(len(payload)))
|
||
self.end_headers()
|
||
self.wfile.write(payload)
|
||
|
||
def log_message(self, *a):
|
||
pass
|
||
|
||
|
||
ThreadingHTTPServer(("127.0.0.1", PORT), H).serve_forever()
|