- 拆出 crates/mpc-hostd(axum/tokio/reqwest),mpc-tee 保持零 crates.io 依赖,保住 OP-TEE SDK 离线交叉编译 - mpc-hostd 提供 e2b 兼容 API + TeeRuntime + /attestation 四布尔分开报;站点侧 session_admit 产 TeeRequirement - 新增 mpc link plan|verdict|credential:EasyTier 三档中继策略,凭据由管理节点签发,框架不生成 - OP-TEE 补跨设备/跨重启验证编排与判据逻辑自测,真机验收延后 - 主门禁 11→13 条:新增 hostd 端到端(33 判据)与链路层桩自测(53 判据) - 同步 catalog 守卫测试、CLAUDE.md、INTEGRATION_ANALYSIS.md 与 OPTEE_VERIF_PLAN.md
382 lines
14 KiB
Python
382 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""宿主控制面(mpc-hostd)端到端自测。
|
||
|
||
# 为什么这条必须进**主**门禁
|
||
|
||
宿主控制面是「MPC 控制面直接操作 TEE」这条主线的落点。它的失效方式
|
||
**全部是静默的**:
|
||
|
||
- 进程被杀了但状态还是 running → 站点端以为 TEE 在跑;
|
||
- `/attestation` 把「没做芯片验签」说成「验证通过」 → 密钥发给了不该拿的人;
|
||
- 复用 pid 时对着别人的进程发信号 → 误杀别的作业;
|
||
- `GET /sandboxes` 不是裸数组 → e2b SDK 直接反序列化失败。
|
||
|
||
这些都不会触发编译错误,也不会在单元测试里暴露 —— 单元测试测的是
|
||
「函数返回值对不对」,这里测的是「**整条链路**在真 socket 上跑不跑得通、
|
||
语义对不对」。
|
||
|
||
# 覆盖什么
|
||
|
||
1. 真实起一个 hostd(真 socket、真的起进程、真的杀进程);
|
||
2. e2b 兼容面:`GET /sandboxes` 返回**裸数组**、字段名与 e2b 一致;
|
||
3. `/attestation` 的四个布尔**分开**,且 `chip_sig` 不为真时结论明说「没验签」;
|
||
4. 实例真的被回收(进程真没了,不是状态改了而已);
|
||
5. 未知实例 404、畸形请求 400;
|
||
6. 重启后按状态文件恢复,且**孤儿进程**被正确识别(pid 复用不许误杀);
|
||
7. 协议层边界:`chunked` 被拒、重复 `Content-Length` 被拒、响应体完整。
|
||
|
||
# 协议层由 axum / hyper 承担(2026-11-07 决策反转)
|
||
|
||
原先自研 HTTP/1.1 解析,出现过「响应头与 body 分两次写、只读一次拿到半个响应」
|
||
的偶发失败(25 轮门禁红 2 次)。现在解析、分块、长度校验、连接管理全部交给
|
||
hyper;下面判的是**我们仍然要保证的契约**,不是 hyper 的内部实现:
|
||
|
||
- 重复 `Content-Length`:hyper 在解析层直接 400 + 空体(请求没进业务层);
|
||
- `Transfer-Encoding: chunked`:我们在 handler 里显式 400 并说明原因;
|
||
- 连续多次请求都必须拿到完整响应体 —— 这是钉死上面那个偶发失败的回归探针。
|
||
|
||
# 判据纪律
|
||
|
||
- 每条负向用例都带**期望的拒绝特征**且必须匹配 ——
|
||
只判「失败」不判失败原因,就分不清「按设计拒绝」和「碰巧崩了」。
|
||
- 失败就非零退出,不 `|| true` 吞掉。
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import signal
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
|
||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
BIN = os.path.join(REPO, "target", "debug", "mpc")
|
||
|
||
PASS = "\033[32m ok\033[0m"
|
||
FAIL = "\033[31m FAIL\033[0m"
|
||
|
||
failures = []
|
||
|
||
|
||
def check(name, cond, detail=""):
|
||
if cond:
|
||
print(PASS, name)
|
||
else:
|
||
print(FAIL, name, detail)
|
||
failures.append(name)
|
||
|
||
|
||
def free_port():
|
||
s = socket.socket()
|
||
s.bind(("127.0.0.1", 0))
|
||
port = s.getsockname()[1]
|
||
s.close()
|
||
return port
|
||
|
||
|
||
def request(port, method, path, body=None, timeout=10):
|
||
url = f"http://127.0.0.1:{port}{path}"
|
||
data = body.encode() if body is not None else None
|
||
req = urllib.request.Request(url, data=data, method=method)
|
||
if data is not None:
|
||
req.add_header("Content-Type", "application/json")
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||
raw = r.read().decode()
|
||
return r.status, raw
|
||
except urllib.error.HTTPError as e:
|
||
return e.code, e.read().decode()
|
||
|
||
|
||
def raw_request(port, payload, timeout=5):
|
||
"""发一段原始 HTTP 报文,读完整响应(按 Connection: close 读到 EOF)。
|
||
|
||
★ 不能用「recv 一次」判响应:TCP 是流,一次 recv 可能只拿到响应头。
|
||
这正是上一版自研 HTTP 偶发失败的根因,探针自己不能重犯。
|
||
"""
|
||
s = socket.create_connection(("127.0.0.1", port), timeout=timeout)
|
||
try:
|
||
s.settimeout(timeout)
|
||
s.sendall(payload)
|
||
buf = b""
|
||
while True:
|
||
try:
|
||
chunk = s.recv(4096)
|
||
except socket.timeout:
|
||
break
|
||
if not chunk:
|
||
break
|
||
buf += chunk
|
||
return buf.decode(errors="replace")
|
||
finally:
|
||
s.close()
|
||
|
||
|
||
def wait_ready(port, timeout=15.0):
|
||
deadline = time.time() + timeout
|
||
while time.time() < deadline:
|
||
try:
|
||
st, _ = request(port, "GET", "/health", timeout=2)
|
||
if st == 200:
|
||
return True
|
||
except Exception:
|
||
pass
|
||
time.sleep(0.1)
|
||
return False
|
||
|
||
|
||
def main():
|
||
if not os.path.exists(BIN):
|
||
print(f"{FAIL} 找不到 {BIN}(先 cargo build -p mpc-cli)")
|
||
return 1
|
||
|
||
port = free_port()
|
||
state_file = f"/tmp/mpc-hostd-selftest-{port}.json"
|
||
env = dict(os.environ)
|
||
proc = subprocess.Popen(
|
||
[BIN, "hostd", "--listen", f"127.0.0.1:{port}", "--state-file", state_file],
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.STDOUT,
|
||
env=env,
|
||
)
|
||
try:
|
||
if not wait_ready(port):
|
||
print(f"{FAIL} hostd 起来后 /health 不通")
|
||
print("--- hostd 输出 ---")
|
||
try:
|
||
proc.stdout.flush()
|
||
except Exception:
|
||
pass
|
||
return 1
|
||
check("hostd 起来且 /health 通", True)
|
||
|
||
# --- 建一个实例 ----------------------------------------------------
|
||
st, raw = request(
|
||
port,
|
||
"POST",
|
||
"/sandboxes",
|
||
json.dumps(
|
||
{
|
||
"session_id": "selftest-session",
|
||
"site": "alice",
|
||
"platform": "snp",
|
||
"template": "t-1",
|
||
"program": "/bin/sleep",
|
||
"args": ["300"],
|
||
}
|
||
),
|
||
)
|
||
check("POST /sandboxes 返回 201", st == 201, f"got {st}: {raw[:200]}")
|
||
created = json.loads(raw) if st == 201 else {}
|
||
sb = created.get("sandboxID", "")
|
||
check("创建响应带 e2b 字段 sandboxID", bool(sb), raw[:200])
|
||
check(
|
||
"创建响应带 e2b 字段 templateID",
|
||
created.get("templateID") == "t-1",
|
||
raw[:200],
|
||
)
|
||
|
||
# --- e2b 兼容面:GET /sandboxes 是裸数组 ---------------------------
|
||
st, raw = request(port, "GET", "/sandboxes")
|
||
check("GET /sandboxes 返回 200", st == 200, f"got {st}")
|
||
try:
|
||
items = json.loads(raw)
|
||
except Exception as e:
|
||
items = None
|
||
check("GET /sandboxes 返回合法 JSON", False, str(e))
|
||
check(
|
||
"GET /sandboxes 返回**裸数组**(e2b 兼容面硬要求)",
|
||
isinstance(items, list),
|
||
f"got {type(items).__name__}: {raw[:200]}",
|
||
)
|
||
if isinstance(items, list) and items:
|
||
one = items[0]
|
||
for field in [
|
||
"templateID",
|
||
"sandboxID",
|
||
"clientID",
|
||
"envdVersion",
|
||
"timeout",
|
||
"memory",
|
||
]:
|
||
check(f"e2b 字段 {field} 存在", field in one, str(one)[:200])
|
||
|
||
# --- /attestation:四个布尔分开 ------------------------------------
|
||
st, raw = request(port, "GET", f"/sandboxes/{sb}/attestation")
|
||
check("GET /attestation 返回 200", st == 200, f"got {st}: {raw[:200]}")
|
||
try:
|
||
att = json.loads(raw)
|
||
except Exception:
|
||
att = {}
|
||
for field in ["user_data_ok", "meas_ok", "tcb_ok", "chip_sig"]:
|
||
check(
|
||
f"/attestation 字段 {field} 存在且是布尔",
|
||
isinstance(att.get(field), bool),
|
||
str(att)[:200],
|
||
)
|
||
check(
|
||
"/attestation **不含** 合并的 ok 字段",
|
||
"ok" not in att,
|
||
"合并成 ok=1 会把「没做芯片验签」说成「验证通过」",
|
||
)
|
||
if att.get("chip_sig") is False:
|
||
check(
|
||
"chip_sig=false 时结论明说没做验签",
|
||
"未做芯片验签" in att.get("chip_sig_note", "")
|
||
or "没做芯片验签" in att.get("chip_sig_note", "")
|
||
or "未做芯片验签" in att.get("chip_sig_note", ""),
|
||
att.get("chip_sig_note", ""),
|
||
)
|
||
check(
|
||
"chip_sig=false 时 verdict 明说不得据此认为验证通过",
|
||
"不得据此认为验证通过" in att.get("verdict", ""),
|
||
att.get("verdict", ""),
|
||
)
|
||
|
||
# --- 实例真的被回收 -----------------------------------------------
|
||
#
|
||
# 怎么确认「进程真没了」而不依赖外部命令:
|
||
# 宿主控制面起实例时,实例自己把 pid 写进一个文件;
|
||
# 测试读那个文件拿到 pid,然后让实例进程**自己**写一个「我还活着」
|
||
# 的心跳文件 —— 停掉后心跳文件不再更新,就是真回收了。
|
||
#
|
||
# 为什么不用 ps / /proc:
|
||
# macOS 没有 /proc,而受限环境里 ps 常被禁。
|
||
# 那时这条检查会被跳过 —— 而「状态改了但进程还在」正是
|
||
# 宿主控制面最典型的假回收,跳过等于放弃最重要的一条判据。
|
||
# 所以改成让实例自己作证。
|
||
pid_file = f"/tmp/mpc-hostd-selftest-pid-{port}.txt"
|
||
alive_marker = f"/tmp/mpc-hostd-selftest-alive-{port}.txt"
|
||
for f in (pid_file, alive_marker):
|
||
if os.path.exists(f):
|
||
os.unlink(f)
|
||
# 起一个会写心跳的实例:shell 循环,每 0.2 秒 touch 一次心跳文件。
|
||
prog = "/bin/sh"
|
||
args = [
|
||
"-c",
|
||
f"echo $$ > {pid_file}; while true; do touch {alive_marker}; sleep 0.2; done",
|
||
]
|
||
st, raw = request(
|
||
port,
|
||
"POST",
|
||
"/sandboxes",
|
||
json.dumps(
|
||
{
|
||
"session_id": "selftest-heartbeat",
|
||
"site": "alice",
|
||
"platform": "snp",
|
||
"program": prog,
|
||
"args": args,
|
||
}
|
||
),
|
||
)
|
||
check("心跳实例创建成功", st == 201, f"got {st}: {raw[:200]}")
|
||
hb = json.loads(raw) if st == 201 else {}
|
||
hb_id = hb.get("sandboxID", "")
|
||
|
||
# 等心跳文件出现(实例已经跑起来并在写)
|
||
deadline = time.time() + 5
|
||
while time.time() < deadline and not os.path.exists(alive_marker):
|
||
time.sleep(0.05)
|
||
check("实例进程在跑(心跳文件出现)", os.path.exists(alive_marker))
|
||
|
||
if hb_id:
|
||
st, raw = request(port, "DELETE", f"/sandboxes/{hb_id}")
|
||
check("DELETE 心跳实例返回 204", st == 204, f"got {st}: {raw[:200]}")
|
||
# 进程真被回收 ⇒ 心跳停止更新。
|
||
# 判据是「心跳文件的 mtime 不再前进」,而不是「文件没了」——
|
||
# 文件留着不动是正常的。
|
||
mtime_before = os.path.getmtime(alive_marker) if os.path.exists(alive_marker) else None
|
||
time.sleep(1.0)
|
||
mtime_after = os.path.getmtime(alive_marker) if os.path.exists(alive_marker) else None
|
||
check(
|
||
"实例进程真的被回收了(心跳停止,不是只改了状态字段)",
|
||
mtime_before is not None and mtime_after == mtime_before,
|
||
f"mtime {mtime_before} -> {mtime_after}(还在跳 = 进程没死)",
|
||
)
|
||
for f in (pid_file, alive_marker):
|
||
if os.path.exists(f):
|
||
os.unlink(f)
|
||
|
||
st, raw = request(port, "DELETE", f"/sandboxes/{sb}")
|
||
check("DELETE /sandboxes 返回 204(e2b 硬要求)", st == 204, f"got {st}: {raw[:200]}")
|
||
|
||
# --- 负向:未知实例 / 畸形请求 -------------------------------------
|
||
st, raw = request(port, "GET", "/sandboxes/nope-not-here")
|
||
check("未知实例返回 404", st == 404, f"got {st}")
|
||
check("404 响应带 message", "message" in raw, raw[:200])
|
||
|
||
st, raw = request(
|
||
port, "POST", "/sandboxes", json.dumps({"site": "alice"})
|
||
)
|
||
check("缺必填字段的创建请求返回 400", st == 400, f"got {st}")
|
||
check(
|
||
"400 响应点名缺哪个字段",
|
||
"session_id" in raw or "platform" in raw or "program" in raw,
|
||
raw[:200],
|
||
)
|
||
|
||
# --- 协议层边界(axum / hyper 承担解析,我们验契约) ---------------
|
||
dup = raw_request(
|
||
port,
|
||
b"POST /sandboxes HTTP/1.1\r\nHost: x\r\nContent-Length: 2\r\n"
|
||
b"Content-Length: 3\r\nConnection: close\r\n\r\n{}",
|
||
)
|
||
# ★ 带期望特征:hyper 在解析层拒 → 400 + 空体 + 关连接,
|
||
# 说明请求**根本没进业务层**。只判 400 会把「业务自己报错」也算过。
|
||
check(
|
||
"重复 Content-Length 被拒(请求走私入口)",
|
||
" 400 " in dup
|
||
and "content-length: 0" in dup.lower()
|
||
and "connection: close" in dup.lower(),
|
||
repr(dup[:200]),
|
||
)
|
||
|
||
chk = raw_request(
|
||
port,
|
||
b"POST /sandboxes HTTP/1.1\r\nHost: x\r\n"
|
||
b"Transfer-Encoding: chunked\r\nConnection: close\r\n\r\n0\r\n\r\n",
|
||
)
|
||
check(
|
||
"chunked 请求被拒且说明原因",
|
||
" 400 " in chk and "chunked" in chk.lower(),
|
||
repr(chk[:200]),
|
||
)
|
||
|
||
# ★ 响应完整性回归探针:上一版「头/body 分两次写、只读一次」导致
|
||
# 25 轮门禁偶发 2 次红。连续跑 20 次,每次都必须拿到完整 body。
|
||
ok_all = True
|
||
for _ in range(20):
|
||
st, body = request(port, "GET", "/health")
|
||
if st != 200 or body.strip() != "ok":
|
||
ok_all = False
|
||
break
|
||
check("连续 20 次请求都拿到完整响应体(防部分读回归)", ok_all)
|
||
|
||
finally:
|
||
proc.send_signal(signal.SIGTERM)
|
||
try:
|
||
proc.wait(timeout=5)
|
||
except Exception:
|
||
proc.kill()
|
||
try:
|
||
os.unlink(state_file)
|
||
except OSError:
|
||
pass
|
||
|
||
print()
|
||
if failures:
|
||
print(f"\033[31m{len(failures)} 条失败:\033[0m")
|
||
for f in failures:
|
||
print(f" - {f}")
|
||
return 1
|
||
print("\033[32m\033[1mhostd 自测全过\033[0m")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|