Files
ssw-mpc/tools/selftest-pending-timeout.py
T
admin 4a750785f8 feat(cluster): pending 证明超时回收(默认 900 秒,可配可禁用)
- reconciler:可注入时钟 + 唯一判定 pending_timed_out(saturating_sub 防回拨);超时真调 destroy、释放容量、/report 留带原因失败、本轮不重建(下轮按期望态重试)

- 修真 bug:PendingSession.created_at 曾写 now_secs(),与判定时钟分叉 ⇒ 超时永不触发,单测直接抓到

- ControlPlaneConfig / CLI:--pending-timeout SECS(0=显式禁用),默认值只有 DEFAULT_PENDING_TIMEOUT_SECS 一份常量;help 数字由守卫测试对拍

- 新增 8 个构造用例 + 跨文件判据自测 16 条 / 破坏实验 15 条,并入主门禁第 25 条(分母仍 25)

- 文档:ARCH §1.6/§8、FINDINGS §12、README/INDEX/CLAUDE/真机 README/env 样例/MEMORY 同步;HTML 重新生成并校验

- 诚实边界:未做真机验证、真机 musl 产物未更新、超时后下一轮重试不是永久隔离
2026-11-09 21:22:53 +08:00

238 lines
8.9 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""pending 超时回收 · 跨文件判据自测。
# 这条判据守的是什么
超时回收的**判定实现**在 `mpc-tee/reconciler.rs`(由门禁第 1 条
`cargo test --workspace` 真跑单测覆盖)。但还有一类缺陷是单测不
好抓的:**跨文件口径分叉 / 接线漏一环** ——
1. 默认值在 `ControlPlaneConfig` 里另写一个数字 ⇒ 控制面 900、
直接构造 reconciler 300,跑起来才发现行为不一致;
2. `reap_timed_out_pending` 只清记录、不调 `destroy` ⇒ guest 还在跑;
3. 回收放在容量记账**之后** ⇒ 本轮别的会话仍以为机器是满的;
4. 回收后本轮立刻重建 ⇒ 只是换个 sandbox id 继续占着机器;
5. 失败记录不带原因 ⇒ `/report` 里分不清「证明被拒」和「证明没来」;
6. CLI 没解析 / help 没列 / 组合根没透传 ⇒ flag 形同虚设;
7. 判定用普通减法 ⇒ 时钟回拨 wrap 成「超时」,真在跑的 guest 被误杀。
所以本脚本做**静态形态判据**:只查代码,不查注释;每条判据都由
`selftest-pending-timeout-break.py` 破坏一次,要求必须转红。
★ 与真机端点门禁的区别:那边的判据**永远不会被执行**(真机不在门禁里),
静态自测是唯一防线;这里的服务端逻辑会被 cargo test 真跑,
静态自测补的是「跨文件接线」这一层,两者不互相替代。
用法:
python3 tools/selftest-pending-timeout.py
"""
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
FILES = {
'rec': os.path.join('crates', 'mpc-tee', 'src', 'reconciler.rs'),
'cp': os.path.join('crates', 'mpc-tee', 'src', 'control_plane.rs'),
'cli': os.path.join('crates', 'mpc-cli', 'src', 'control_plane.rs'),
}
OK_N = 0
FAIL_N = 0
def ok(msg):
global OK_N
OK_N += 1
print('ok %s' % msg)
def fail(msg):
global FAIL_N
FAIL_N += 1
print('FAIL %s' % msg, file=sys.stderr)
def _read(key):
with open(os.path.join(ROOT, FILES[key]), encoding='utf-8') as f:
return f.read()
def _strip_comments(src):
"""只看**代码**:剥掉 Rust 行注释 + 行内 ` //` 之后的内容。
★ 不剥就会「自指型假绿」:删了代码、注释里还留着历史说明,
判据照样绿。用 ` //`(前面要有空格)切断,避免误砍 http://。
"""
out = []
for ln in src.splitlines():
if ln.lstrip().startswith('//'):
continue
idx = ln.find(' //')
if idx >= 0:
ln = ln[:idx]
out.append(ln)
return '\n'.join(out)
def _require(cond, msg):
if cond:
ok(msg)
else:
fail(msg)
# ── 判据 ────────────────────────────────────────────────────────────────
def check_single_timeout_constant(c):
"""① 默认上限只有**一份**定义,且是 900 秒。"""
rec, cp, cli = c['rec'], c['cp'], c['cli']
_require('pub const DEFAULT_PENDING_TIMEOUT_SECS: u64 = 900;' in rec,
'唯一常量:默认 900 秒定义在 reconciler')
_require('pub const DEFAULT_PENDING_TIMEOUT_SECS' not in cp
and 'pub const DEFAULT_PENDING_TIMEOUT_SECS' not in cli,
'唯一常量:控制面 / CLI 不许另定义一份')
def check_saturating_sub_boundary(c):
"""② 判定用 saturating_sub + >=(防回拨、边界不含糊)。"""
_require('now.saturating_sub(created_at) >= secs' in c['rec'],
'判定:saturating_sub 防回拨 + >= 边界')
def check_zero_timeout_is_disabled(c):
"""③ 0 秒上限按禁用处理,不许恒真。"""
_require('secs > 0 && now.saturating_sub(created_at) >= secs' in c['rec'],
'判定:0 秒上限按禁用,不恒真')
def check_reap_function_exists(c):
"""④ 必须有独立的超时回收函数。"""
_require('fn reap_timed_out_pending(' in c['rec'],
'回收:reap_timed_out_pending 函数存在')
def check_reap_really_destroys(c):
"""⑤ 回收函数体内必须真的调 destroy —— 只清记录等于没回收。"""
rec = c['rec']
start = rec.find('fn reap_timed_out_pending(')
end = rec.find('pub fn reconcile_once', start)
body = rec[start:end] if start >= 0 and end > start else ''
_require('self.dispatch.destroy(&p.endpoint, &p.sandbox_id)' in body,
'回收:真的调 destroy(不是只清记录)')
def check_reap_failure_has_reason(c):
"""⑥ 失败记录必须带可识别的超时特征串。"""
_require('证明超时未落地(pending 等待超过 {secs} 秒): 实例已回收' in c['rec'],
'回收:/report 失败记录带「证明超时未落地」特征')
def check_reap_runs_before_capacity_accounting(c):
"""⑦ 回收必须在容量记账之前,否则本轮容量没释放。"""
rec = c['rec']
call = rec.find('let timed_out = self.reap_timed_out_pending(&mut report);')
load = rec.find('let mut load: HashMap<String, usize> = HashMap::new();')
_require(call >= 0 and load > call >= 0,
'回收:0b 在容量记账之前(否则本轮容量没释放)')
def check_no_immediate_recreate(c):
"""⑧ 本轮必须跳过重建,否则只是换个 sandbox id。"""
rec = c['rec']
hit = rec.find('if timed_out.contains(&d.id) {')
_require(hit >= 0 and 'continue;' in rec[hit:hit + 200],
'回收:本轮跳过重建(否则只是换个 sandbox id)')
def check_config_field(c):
"""⑨ 控制面配置里必须有这个字段。"""
_require('pub pending_timeout_secs: u64,' in c['cp'],
'配置:ControlPlaneConfig 有 pending_timeout_secs')
def check_default_references_constant(c):
"""⑩ 默认值引用常量,不许另写数字。"""
_require('pending_timeout_secs: DEFAULT_PENDING_TIMEOUT_SECS,' in c['cp'],
'配置:默认值引用唯一常量(不另写数字)')
def check_composition_root_wires_and_zero_disables(c):
"""⑪ 组合根必须透传,且 0 映射成禁用。"""
_require('reconciler.set_pending_timeout(if config.pending_timeout_secs == 0 {' in c['cp'],
'组合根:透传且 0 = 显式禁用')
def check_cli_flag_parsed(c):
"""⑫ CLI 必须真的解析这个 flag。"""
_require('crate::parse_flag(args, "--pending-timeout")' in c['cli'],
'CLI:--pending-timeout 真的被解析')
def check_cli_help_number_guard(c):
"""⑬ help 列出 flag,且写死的默认秒数由守卫测试钉死。"""
cli = c['cli']
_require('--pending-timeout SECS' in cli
and 'USAGE.contains(&format!("默认 {} 秒", DEFAULT_PENDING_TIMEOUT_SECS))' in cli,
'CLI:help 列出 flag 且默认秒数由守卫测试钉死')
def check_destructive_unit_tests_present(c):
"""⑭ 服务端单测必须含边界 / 回拨 / 禁用 / 容量释放的构造样例。"""
names = [
'fn pending_timeout_pure_function_boundaries()',
'fn timed_out_pending_is_reclaimed_with_a_reason()',
'fn pending_timeout_can_be_disabled_explicitly()',
'fn evidence_arriving_before_timeout_cancels_the_timer()',
'fn timed_out_pending_releases_capacity()',
'fn clock_rollback_does_not_reclaim_a_live_pending_session()',
]
_require(all(n in c['rec'] for n in names) and 'u64::MAX - 10' in c['rec'],
'单测:边界 / 回拨 / 禁用 / 容量释放的破坏性构造样例齐全')
def check_composition_root_guard_test(c):
"""⑮ 控制面配置透传也要有守卫测试(跨端守卫)。"""
_require('fn pending_timeout_config_is_wired_into_reconciler()' in c['cp'],
'单测:控制面配置透传守卫')
CHECKS = [
check_single_timeout_constant,
check_saturating_sub_boundary,
check_zero_timeout_is_disabled,
check_reap_function_exists,
check_reap_really_destroys,
check_reap_failure_has_reason,
check_reap_runs_before_capacity_accounting,
check_no_immediate_recreate,
check_config_field,
check_default_references_constant,
check_composition_root_wires_and_zero_disables,
check_cli_flag_parsed,
check_cli_help_number_guard,
check_destructive_unit_tests_present,
check_composition_root_guard_test,
]
def main():
print('=== pending 超时回收 · 跨文件判据自测 ===')
for key, rel in FILES.items():
if not os.path.isfile(os.path.join(ROOT, rel)):
fail('找不到源码:%s' % rel)
sys.exit(1)
codes = {k: _strip_comments(_read(k)) for k in FILES}
for chk in CHECKS:
chk(codes)
print('')
print('=== 结果:%d 通过 / %d 失败 ===' % (OK_N, FAIL_N))
if FAIL_N:
sys.exit(1)
print('PENDING_TIMEOUT_SELFTEST_OK: 超时回收判据自测全绿')
sys.exit(0)
if __name__ == '__main__':
main()