- reconciler:可注入时钟 + 唯一判定 pending_timed_out(saturating_sub 防回拨);超时真调 destroy、释放容量、/report 留带原因失败、本轮不重建(下轮按期望态重试) - 修真 bug:PendingSession.created_at 曾写 now_secs(),与判定时钟分叉 ⇒ 超时永不触发,单测直接抓到 - ControlPlaneConfig / CLI:--pending-timeout SECS(0=显式禁用),默认值只有 DEFAULT_PENDING_TIMEOUT_SECS 一份常量;help 数字由守卫测试对拍 - 新增 8 个构造用例 + 跨文件判据自测 16 条 / 破坏实验 15 条,并入主门禁第 25 条(分母仍 25) - 文档:ARCH §1.6/§8、FINDINGS §12、README/INDEX/CLAUDE/真机 README/env 样例/MEMORY 同步;HTML 重新生成并校验 - 诚实边界:未做真机验证、真机 musl 产物未更新、超时后下一轮重试不是永久隔离
238 lines
8.9 KiB
Python
238 lines
8.9 KiB
Python
#!/usr/bin/env python3
|
||
"""pending 超时回收 · 跨文件判据自测。
|
||
|
||
# 这条判据守的是什么
|
||
|
||
超时回收的**判定实现**在 `mpc-tee/reconciler.rs`(由门禁第 1 条
|
||
`cargo test --workspace` 真跑单测覆盖)。但还有一类缺陷是单测不
|
||
好抓的:**跨文件口径分叉 / 接线漏一环** ——
|
||
|
||
1. 默认值在 `ControlPlaneConfig` 里另写一个数字 ⇒ 控制面 900、
|
||
直接构造 reconciler 300,跑起来才发现行为不一致;
|
||
2. `reap_timed_out_pending` 只清记录、不调 `destroy` ⇒ guest 还在跑;
|
||
3. 回收放在容量记账**之后** ⇒ 本轮别的会话仍以为机器是满的;
|
||
4. 回收后本轮立刻重建 ⇒ 只是换个 sandbox id 继续占着机器;
|
||
5. 失败记录不带原因 ⇒ `/report` 里分不清「证明被拒」和「证明没来」;
|
||
6. CLI 没解析 / help 没列 / 组合根没透传 ⇒ flag 形同虚设;
|
||
7. 判定用普通减法 ⇒ 时钟回拨 wrap 成「超时」,真在跑的 guest 被误杀。
|
||
|
||
所以本脚本做**静态形态判据**:只查代码,不查注释;每条判据都由
|
||
`selftest-pending-timeout-break.py` 破坏一次,要求必须转红。
|
||
|
||
★ 与真机端点门禁的区别:那边的判据**永远不会被执行**(真机不在门禁里),
|
||
静态自测是唯一防线;这里的服务端逻辑会被 cargo test 真跑,
|
||
静态自测补的是「跨文件接线」这一层,两者不互相替代。
|
||
|
||
用法:
|
||
python3 tools/selftest-pending-timeout.py
|
||
"""
|
||
|
||
import os
|
||
import sys
|
||
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
ROOT = os.path.dirname(HERE)
|
||
FILES = {
|
||
'rec': os.path.join('crates', 'mpc-tee', 'src', 'reconciler.rs'),
|
||
'cp': os.path.join('crates', 'mpc-tee', 'src', 'control_plane.rs'),
|
||
'cli': os.path.join('crates', 'mpc-cli', 'src', 'control_plane.rs'),
|
||
}
|
||
|
||
OK_N = 0
|
||
FAIL_N = 0
|
||
|
||
|
||
def ok(msg):
|
||
global OK_N
|
||
OK_N += 1
|
||
print('ok %s' % msg)
|
||
|
||
|
||
def fail(msg):
|
||
global FAIL_N
|
||
FAIL_N += 1
|
||
print('FAIL %s' % msg, file=sys.stderr)
|
||
|
||
|
||
def _read(key):
|
||
with open(os.path.join(ROOT, FILES[key]), encoding='utf-8') as f:
|
||
return f.read()
|
||
|
||
|
||
def _strip_comments(src):
|
||
"""只看**代码**:剥掉 Rust 行注释 + 行内 ` //` 之后的内容。
|
||
|
||
★ 不剥就会「自指型假绿」:删了代码、注释里还留着历史说明,
|
||
判据照样绿。用 ` //`(前面要有空格)切断,避免误砍 http://。
|
||
"""
|
||
out = []
|
||
for ln in src.splitlines():
|
||
if ln.lstrip().startswith('//'):
|
||
continue
|
||
idx = ln.find(' //')
|
||
if idx >= 0:
|
||
ln = ln[:idx]
|
||
out.append(ln)
|
||
return '\n'.join(out)
|
||
|
||
|
||
def _require(cond, msg):
|
||
if cond:
|
||
ok(msg)
|
||
else:
|
||
fail(msg)
|
||
|
||
|
||
# ── 判据 ────────────────────────────────────────────────────────────────
|
||
def check_single_timeout_constant(c):
|
||
"""① 默认上限只有**一份**定义,且是 900 秒。"""
|
||
rec, cp, cli = c['rec'], c['cp'], c['cli']
|
||
_require('pub const DEFAULT_PENDING_TIMEOUT_SECS: u64 = 900;' in rec,
|
||
'唯一常量:默认 900 秒定义在 reconciler')
|
||
_require('pub const DEFAULT_PENDING_TIMEOUT_SECS' not in cp
|
||
and 'pub const DEFAULT_PENDING_TIMEOUT_SECS' not in cli,
|
||
'唯一常量:控制面 / CLI 不许另定义一份')
|
||
|
||
|
||
def check_saturating_sub_boundary(c):
|
||
"""② 判定用 saturating_sub + >=(防回拨、边界不含糊)。"""
|
||
_require('now.saturating_sub(created_at) >= secs' in c['rec'],
|
||
'判定:saturating_sub 防回拨 + >= 边界')
|
||
|
||
|
||
def check_zero_timeout_is_disabled(c):
|
||
"""③ 0 秒上限按禁用处理,不许恒真。"""
|
||
_require('secs > 0 && now.saturating_sub(created_at) >= secs' in c['rec'],
|
||
'判定:0 秒上限按禁用,不恒真')
|
||
|
||
|
||
def check_reap_function_exists(c):
|
||
"""④ 必须有独立的超时回收函数。"""
|
||
_require('fn reap_timed_out_pending(' in c['rec'],
|
||
'回收:reap_timed_out_pending 函数存在')
|
||
|
||
|
||
def check_reap_really_destroys(c):
|
||
"""⑤ 回收函数体内必须真的调 destroy —— 只清记录等于没回收。"""
|
||
rec = c['rec']
|
||
start = rec.find('fn reap_timed_out_pending(')
|
||
end = rec.find('pub fn reconcile_once', start)
|
||
body = rec[start:end] if start >= 0 and end > start else ''
|
||
_require('self.dispatch.destroy(&p.endpoint, &p.sandbox_id)' in body,
|
||
'回收:真的调 destroy(不是只清记录)')
|
||
|
||
|
||
def check_reap_failure_has_reason(c):
|
||
"""⑥ 失败记录必须带可识别的超时特征串。"""
|
||
_require('证明超时未落地(pending 等待超过 {secs} 秒): 实例已回收' in c['rec'],
|
||
'回收:/report 失败记录带「证明超时未落地」特征')
|
||
|
||
|
||
def check_reap_runs_before_capacity_accounting(c):
|
||
"""⑦ 回收必须在容量记账之前,否则本轮容量没释放。"""
|
||
rec = c['rec']
|
||
call = rec.find('let timed_out = self.reap_timed_out_pending(&mut report);')
|
||
load = rec.find('let mut load: HashMap<String, usize> = HashMap::new();')
|
||
_require(call >= 0 and load > call >= 0,
|
||
'回收:0b 在容量记账之前(否则本轮容量没释放)')
|
||
|
||
|
||
def check_no_immediate_recreate(c):
|
||
"""⑧ 本轮必须跳过重建,否则只是换个 sandbox id。"""
|
||
rec = c['rec']
|
||
hit = rec.find('if timed_out.contains(&d.id) {')
|
||
_require(hit >= 0 and 'continue;' in rec[hit:hit + 200],
|
||
'回收:本轮跳过重建(否则只是换个 sandbox id)')
|
||
|
||
|
||
def check_config_field(c):
|
||
"""⑨ 控制面配置里必须有这个字段。"""
|
||
_require('pub pending_timeout_secs: u64,' in c['cp'],
|
||
'配置:ControlPlaneConfig 有 pending_timeout_secs')
|
||
|
||
|
||
def check_default_references_constant(c):
|
||
"""⑩ 默认值引用常量,不许另写数字。"""
|
||
_require('pending_timeout_secs: DEFAULT_PENDING_TIMEOUT_SECS,' in c['cp'],
|
||
'配置:默认值引用唯一常量(不另写数字)')
|
||
|
||
|
||
def check_composition_root_wires_and_zero_disables(c):
|
||
"""⑪ 组合根必须透传,且 0 映射成禁用。"""
|
||
_require('reconciler.set_pending_timeout(if config.pending_timeout_secs == 0 {' in c['cp'],
|
||
'组合根:透传且 0 = 显式禁用')
|
||
|
||
|
||
def check_cli_flag_parsed(c):
|
||
"""⑫ CLI 必须真的解析这个 flag。"""
|
||
_require('crate::parse_flag(args, "--pending-timeout")' in c['cli'],
|
||
'CLI:--pending-timeout 真的被解析')
|
||
|
||
|
||
def check_cli_help_number_guard(c):
|
||
"""⑬ help 列出 flag,且写死的默认秒数由守卫测试钉死。"""
|
||
cli = c['cli']
|
||
_require('--pending-timeout SECS' in cli
|
||
and 'USAGE.contains(&format!("默认 {} 秒", DEFAULT_PENDING_TIMEOUT_SECS))' in cli,
|
||
'CLI:help 列出 flag 且默认秒数由守卫测试钉死')
|
||
|
||
|
||
def check_destructive_unit_tests_present(c):
|
||
"""⑭ 服务端单测必须含边界 / 回拨 / 禁用 / 容量释放的构造样例。"""
|
||
names = [
|
||
'fn pending_timeout_pure_function_boundaries()',
|
||
'fn timed_out_pending_is_reclaimed_with_a_reason()',
|
||
'fn pending_timeout_can_be_disabled_explicitly()',
|
||
'fn evidence_arriving_before_timeout_cancels_the_timer()',
|
||
'fn timed_out_pending_releases_capacity()',
|
||
'fn clock_rollback_does_not_reclaim_a_live_pending_session()',
|
||
]
|
||
_require(all(n in c['rec'] for n in names) and 'u64::MAX - 10' in c['rec'],
|
||
'单测:边界 / 回拨 / 禁用 / 容量释放的破坏性构造样例齐全')
|
||
|
||
|
||
def check_composition_root_guard_test(c):
|
||
"""⑮ 控制面配置透传也要有守卫测试(跨端守卫)。"""
|
||
_require('fn pending_timeout_config_is_wired_into_reconciler()' in c['cp'],
|
||
'单测:控制面配置透传守卫')
|
||
|
||
|
||
CHECKS = [
|
||
check_single_timeout_constant,
|
||
check_saturating_sub_boundary,
|
||
check_zero_timeout_is_disabled,
|
||
check_reap_function_exists,
|
||
check_reap_really_destroys,
|
||
check_reap_failure_has_reason,
|
||
check_reap_runs_before_capacity_accounting,
|
||
check_no_immediate_recreate,
|
||
check_config_field,
|
||
check_default_references_constant,
|
||
check_composition_root_wires_and_zero_disables,
|
||
check_cli_flag_parsed,
|
||
check_cli_help_number_guard,
|
||
check_destructive_unit_tests_present,
|
||
check_composition_root_guard_test,
|
||
]
|
||
|
||
|
||
def main():
|
||
print('=== pending 超时回收 · 跨文件判据自测 ===')
|
||
for key, rel in FILES.items():
|
||
if not os.path.isfile(os.path.join(ROOT, rel)):
|
||
fail('找不到源码:%s' % rel)
|
||
sys.exit(1)
|
||
codes = {k: _strip_comments(_read(k)) for k in FILES}
|
||
for chk in CHECKS:
|
||
chk(codes)
|
||
print('')
|
||
print('=== 结果:%d 通过 / %d 失败 ===' % (OK_N, FAIL_N))
|
||
if FAIL_N:
|
||
sys.exit(1)
|
||
print('PENDING_TIMEOUT_SELFTEST_OK: 超时回收判据自测全绿')
|
||
sys.exit(0)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|