Files
ssw-mpc/tools/selftest-pending-timeout-break.py
T
admin 4a750785f8 feat(cluster): pending 证明超时回收(默认 900 秒,可配可禁用)
- reconciler:可注入时钟 + 唯一判定 pending_timed_out(saturating_sub 防回拨);超时真调 destroy、释放容量、/report 留带原因失败、本轮不重建(下轮按期望态重试)

- 修真 bug:PendingSession.created_at 曾写 now_secs(),与判定时钟分叉 ⇒ 超时永不触发,单测直接抓到

- ControlPlaneConfig / CLI:--pending-timeout SECS(0=显式禁用),默认值只有 DEFAULT_PENDING_TIMEOUT_SECS 一份常量;help 数字由守卫测试对拍

- 新增 8 个构造用例 + 跨文件判据自测 16 条 / 破坏实验 15 条,并入主门禁第 25 条(分母仍 25)

- 文档:ARCH §1.6/§8、FINDINGS §12、README/INDEX/CLAUDE/真机 README/env 样例/MEMORY 同步;HTML 重新生成并校验

- 诚实边界:未做真机验证、真机 musl 产物未更新、超时后下一轮重试不是永久隔离
2026-11-09 21:22:53 +08:00

260 lines
9.0 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""pending 超时回收 · 跨文件判据**破坏实验**。
对 `selftest-pending-timeout.py` 的每条判据做一次变异,要求自测
**必须转红**。正向路径通过证明不了判据在起作用 —— 恒绿判据和真判据
在正向路径上表现完全一样。
★ 每条变异都**双向确认生效**(证据串消失 / 新增),SKIP 一律不算通过。
★ 只改临时副本,仓库文件一个都不动;不碰真机、不解密任何真口令。
用法:
python3 tools/selftest-pending-timeout-break.py
"""
import os
import shutil
import subprocess
import sys
import tempfile
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
FILES = {
'rec': os.path.join('crates', 'mpc-tee', 'src', 'reconciler.rs'),
'cp': os.path.join('crates', 'mpc-tee', 'src', 'control_plane.rs'),
'cli': os.path.join('crates', 'mpc-cli', 'src', 'control_plane.rs'),
}
SELFTEST_REL = os.path.join('tools', 'selftest-pending-timeout.py')
OK_N = 0
FAIL_N = 0
def ok(msg):
global OK_N
OK_N += 1
print('ok %s' % msg)
def fail(msg):
global FAIL_N
FAIL_N += 1
print('FAIL %s' % msg, file=sys.stderr)
def _prep():
work = tempfile.mkdtemp(prefix='pt-break-')
for rel in list(FILES.values()) + [SELFTEST_REL]:
dst = os.path.join(work, rel)
os.makedirs(os.path.dirname(dst), exist_ok=True)
shutil.copy(os.path.join(ROOT, rel), dst)
return work
def _read(work, rel):
with open(os.path.join(work, rel), encoding='utf-8') as f:
return f.read()
def _write(work, rel, text):
with open(os.path.join(work, rel), 'w', encoding='utf-8') as f:
f.write(text)
def _run_selftest(work):
p = subprocess.run(
[sys.executable, os.path.join(work, SELFTEST_REL)],
capture_output=True, text=True, cwd=work, timeout=120)
return p.returncode, p.stdout + p.stderr
def _replace(src, old, new, evidence=None):
if old not in src:
return None, (evidence or old)
return src.replace(old, new, 1), (evidence or old)
def _drop_line(src, needle):
out = [ln for ln in src.splitlines(keepends=True) if needle not in ln]
if len(out) == len(src.splitlines(keepends=True)):
return None, needle
return ''.join(out), needle
def _variant(key, mutate, mode, expect, label):
work = _prep()
try:
rel = FILES[key]
src = _read(work, rel)
new, evidence = mutate(src)
if new is None or new == src:
fail('破坏·%s:变异未生效' % label)
return False
present = evidence in new
if mode == 'gone':
good = (not present) and (evidence in src)
else:
good = present and (evidence not in src)
if not good:
fail('破坏·%s:证据串校验失败(present=%s, 证据 %r)'
% (label, present, evidence[:70]))
return False
_write(work, rel, new)
rc, out = _run_selftest(work)
if rc != 0 and ('FAIL' in out) and (expect in out):
ok('破坏·%s:判据如预期转红(证明它不是恒绿)' % label)
return True
fail('破坏·%s:判据**没有**转红 ⇒ 它是恒绿' % label)
for ln in out.splitlines()[:22]:
print(' | %s' % ln, file=sys.stderr)
return False
finally:
shutil.rmtree(work, ignore_errors=True)
# ── 变异 ────────────────────────────────────────────────────────────────
def m_drop_constant(src):
return _drop_line(src, 'pub const DEFAULT_PENDING_TIMEOUT_SECS: u64 = 900;')
def m_wrapping_sub(src):
return _replace(src,
'now.saturating_sub(created_at) >= secs',
'now.wrapping_sub(created_at) >= secs')
def m_zero_not_disabled(src):
return _replace(src,
'secs > 0 && now.saturating_sub(created_at) >= secs',
'now.saturating_sub(created_at) >= secs',
evidence='secs > 0 && ')
def m_no_reap_fn(src):
return _replace(src, 'fn reap_timed_out_pending(',
'fn reap_timed_out_pending_disabled(')
def m_no_destroy(src):
"""只在 reap_timed_out_pending 函数体内替换,别误伤 reap_unwanted。"""
a = src.find('fn reap_timed_out_pending(')
b = src.find('pub fn reconcile_once', a)
if a < 0 or b < 0:
return None, 'fn reap_timed_out_pending('
seg = src[a:b]
old = 'self.dispatch.destroy(&p.endpoint, &p.sandbox_id)'
if old not in seg:
return None, old
new_seg = seg.replace(old, 'let _ = (&p.endpoint, &p.sandbox_id)', 1)
return src[:a] + new_seg + src[b:], 'let _ = (&p.endpoint, &p.sandbox_id)'
def m_no_reason(src):
return _replace(src,
'证明超时未落地(pending 等待超过 {secs} 秒): 实例已回收',
'超时回收')
def m_no_reap_call(src):
return _drop_line(src,
'let timed_out = self.reap_timed_out_pending(&mut report);')
def m_immediate_recreate(src):
return _replace(src, 'if timed_out.contains(&d.id) {', 'if false {')
def m_config_wrong_type(src):
return _replace(src, 'pub pending_timeout_secs: u64,',
'pub pending_timeout_secs: u32,')
def m_default_hardcoded(src):
return _replace(src,
'pending_timeout_secs: DEFAULT_PENDING_TIMEOUT_SECS,',
'pending_timeout_secs: 300,')
def m_no_wire(src):
return _replace(src,
'reconciler.set_pending_timeout(if config.pending_timeout_secs == 0 {',
'reconciler.set_pending_timeout(None); if false {')
def m_cli_flag_renamed(src):
return _replace(src,
'crate::parse_flag(args, "--pending-timeout")',
'crate::parse_flag(args, "--timeout")')
def m_cli_guard_dropped(src):
return _drop_line(
src,
'USAGE.contains(&format!("默认 {} 秒", DEFAULT_PENDING_TIMEOUT_SECS))')
def m_unit_test_dropped(src):
return _replace(src,
'fn timed_out_pending_is_reclaimed_with_a_reason()',
'fn timed_out_pending_is_reclaimed_with_a_reason_disabled()')
def m_cp_guard_dropped(src):
return _replace(src,
'fn pending_timeout_config_is_wired_into_reconciler()',
'fn pending_timeout_config_is_wired_into_reconciler_disabled()')
VARIANTS = [
('rec', m_drop_constant, 'gone',
'唯一常量:默认 900 秒定义在 reconciler', '删掉唯一常量定义'),
('rec', m_wrapping_sub, 'gone',
'判定:saturating_sub 防回拨 + >= 边界', '改成 wrapping_sub(时钟回拨会误杀)'),
('rec', m_zero_not_disabled, 'gone',
'判定:0 秒上限按禁用,不恒真', '去掉 0 秒禁用保护'),
('rec', m_no_reap_fn, 'gone',
'回收:reap_timed_out_pending 函数存在', '回收函数改名(接线断掉)'),
('rec', m_no_destroy, 'added',
'回收:真的调 destroy(不是只清记录)', '只清记录不调 destroy'),
('rec', m_no_reason, 'gone',
'回收:/report 失败记录带「证明超时未落地」特征', '失败记录不带原因'),
('rec', m_no_reap_call, 'gone',
'回收:0b 在容量记账之前(否则本轮容量没释放)', '删掉 0b 回收调用'),
('rec', m_immediate_recreate, 'gone',
'回收:本轮跳过重建(否则只是换个 sandbox id)', '回收后本轮立刻重建'),
('cp', m_config_wrong_type, 'gone',
'配置:ControlPlaneConfig 有 pending_timeout_secs', '配置字段类型改成 u32'),
('cp', m_default_hardcoded, 'gone',
'配置:默认值引用唯一常量(不另写数字)', '默认值另写一个数字(口径分叉)'),
('cp', m_no_wire, 'gone',
'组合根:透传且 0 = 显式禁用', '组合根不透传 / 忽略 0 的禁用语义'),
('cli', m_cli_flag_renamed, 'gone',
'CLI:--pending-timeout 真的被解析', 'CLI flag 改名'),
('cli', m_cli_guard_dropped, 'gone',
'CLI:help 列出 flag 且默认秒数由守卫测试钉死', '删掉 help 数字守卫'),
('rec', m_unit_test_dropped, 'gone',
'单测:边界 / 回拨 / 禁用 / 容量释放的破坏性构造样例齐全', '删掉一条破坏性构造单测'),
('cp', m_cp_guard_dropped, 'gone',
'单测:控制面配置透传守卫', '删掉跨端守卫测试'),
]
def main():
print('=== pending 超时回收 · 跨文件判据破坏实验 ===')
print('(每条判据都破坏一次,要求自测脚本必须转红)')
print('')
for key, mutate, mode, expect, label in VARIANTS:
print(' · %s' % label)
_variant(key, mutate, mode, expect, label)
print('')
print('=== 结果:%d 通过 / %d 失败 ===' % (OK_N, FAIL_N))
if FAIL_N:
sys.exit(1)
print('PENDING_TIMEOUT_BREAK_OK: 全部破坏实验都被抓到(判据非恒绿)')
sys.exit(0)
if __name__ == '__main__':
main()