Files
dth/openspec/scripts/selftest_check_spec_coverage.sh
T
xstudio agent c1cf6f54e5 merge sandbox/ai-1790562657-0EF743: chg_0001 dth scaffold + chg_0002 experiment_platform MVP
- chg_0001_dth_scaffold (skip_coverage): openspec framework + NestJS+React+PGlite scaffold
- chg_0002_sp1_experiment_platform_mvp: 6 scenarios S1-S6 + 21 perm-smoke + 16 e2e + 3 demo operators (hello-mpc/pir/vertical-lr)

Sandbox task exit 0. Independent verification: check_spec_coverage.sh gap=0 PASS.
Co-Authored-By: Claude Opus 4.8 <[email protected]>
2026-09-28 10:54:34 +08:00

302 lines
14 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
# ==============================================================================
# scripts/selftest_check_spec_coverage.sh (skill: ai-collab-framework)
# check_spec_coverage.sh 的 fixture 回归自测(全内联、零外部依赖)。
# 借鉴上游 OpenSpec 的 parity-test 思路:门禁规则改动后必跑本脚本,
# 防止「改了匹配规则却静默漏判/误判」这类漂移(见 skill-audit 2.1/2.6 教训)。
#
# 用法:bash scripts/selftest_check_spec_coverage.sh
# 退出码:0 = 全部通过
# ==============================================================================
set -uo pipefail
SELF_DIR="$(cd "$(dirname "$0")" && pwd)"
GATE="$SELF_DIR/check_spec_coverage.sh"
SKILL_ROOT="$(cd "$SELF_DIR/.." && pwd)"
T=$(mktemp -d)
PASS=0; FAIL=0
cleanup() { rm -rf "$T"; }
trap cleanup EXIT
ck() { # ck <name> <want_exit> <want_grep> <dir> [env assignments...]
local name="$1" want_exit="$2" want_grep="$3" dir="$4"; shift 4
local out rc
out=$(env "$@" PROJECT_ROOT="$dir" bash "$GATE" 2>&1); rc=$?
local ok=1
[ "$rc" = "$want_exit" ] || ok=0
if [ -n "$want_grep" ]; then printf '%s\n' "$out" | grep -qF "$want_grep" || ok=0; fi
if [ "$ok" = "1" ]; then echo "PASS $name"; PASS=$((PASS+1));
else echo "FAIL $name (rc=$rc want=$want_exit; want grep '$want_grep')"; printf '%s\n' "$out" | sed 's/^/ |/'; FAIL=$((FAIL+1)); fi
}
# ---- 最小工件构造工具(非 git 目录白名单只 WARN 不阻塞,正好复用)----------
mkp() { mkdir -p "$1/openspec/changes/chg_0001_demo"
printf -- '---\nchange_id: chg_0001\nskip_coverage: %s\n---\n# Change: t\n' "${2:-false}" \
> "$1/openspec/changes/chg_0001_demo/proposal.md"; }
mkt() { printf '# Tasks\n\n- id: T1\n output: design.md\n- id: T2\n output: src/demo/\n' \
> "$1/openspec/changes/chg_0001_demo/tasks.md"; }
mkc() { local d="$1"; shift
{ printf '# Testcases\n'
for c in "$@"; do printf '\n### %s\n\n- Scenario: S?\n- Bound Task: T2\n' "$c"; done
} > "$d/testcases.md"; }
mkmap() { local d="$1" spec="$2"
{ printf 'scenarios:\n'
IFS=';' read -ra rows <<< "$spec"
for row in "${rows[@]}"; do
printf ' - id: %s\n cases: [%s]\n' "${row%%:*}" "${row#*:}"
done
printf 'gate:\n blocking_gap_tolerance: 0\n implementation_files:\n - src/demo/\n'
} > "$d/testcases.map.yaml"; }
mkspec() { printf '# Spec: %s\n%s\n' "$2" "$3" > "$1/spec.md"; }
B_WHEN='- **WHEN** a
- **THEN** b
'
# ---- S1: 模板派生 happy path(真实 id + git + 已提交 in-scope 改动)--------
D=$T/s1; mkdir -p "$D/openspec/changes/chg_0001_demo"
cp "$SKILL_ROOT"/templates/proposal.md "$SKILL_ROOT"/templates/spec.md "$SKILL_ROOT"/templates/tasks.md \
"$SKILL_ROOT"/templates/testcases.md "$SKILL_ROOT"/templates/testcases.map.yaml "$D/openspec/changes/chg_0001_demo/"
sed -i '' -e 's/C1_<slug>/C1_a/g; s/C2_<slug>/C2_b/g; s/C3_<slug>/C3_c/g; s|src/<module>/|src/module/|g; s|tests/unit/<module>/|tests/unit/module/|g' "$D/openspec/changes/chg_0001_demo"/*
printf '\n### C3_c\n\n- Scenario: S2\n- Bound Task: T2\n' >> "$D/openspec/changes/chg_0001_demo/testcases.md"
git -C "$D" init -q -b main && git -C "$D" add -A && git -C "$D" commit -qm init && git -C "$D" checkout -qb change/x
mkdir -p "$D/src/module" && echo x > "$D/src/module/a.ts" && git -C "$D" add -A && git -C "$D" commit -qm impl
ck "S1 模板派生happy(新capability全ADDED)" 0 "PASS" "$D"
# ---- S2: 旧式全量 spec(无 delta 段)→ 放行 + 迁移提示 ----------------------
D=$T/s2; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a C2_b
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## Concrete Scenarios
### S1 add
$B_WHEN
### S2 del
$B_WHEN"
mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a C2_b;S2:C2_b"
ck "S2 旧式全量spec放行+note" 0 "全量 spec(旧式)" "$D"
# ---- S3: delta 合法(主 spec S1,S2;chg ADDED S3 + MODIFIED S1 + REMOVED S2)
D=$T/s3; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a C3_d
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios
### S3 new thing
$B_WHEN
## MODIFIED Scenarios
### S1 add
- **WHEN** a-modified
- **THEN** b-modified
## REMOVED Scenarios
### S2 del
- **Reason**: superseded by S3
- **Migration**: use S3 path"
mkmap "$D/openspec/changes/chg_0001_demo" "S3:C3_d;S1:C1_a"
mkdir -p "$D/openspec/specs/demo-cap"
mkspec "$D/openspec/specs/demo-cap" "demo-cap" "## Concrete Scenarios
### S1 add
$B_WHEN
### S2 del
$B_WHEN"
ck "S3 delta合法(三种操作+指向齐全)" 0 "PASS" "$D"
# ---- S4: ADDED id 已在主 spec → FAIL ----------------------------------------
D=$T/s4; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios
### S1 add
$B_WHEN"
mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a"
mkdir -p "$D/openspec/specs/demo-cap"
mkspec "$D/openspec/specs/demo-cap" "demo-cap" "### S1 add
$B_WHEN"
ck "S4 ADDED已存在主spec提示改MODIFIED" 1 "应改为 MODIFIED" "$D"
# ---- S5: 主 spec 不存在却有 MODIFIED → FAIL ---------------------------------
D=$T/s5; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## MODIFIED Scenarios
### S1 add
$B_WHEN"
mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a"
ck "S5 新capability禁MODIFIED(无主spec)" 1 "新 capability 只允许 ADDED" "$D"
# ---- S6: REMOVED 缺 Reason/Migration → FAIL ---------------------------------
D=$T/s6; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## REMOVED Scenarios
### S1 add
"
mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a"
mkdir -p "$D/openspec/specs/demo-cap"
mkspec "$D/openspec/specs/demo-cap" "demo-cap" "### S1 add
$B_WHEN"
ck "S6 REMOVED缺Reason+Migration" 1 "REMOVED 块需含" "$D"
# ---- S7: delta 模式下 S 块散落在三段之外 → FAIL ------------------------------
D=$T/s7; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a C2_b
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios
### S1 add
$B_WHEN
## Concrete Scenarios
### S2 stray
$B_WHEN"
mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a C2_b;S2:C2_b"
ck "S7 delta模式散落块FAIL" 1 "散落在 delta 段之外" "$D"
# ---- S8: 空壳场景(缺 THEN)→ FAIL -------------------------------------------
D=$T/s8; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios
### S1 add
- **WHEN** only-when"
mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a"
ck "S8 空壳场景缺THEN FAIL" 1 "缺 THEN" "$D"
# ---- S9: skip_coverage 豁免(只有 proposal)→ SKIP + 放行 --------------------
D=$T/s9; mkdir -p "$D/openspec/changes/chg_0009_refactor"
printf -- '---\nchange_id: chg_0009\nskip_coverage: true\n---\n# Change: pure refactor\n' \
> "$D/openspec/changes/chg_0009_refactor/proposal.md"
ck "S9 skip豁免可见不静默" 0 "SKIP" "$D"
# ---- S10a: 归档项(日期前缀)被扫描:结构缺陷仍 FAIL -------------------------
D=$T/s10a; A="$D/openspec/changes/archive/2026-09-17-chg_0005_demo"; mkdir -p "$A"
printf -- '---\nchange_id: chg_0005\n---\n# t\n' > "$A/proposal.md"
printf '# Tasks\n\n- id: T1\n output: x\n' > "$A/tasks.md"
printf '# Testcases\n\n### C1_a\n\n- Scenario: S1\n- Bound Task: T1\n' > "$A/testcases.md"
printf 'scenarios:\n - id: S1\n cases: [C1_a]\ngate:\n blocking_gap_tolerance: 0\n implementation_files:\n - src/demo/\n' > "$A/testcases.map.yaml"
printf '# Spec: demo-cap\n\n## ADDED Scenarios\n\n### S1 add\n\n- **WHEN** only\n' > "$A/spec.md"
ck "S10a 日期前缀归档仍被扫(空壳FAIL)" 1 "缺 THEN" "$D"
# ---- S10b: 归档项不做指向核对(合并后 ADDED 已在主 spec 仍放行)-------------
D=$T/s10b; A="$D/openspec/changes/archive/2026-09-16-chg_0003_demo"; mkdir -p "$A"
printf -- '---\nchange_id: chg_0003\n---\n# t\n' > "$A/proposal.md"
printf '# Tasks\n\n- id: T2\n output: src/demo/\n' > "$A/tasks.md"
printf '# Testcases\n\n### C1_a\n\n- Scenario: S1\n- Bound Task: T2\n' > "$A/testcases.md"
printf 'scenarios:\n - id: S1\n cases: [C1_a]\ngate:\n blocking_gap_tolerance: 0\n implementation_files:\n - src/demo/\n' > "$A/testcases.map.yaml"
printf '# Spec: demo-cap\n\n## ADDED Scenarios\n\n### S1 add\n%s' "$B_WHEN" > "$A/spec.md"
mkdir -p "$D/openspec/specs/demo-cap"
printf '# Spec: demo-cap\n\n### S1 add\n%s' "$B_WHEN" > "$D/openspec/specs/demo-cap/spec.md"
ck "S10b 归档项豁免指向核对" 0 "PASS" "$D"
# ---- S11: create_change.sh 编号不复用归档号 ----------------------------------
D=$T/s11; mkdir -p "$D/openspec/changes/archive/2026-09-17-chg_0009_old"
OUT=$(PROJECT_ROOT="$D" bash "$SELF_DIR/create_change.sh" new_thing 2>&1)
if printf '%s\n' "$OUT" | grep -q 'chg_0010_new_thing'; then
echo "PASS S11 编号连archive扫(不复用chg_0009)"; PASS=$((PASS+1))
else
echo "FAIL S11 期望 chg_0010,实际:"; printf '%s\n' "$OUT" | sed 's/^/ |/'; FAIL=$((FAIL+1))
fi
# ---- S12: a1 monorepo 路径白名单真实生效(backend/src/foo/)------------------
# 旧实现 'git diff -- src/' 在 monorepo 下静默失效;新实现以 entries 为 pathspec。
D=$T/s12; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios
### S1 add
$B_WHEN"
# 白名单指 backend/src/foo/,仓库根无 src/;应通过路径 pathspec 真实生效 + 提示 note
printf 'scenarios:\n - id: S1\n cases: [C1_a]\ngate:\n blocking_gap_tolerance: 0\n implementation_files:\n - backend/src/foo/\n' > "$D/openspec/changes/chg_0001_demo/testcases.map.yaml"
git -C "$D" init -q -b main && git -C "$D" add -A && git -C "$D" commit -qm init
mkdir -p "$D/backend/src/foo" && echo x > "$D/backend/src/foo/a.ts"
git -C "$D" add -A && git -C "$D" commit -qm impl
# 期望:通过(pathspec 真生效,无 out-of-scope WARN)。
# note 只在写歪成 src/ 但根无 src/ 目录时触发;本用例白名单已是 backend/src/foo/,无 note。
# 关键反向断言:out-of-scope WARN 不应出现(旧实现下会静默失效,新实现下能精准放行)。
ck_neg() { # ck_neg <name> <want_exit> <not_grep> <dir>
local name="$1" want_exit="$2" not_grep="$3" dir="$4"
local out rc; out=$(PROJECT_ROOT="$dir" bash "$GATE" 2>&1); rc=$?
if [ "$rc" = "$want_exit" ] && ! printf '%s\n' "$out" | grep -qF "$not_grep"; then
echo "PASS $name"; PASS=$((PASS+1))
else
echo "FAIL $name (rc=$rc want=$want_exit; should NOT contain '$not_grep')"; printf '%s\n' "$out" | sed 's/^/ |/'; FAIL=$((FAIL+1))
fi
}
ck_neg "S12 monorepo路径白名单真实生效" 0 "out-of-scope" "$D"
# ---- S13: a4 map.yaml '# spec:' 注释 capability 不一致 → FAIL -----------------
D=$T/s13; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios
### S1 add
$B_WHEN"
# 注释里 capability 写歪(demo-cap 与 spec.md 一致,但这里故意写 typo)—— 应 FAIL
printf '# spec: wrong-cap@chg_0001\nscenarios:\n - id: S1\n cases: [C1_a]\ngate:\n blocking_gap_tolerance: 0\n implementation_files:\n - src/demo/\n' > "$D/openspec/changes/chg_0001_demo/testcases.map.yaml"
ck "S13 map.yaml capability注释不一致FAIL" 1 "capability 不一致" "$D"
# ---- S14: --json 模式 PASS(结构合法 + 无文本漂移)--------------------------
# ck_json <name> <want_exit> <jq_assertion> <not_grep_text> <dir>
# - 跑 gate --json,断言退码 + jq 表达式 + stdout 不含指定文本(防 echo 守卫漏改)
ck_json() {
local name="$1" want_exit="$2" jq_expr="$3" not_text="$4" dir="$5"
local out rc
out=$(PROJECT_ROOT="$dir" bash "$GATE" --json 2>&1); rc=$?
local ok=1
[ "$rc" = "$want_exit" ] || ok=0
if [ -n "$not_text" ]; then
printf '%s\n' "$out" | grep -qF "$not_text" && ok=0
fi
# JSON 合法性 + jq 表达式断言(jq 失败 → ok=0)
if ! printf '%s\n' "$out" | jq -e "$jq_expr" >/dev/null 2>&1; then ok=0; fi
if [ "$ok" = "1" ]; then echo "PASS $name"; PASS=$((PASS+1));
else echo "FAIL $name (rc=$rc want=$want_exit; jq='$jq_expr'; not_grep='$not_text')"; printf '%s\n' "$out" | sed 's/^/ |/'; FAIL=$((FAIL+1)); fi
}
# 复用 S3 的 delta 合法 fixture(ADDED + MODIFIED + REMOVED + 主 spec 对齐)
D=$T/s14; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a C3_d
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios
### S3 new thing
$B_WHEN
## MODIFIED Scenarios
### S1 add
- **WHEN** a-modified
- **THEN** b-modified
## REMOVED Scenarios
### S2 del
- **Reason**: superseded by S3
- **Migration**: use S3 path"
mkmap "$D/openspec/changes/chg_0001_demo" "S3:C3_d;S1:C1_a"
mkdir -p "$D/openspec/specs/demo-cap"
mkspec "$D/openspec/specs/demo-cap" "demo-cap" "## Concrete Scenarios
### S1 add
$B_WHEN
### S2 del
$B_WHEN"
# PASS 路径:jq 断言 (status=PASS, summary.pass>=1, 无 status envelope, 无文本漂移)
# 注意 jq `and` 左结合后右侧表达式以左侧 input 为上下文;用括号分组避免 .summary
# 丢失(jq -e 退码 1 = false → ck_json 报 FAIL)。
ck_json "S14 --json PASS退0+合法+无文本漂移" 0 \
'(.summary.fail == 0) and ((.changes | length) >= 1) and (any(.changes[]; .status == "PASS")) and ((.status // null) == null)' \
"==[ PASS ]==" "$D"
# ---- S15: --json 模式 FAIL(信封含期望 code + 文本抑制)---------------------
# 复用 S5 fixture(主 spec 不存在却有 MODIFIED → DELTA_BUT_NO_MAIN_SPEC)
D=$T/s15; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## MODIFIED Scenarios
### S1 add
$B_WHEN"
mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a"
# FAIL 路径:jq 断言 (status envelope 含期望 code, summary.fail>=1, 无 ==[ FAIL ]== 文本)
ck_json "S15 --json FAIL退1+信封含code+文本抑制" 1 \
'(.status | type == "array") and (.status | any(.[]; .code == "DELTA_BUT_NO_MAIN_SPEC")) and (.summary.fail >= 1)' \
"==[ FAIL ]==" "$D"
echo "----"
echo "PASS=$PASS FAIL=$FAIL"
[ "$FAIL" = "0" ]