#!/usr/bin/env bash # ============================================================================== # scripts/selftest_check_spec_coverage.sh (skill: ai-collab-framework) # check_spec_coverage.sh 的 fixture 回归自测(全内联、零外部依赖)。 # 借鉴上游 OpenSpec 的 parity-test 思路:门禁规则改动后必跑本脚本, # 防止「改了匹配规则却静默漏判/误判」这类漂移(见 skill-audit 2.1/2.6 教训)。 # # 用法:bash scripts/selftest_check_spec_coverage.sh # 退出码:0 = 全部通过 # ============================================================================== set -uo pipefail SELF_DIR="$(cd "$(dirname "$0")" && pwd)" GATE="$SELF_DIR/check_spec_coverage.sh" SKILL_ROOT="$(cd "$SELF_DIR/.." && pwd)" T=$(mktemp -d) PASS=0; FAIL=0 cleanup() { rm -rf "$T"; } trap cleanup EXIT ck() { # ck [env assignments...] local name="$1" want_exit="$2" want_grep="$3" dir="$4"; shift 4 local out rc out=$(env "$@" PROJECT_ROOT="$dir" bash "$GATE" 2>&1); rc=$? local ok=1 [ "$rc" = "$want_exit" ] || ok=0 if [ -n "$want_grep" ]; then printf '%s\n' "$out" | grep -qF "$want_grep" || ok=0; fi if [ "$ok" = "1" ]; then echo "PASS $name"; PASS=$((PASS+1)); else echo "FAIL $name (rc=$rc want=$want_exit; want grep '$want_grep')"; printf '%s\n' "$out" | sed 's/^/ |/'; FAIL=$((FAIL+1)); fi } # ---- 最小工件构造工具(非 git 目录白名单只 WARN 不阻塞,正好复用)---------- mkp() { mkdir -p "$1/openspec/changes/chg_0001_demo" printf -- '---\nchange_id: chg_0001\nskip_coverage: %s\n---\n# Change: t\n' "${2:-false}" \ > "$1/openspec/changes/chg_0001_demo/proposal.md"; } mkt() { printf '# Tasks\n\n- id: T1\n output: design.md\n- id: T2\n output: src/demo/\n' \ > "$1/openspec/changes/chg_0001_demo/tasks.md"; } mkc() { local d="$1"; shift { printf '# Testcases\n' for c in "$@"; do printf '\n### %s\n\n- Scenario: S?\n- Bound Task: T2\n' "$c"; done } > "$d/testcases.md"; } mkmap() { local d="$1" spec="$2" { printf 'scenarios:\n' IFS=';' read -ra rows <<< "$spec" for row in "${rows[@]}"; do printf ' - id: %s\n cases: [%s]\n' "${row%%:*}" "${row#*:}" done printf 'gate:\n blocking_gap_tolerance: 0\n implementation_files:\n - src/demo/\n' } > "$d/testcases.map.yaml"; } mkspec() { printf '# Spec: %s\n%s\n' "$2" "$3" > "$1/spec.md"; } B_WHEN='- **WHEN** a - **THEN** b ' # ---- S1: 模板派生 happy path(真实 id + git + 已提交 in-scope 改动)-------- D=$T/s1; mkdir -p "$D/openspec/changes/chg_0001_demo" cp "$SKILL_ROOT"/templates/proposal.md "$SKILL_ROOT"/templates/spec.md "$SKILL_ROOT"/templates/tasks.md \ "$SKILL_ROOT"/templates/testcases.md "$SKILL_ROOT"/templates/testcases.map.yaml "$D/openspec/changes/chg_0001_demo/" sed -i '' -e 's/C1_/C1_a/g; s/C2_/C2_b/g; s/C3_/C3_c/g; s|src//|src/module/|g; s|tests/unit//|tests/unit/module/|g' "$D/openspec/changes/chg_0001_demo"/* printf '\n### C3_c\n\n- Scenario: S2\n- Bound Task: T2\n' >> "$D/openspec/changes/chg_0001_demo/testcases.md" git -C "$D" init -q -b main && git -C "$D" add -A && git -C "$D" commit -qm init && git -C "$D" checkout -qb change/x mkdir -p "$D/src/module" && echo x > "$D/src/module/a.ts" && git -C "$D" add -A && git -C "$D" commit -qm impl ck "S1 模板派生happy(新capability全ADDED)" 0 "PASS" "$D" # ---- S2: 旧式全量 spec(无 delta 段)→ 放行 + 迁移提示 ---------------------- D=$T/s2; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a C2_b mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## Concrete Scenarios ### S1 add $B_WHEN ### S2 del $B_WHEN" mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a C2_b;S2:C2_b" ck "S2 旧式全量spec放行+note" 0 "全量 spec(旧式)" "$D" # ---- S3: delta 合法(主 spec S1,S2;chg ADDED S3 + MODIFIED S1 + REMOVED S2) D=$T/s3; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a C3_d mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios ### S3 new thing $B_WHEN ## MODIFIED Scenarios ### S1 add - **WHEN** a-modified - **THEN** b-modified ## REMOVED Scenarios ### S2 del - **Reason**: superseded by S3 - **Migration**: use S3 path" mkmap "$D/openspec/changes/chg_0001_demo" "S3:C3_d;S1:C1_a" mkdir -p "$D/openspec/specs/demo-cap" mkspec "$D/openspec/specs/demo-cap" "demo-cap" "## Concrete Scenarios ### S1 add $B_WHEN ### S2 del $B_WHEN" ck "S3 delta合法(三种操作+指向齐全)" 0 "PASS" "$D" # ---- S4: ADDED id 已在主 spec → FAIL ---------------------------------------- D=$T/s4; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios ### S1 add $B_WHEN" mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a" mkdir -p "$D/openspec/specs/demo-cap" mkspec "$D/openspec/specs/demo-cap" "demo-cap" "### S1 add $B_WHEN" ck "S4 ADDED已存在主spec提示改MODIFIED" 1 "应改为 MODIFIED" "$D" # ---- S5: 主 spec 不存在却有 MODIFIED → FAIL --------------------------------- D=$T/s5; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## MODIFIED Scenarios ### S1 add $B_WHEN" mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a" ck "S5 新capability禁MODIFIED(无主spec)" 1 "新 capability 只允许 ADDED" "$D" # ---- S6: REMOVED 缺 Reason/Migration → FAIL --------------------------------- D=$T/s6; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## REMOVED Scenarios ### S1 add " mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a" mkdir -p "$D/openspec/specs/demo-cap" mkspec "$D/openspec/specs/demo-cap" "demo-cap" "### S1 add $B_WHEN" ck "S6 REMOVED缺Reason+Migration" 1 "REMOVED 块需含" "$D" # ---- S7: delta 模式下 S 块散落在三段之外 → FAIL ------------------------------ D=$T/s7; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a C2_b mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios ### S1 add $B_WHEN ## Concrete Scenarios ### S2 stray $B_WHEN" mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a C2_b;S2:C2_b" ck "S7 delta模式散落块FAIL" 1 "散落在 delta 段之外" "$D" # ---- S8: 空壳场景(缺 THEN)→ FAIL ------------------------------------------- D=$T/s8; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios ### S1 add - **WHEN** only-when" mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a" ck "S8 空壳场景缺THEN FAIL" 1 "缺 THEN" "$D" # ---- S9: skip_coverage 豁免(只有 proposal)→ SKIP + 放行 -------------------- D=$T/s9; mkdir -p "$D/openspec/changes/chg_0009_refactor" printf -- '---\nchange_id: chg_0009\nskip_coverage: true\n---\n# Change: pure refactor\n' \ > "$D/openspec/changes/chg_0009_refactor/proposal.md" ck "S9 skip豁免可见不静默" 0 "SKIP" "$D" # ---- S10a: 归档项(日期前缀)被扫描:结构缺陷仍 FAIL ------------------------- D=$T/s10a; A="$D/openspec/changes/archive/2026-09-17-chg_0005_demo"; mkdir -p "$A" printf -- '---\nchange_id: chg_0005\n---\n# t\n' > "$A/proposal.md" printf '# Tasks\n\n- id: T1\n output: x\n' > "$A/tasks.md" printf '# Testcases\n\n### C1_a\n\n- Scenario: S1\n- Bound Task: T1\n' > "$A/testcases.md" printf 'scenarios:\n - id: S1\n cases: [C1_a]\ngate:\n blocking_gap_tolerance: 0\n implementation_files:\n - src/demo/\n' > "$A/testcases.map.yaml" printf '# Spec: demo-cap\n\n## ADDED Scenarios\n\n### S1 add\n\n- **WHEN** only\n' > "$A/spec.md" ck "S10a 日期前缀归档仍被扫(空壳FAIL)" 1 "缺 THEN" "$D" # ---- S10b: 归档项不做指向核对(合并后 ADDED 已在主 spec 仍放行)------------- D=$T/s10b; A="$D/openspec/changes/archive/2026-09-16-chg_0003_demo"; mkdir -p "$A" printf -- '---\nchange_id: chg_0003\n---\n# t\n' > "$A/proposal.md" printf '# Tasks\n\n- id: T2\n output: src/demo/\n' > "$A/tasks.md" printf '# Testcases\n\n### C1_a\n\n- Scenario: S1\n- Bound Task: T2\n' > "$A/testcases.md" printf 'scenarios:\n - id: S1\n cases: [C1_a]\ngate:\n blocking_gap_tolerance: 0\n implementation_files:\n - src/demo/\n' > "$A/testcases.map.yaml" printf '# Spec: demo-cap\n\n## ADDED Scenarios\n\n### S1 add\n%s' "$B_WHEN" > "$A/spec.md" mkdir -p "$D/openspec/specs/demo-cap" printf '# Spec: demo-cap\n\n### S1 add\n%s' "$B_WHEN" > "$D/openspec/specs/demo-cap/spec.md" ck "S10b 归档项豁免指向核对" 0 "PASS" "$D" # ---- S11: create_change.sh 编号不复用归档号 ---------------------------------- D=$T/s11; mkdir -p "$D/openspec/changes/archive/2026-09-17-chg_0009_old" OUT=$(PROJECT_ROOT="$D" bash "$SELF_DIR/create_change.sh" new_thing 2>&1) if printf '%s\n' "$OUT" | grep -q 'chg_0010_new_thing'; then echo "PASS S11 编号连archive扫(不复用chg_0009)"; PASS=$((PASS+1)) else echo "FAIL S11 期望 chg_0010,实际:"; printf '%s\n' "$OUT" | sed 's/^/ |/'; FAIL=$((FAIL+1)) fi # ---- S12: a1 monorepo 路径白名单真实生效(backend/src/foo/)------------------ # 旧实现 'git diff -- src/' 在 monorepo 下静默失效;新实现以 entries 为 pathspec。 D=$T/s12; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios ### S1 add $B_WHEN" # 白名单指 backend/src/foo/,仓库根无 src/;应通过路径 pathspec 真实生效 + 提示 note printf 'scenarios:\n - id: S1\n cases: [C1_a]\ngate:\n blocking_gap_tolerance: 0\n implementation_files:\n - backend/src/foo/\n' > "$D/openspec/changes/chg_0001_demo/testcases.map.yaml" git -C "$D" init -q -b main && git -C "$D" add -A && git -C "$D" commit -qm init mkdir -p "$D/backend/src/foo" && echo x > "$D/backend/src/foo/a.ts" git -C "$D" add -A && git -C "$D" commit -qm impl # 期望:通过(pathspec 真生效,无 out-of-scope WARN)。 # note 只在写歪成 src/ 但根无 src/ 目录时触发;本用例白名单已是 backend/src/foo/,无 note。 # 关键反向断言:out-of-scope WARN 不应出现(旧实现下会静默失效,新实现下能精准放行)。 ck_neg() { # ck_neg local name="$1" want_exit="$2" not_grep="$3" dir="$4" local out rc; out=$(PROJECT_ROOT="$dir" bash "$GATE" 2>&1); rc=$? if [ "$rc" = "$want_exit" ] && ! printf '%s\n' "$out" | grep -qF "$not_grep"; then echo "PASS $name"; PASS=$((PASS+1)) else echo "FAIL $name (rc=$rc want=$want_exit; should NOT contain '$not_grep')"; printf '%s\n' "$out" | sed 's/^/ |/'; FAIL=$((FAIL+1)) fi } ck_neg "S12 monorepo路径白名单真实生效" 0 "out-of-scope" "$D" # ---- S13: a4 map.yaml '# spec:' 注释 capability 不一致 → FAIL ----------------- D=$T/s13; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios ### S1 add $B_WHEN" # 注释里 capability 写歪(demo-cap 与 spec.md 一致,但这里故意写 typo)—— 应 FAIL printf '# spec: wrong-cap@chg_0001\nscenarios:\n - id: S1\n cases: [C1_a]\ngate:\n blocking_gap_tolerance: 0\n implementation_files:\n - src/demo/\n' > "$D/openspec/changes/chg_0001_demo/testcases.map.yaml" ck "S13 map.yaml capability注释不一致FAIL" 1 "capability 不一致" "$D" # ---- S14: --json 模式 PASS(结构合法 + 无文本漂移)-------------------------- # ck_json # - 跑 gate --json,断言退码 + jq 表达式 + stdout 不含指定文本(防 echo 守卫漏改) ck_json() { local name="$1" want_exit="$2" jq_expr="$3" not_text="$4" dir="$5" local out rc out=$(PROJECT_ROOT="$dir" bash "$GATE" --json 2>&1); rc=$? local ok=1 [ "$rc" = "$want_exit" ] || ok=0 if [ -n "$not_text" ]; then printf '%s\n' "$out" | grep -qF "$not_text" && ok=0 fi # JSON 合法性 + jq 表达式断言(jq 失败 → ok=0) if ! printf '%s\n' "$out" | jq -e "$jq_expr" >/dev/null 2>&1; then ok=0; fi if [ "$ok" = "1" ]; then echo "PASS $name"; PASS=$((PASS+1)); else echo "FAIL $name (rc=$rc want=$want_exit; jq='$jq_expr'; not_grep='$not_text')"; printf '%s\n' "$out" | sed 's/^/ |/'; FAIL=$((FAIL+1)); fi } # 复用 S3 的 delta 合法 fixture(ADDED + MODIFIED + REMOVED + 主 spec 对齐) D=$T/s14; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a C3_d mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios ### S3 new thing $B_WHEN ## MODIFIED Scenarios ### S1 add - **WHEN** a-modified - **THEN** b-modified ## REMOVED Scenarios ### S2 del - **Reason**: superseded by S3 - **Migration**: use S3 path" mkmap "$D/openspec/changes/chg_0001_demo" "S3:C3_d;S1:C1_a" mkdir -p "$D/openspec/specs/demo-cap" mkspec "$D/openspec/specs/demo-cap" "demo-cap" "## Concrete Scenarios ### S1 add $B_WHEN ### S2 del $B_WHEN" # PASS 路径:jq 断言 (status=PASS, summary.pass>=1, 无 status envelope, 无文本漂移) # 注意 jq `and` 左结合后右侧表达式以左侧 input 为上下文;用括号分组避免 .summary # 丢失(jq -e 退码 1 = false → ck_json 报 FAIL)。 ck_json "S14 --json PASS退0+合法+无文本漂移" 0 \ '(.summary.fail == 0) and ((.changes | length) >= 1) and (any(.changes[]; .status == "PASS")) and ((.status // null) == null)' \ "==[ PASS ]==" "$D" # ---- S15: --json 模式 FAIL(信封含期望 code + 文本抑制)--------------------- # 复用 S5 fixture(主 spec 不存在却有 MODIFIED → DELTA_BUT_NO_MAIN_SPEC) D=$T/s15; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## MODIFIED Scenarios ### S1 add $B_WHEN" mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a" # FAIL 路径:jq 断言 (status envelope 含期望 code, summary.fail>=1, 无 ==[ FAIL ]== 文本) ck_json "S15 --json FAIL退1+信封含code+文本抑制" 1 \ '(.status | type == "array") and (.status | any(.[]; .code == "DELTA_BUT_NO_MAIN_SPEC")) and (.summary.fail >= 1)' \ "==[ FAIL ]==" "$D" echo "----" echo "PASS=$PASS FAIL=$FAIL" [ "$FAIL" = "0" ]