- chg_0001_dth_scaffold (skip_coverage): openspec framework + NestJS+React+PGlite scaffold - chg_0002_sp1_experiment_platform_mvp: 6 scenarios S1-S6 + 21 perm-smoke + 16 e2e + 3 demo operators (hello-mpc/pir/vertical-lr) Sandbox task exit 0. Independent verification: check_spec_coverage.sh gap=0 PASS. Co-Authored-By: Claude Opus 4.8 <[email protected]>
302 lines
14 KiB
Bash
Executable File
302 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# ==============================================================================
|
||
# scripts/selftest_check_spec_coverage.sh (skill: ai-collab-framework)
|
||
# check_spec_coverage.sh 的 fixture 回归自测(全内联、零外部依赖)。
|
||
# 借鉴上游 OpenSpec 的 parity-test 思路:门禁规则改动后必跑本脚本,
|
||
# 防止「改了匹配规则却静默漏判/误判」这类漂移(见 skill-audit 2.1/2.6 教训)。
|
||
#
|
||
# 用法:bash scripts/selftest_check_spec_coverage.sh
|
||
# 退出码:0 = 全部通过
|
||
# ==============================================================================
|
||
set -uo pipefail
|
||
|
||
SELF_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||
GATE="$SELF_DIR/check_spec_coverage.sh"
|
||
SKILL_ROOT="$(cd "$SELF_DIR/.." && pwd)"
|
||
T=$(mktemp -d)
|
||
PASS=0; FAIL=0
|
||
|
||
cleanup() { rm -rf "$T"; }
|
||
trap cleanup EXIT
|
||
|
||
ck() { # ck <name> <want_exit> <want_grep> <dir> [env assignments...]
|
||
local name="$1" want_exit="$2" want_grep="$3" dir="$4"; shift 4
|
||
local out rc
|
||
out=$(env "$@" PROJECT_ROOT="$dir" bash "$GATE" 2>&1); rc=$?
|
||
local ok=1
|
||
[ "$rc" = "$want_exit" ] || ok=0
|
||
if [ -n "$want_grep" ]; then printf '%s\n' "$out" | grep -qF "$want_grep" || ok=0; fi
|
||
if [ "$ok" = "1" ]; then echo "PASS $name"; PASS=$((PASS+1));
|
||
else echo "FAIL $name (rc=$rc want=$want_exit; want grep '$want_grep')"; printf '%s\n' "$out" | sed 's/^/ |/'; FAIL=$((FAIL+1)); fi
|
||
}
|
||
|
||
# ---- 最小工件构造工具(非 git 目录白名单只 WARN 不阻塞,正好复用)----------
|
||
mkp() { mkdir -p "$1/openspec/changes/chg_0001_demo"
|
||
printf -- '---\nchange_id: chg_0001\nskip_coverage: %s\n---\n# Change: t\n' "${2:-false}" \
|
||
> "$1/openspec/changes/chg_0001_demo/proposal.md"; }
|
||
mkt() { printf '# Tasks\n\n- id: T1\n output: design.md\n- id: T2\n output: src/demo/\n' \
|
||
> "$1/openspec/changes/chg_0001_demo/tasks.md"; }
|
||
mkc() { local d="$1"; shift
|
||
{ printf '# Testcases\n'
|
||
for c in "$@"; do printf '\n### %s\n\n- Scenario: S?\n- Bound Task: T2\n' "$c"; done
|
||
} > "$d/testcases.md"; }
|
||
mkmap() { local d="$1" spec="$2"
|
||
{ printf 'scenarios:\n'
|
||
IFS=';' read -ra rows <<< "$spec"
|
||
for row in "${rows[@]}"; do
|
||
printf ' - id: %s\n cases: [%s]\n' "${row%%:*}" "${row#*:}"
|
||
done
|
||
printf 'gate:\n blocking_gap_tolerance: 0\n implementation_files:\n - src/demo/\n'
|
||
} > "$d/testcases.map.yaml"; }
|
||
mkspec() { printf '# Spec: %s\n%s\n' "$2" "$3" > "$1/spec.md"; }
|
||
B_WHEN='- **WHEN** a
|
||
- **THEN** b
|
||
'
|
||
|
||
# ---- S1: 模板派生 happy path(真实 id + git + 已提交 in-scope 改动)--------
|
||
D=$T/s1; mkdir -p "$D/openspec/changes/chg_0001_demo"
|
||
cp "$SKILL_ROOT"/templates/proposal.md "$SKILL_ROOT"/templates/spec.md "$SKILL_ROOT"/templates/tasks.md \
|
||
"$SKILL_ROOT"/templates/testcases.md "$SKILL_ROOT"/templates/testcases.map.yaml "$D/openspec/changes/chg_0001_demo/"
|
||
sed -i '' -e 's/C1_<slug>/C1_a/g; s/C2_<slug>/C2_b/g; s/C3_<slug>/C3_c/g; s|src/<module>/|src/module/|g; s|tests/unit/<module>/|tests/unit/module/|g' "$D/openspec/changes/chg_0001_demo"/*
|
||
printf '\n### C3_c\n\n- Scenario: S2\n- Bound Task: T2\n' >> "$D/openspec/changes/chg_0001_demo/testcases.md"
|
||
git -C "$D" init -q -b main && git -C "$D" add -A && git -C "$D" commit -qm init && git -C "$D" checkout -qb change/x
|
||
mkdir -p "$D/src/module" && echo x > "$D/src/module/a.ts" && git -C "$D" add -A && git -C "$D" commit -qm impl
|
||
ck "S1 模板派生happy(新capability全ADDED)" 0 "PASS" "$D"
|
||
|
||
# ---- S2: 旧式全量 spec(无 delta 段)→ 放行 + 迁移提示 ----------------------
|
||
D=$T/s2; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a C2_b
|
||
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## Concrete Scenarios
|
||
|
||
### S1 add
|
||
$B_WHEN
|
||
### S2 del
|
||
$B_WHEN"
|
||
mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a C2_b;S2:C2_b"
|
||
ck "S2 旧式全量spec放行+note" 0 "全量 spec(旧式)" "$D"
|
||
|
||
# ---- S3: delta 合法(主 spec S1,S2;chg ADDED S3 + MODIFIED S1 + REMOVED S2)
|
||
D=$T/s3; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a C3_d
|
||
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios
|
||
|
||
### S3 new thing
|
||
$B_WHEN
|
||
|
||
## MODIFIED Scenarios
|
||
|
||
### S1 add
|
||
- **WHEN** a-modified
|
||
- **THEN** b-modified
|
||
|
||
## REMOVED Scenarios
|
||
|
||
### S2 del
|
||
|
||
- **Reason**: superseded by S3
|
||
- **Migration**: use S3 path"
|
||
mkmap "$D/openspec/changes/chg_0001_demo" "S3:C3_d;S1:C1_a"
|
||
mkdir -p "$D/openspec/specs/demo-cap"
|
||
mkspec "$D/openspec/specs/demo-cap" "demo-cap" "## Concrete Scenarios
|
||
|
||
### S1 add
|
||
$B_WHEN
|
||
### S2 del
|
||
$B_WHEN"
|
||
ck "S3 delta合法(三种操作+指向齐全)" 0 "PASS" "$D"
|
||
|
||
# ---- S4: ADDED id 已在主 spec → FAIL ----------------------------------------
|
||
D=$T/s4; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a
|
||
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios
|
||
|
||
### S1 add
|
||
$B_WHEN"
|
||
mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a"
|
||
mkdir -p "$D/openspec/specs/demo-cap"
|
||
mkspec "$D/openspec/specs/demo-cap" "demo-cap" "### S1 add
|
||
$B_WHEN"
|
||
ck "S4 ADDED已存在主spec提示改MODIFIED" 1 "应改为 MODIFIED" "$D"
|
||
|
||
# ---- S5: 主 spec 不存在却有 MODIFIED → FAIL ---------------------------------
|
||
D=$T/s5; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a
|
||
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## MODIFIED Scenarios
|
||
|
||
### S1 add
|
||
$B_WHEN"
|
||
mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a"
|
||
ck "S5 新capability禁MODIFIED(无主spec)" 1 "新 capability 只允许 ADDED" "$D"
|
||
|
||
# ---- S6: REMOVED 缺 Reason/Migration → FAIL ---------------------------------
|
||
D=$T/s6; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a
|
||
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## REMOVED Scenarios
|
||
|
||
### S1 add
|
||
"
|
||
mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a"
|
||
mkdir -p "$D/openspec/specs/demo-cap"
|
||
mkspec "$D/openspec/specs/demo-cap" "demo-cap" "### S1 add
|
||
$B_WHEN"
|
||
ck "S6 REMOVED缺Reason+Migration" 1 "REMOVED 块需含" "$D"
|
||
|
||
# ---- S7: delta 模式下 S 块散落在三段之外 → FAIL ------------------------------
|
||
D=$T/s7; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a C2_b
|
||
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios
|
||
|
||
### S1 add
|
||
$B_WHEN
|
||
|
||
## Concrete Scenarios
|
||
|
||
### S2 stray
|
||
$B_WHEN"
|
||
mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a C2_b;S2:C2_b"
|
||
ck "S7 delta模式散落块FAIL" 1 "散落在 delta 段之外" "$D"
|
||
|
||
# ---- S8: 空壳场景(缺 THEN)→ FAIL -------------------------------------------
|
||
D=$T/s8; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a
|
||
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios
|
||
|
||
### S1 add
|
||
|
||
- **WHEN** only-when"
|
||
mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a"
|
||
ck "S8 空壳场景缺THEN FAIL" 1 "缺 THEN" "$D"
|
||
|
||
# ---- S9: skip_coverage 豁免(只有 proposal)→ SKIP + 放行 --------------------
|
||
D=$T/s9; mkdir -p "$D/openspec/changes/chg_0009_refactor"
|
||
printf -- '---\nchange_id: chg_0009\nskip_coverage: true\n---\n# Change: pure refactor\n' \
|
||
> "$D/openspec/changes/chg_0009_refactor/proposal.md"
|
||
ck "S9 skip豁免可见不静默" 0 "SKIP" "$D"
|
||
|
||
# ---- S10a: 归档项(日期前缀)被扫描:结构缺陷仍 FAIL -------------------------
|
||
D=$T/s10a; A="$D/openspec/changes/archive/2026-09-17-chg_0005_demo"; mkdir -p "$A"
|
||
printf -- '---\nchange_id: chg_0005\n---\n# t\n' > "$A/proposal.md"
|
||
printf '# Tasks\n\n- id: T1\n output: x\n' > "$A/tasks.md"
|
||
printf '# Testcases\n\n### C1_a\n\n- Scenario: S1\n- Bound Task: T1\n' > "$A/testcases.md"
|
||
printf 'scenarios:\n - id: S1\n cases: [C1_a]\ngate:\n blocking_gap_tolerance: 0\n implementation_files:\n - src/demo/\n' > "$A/testcases.map.yaml"
|
||
printf '# Spec: demo-cap\n\n## ADDED Scenarios\n\n### S1 add\n\n- **WHEN** only\n' > "$A/spec.md"
|
||
ck "S10a 日期前缀归档仍被扫(空壳FAIL)" 1 "缺 THEN" "$D"
|
||
|
||
# ---- S10b: 归档项不做指向核对(合并后 ADDED 已在主 spec 仍放行)-------------
|
||
D=$T/s10b; A="$D/openspec/changes/archive/2026-09-16-chg_0003_demo"; mkdir -p "$A"
|
||
printf -- '---\nchange_id: chg_0003\n---\n# t\n' > "$A/proposal.md"
|
||
printf '# Tasks\n\n- id: T2\n output: src/demo/\n' > "$A/tasks.md"
|
||
printf '# Testcases\n\n### C1_a\n\n- Scenario: S1\n- Bound Task: T2\n' > "$A/testcases.md"
|
||
printf 'scenarios:\n - id: S1\n cases: [C1_a]\ngate:\n blocking_gap_tolerance: 0\n implementation_files:\n - src/demo/\n' > "$A/testcases.map.yaml"
|
||
printf '# Spec: demo-cap\n\n## ADDED Scenarios\n\n### S1 add\n%s' "$B_WHEN" > "$A/spec.md"
|
||
mkdir -p "$D/openspec/specs/demo-cap"
|
||
printf '# Spec: demo-cap\n\n### S1 add\n%s' "$B_WHEN" > "$D/openspec/specs/demo-cap/spec.md"
|
||
ck "S10b 归档项豁免指向核对" 0 "PASS" "$D"
|
||
|
||
# ---- S11: create_change.sh 编号不复用归档号 ----------------------------------
|
||
D=$T/s11; mkdir -p "$D/openspec/changes/archive/2026-09-17-chg_0009_old"
|
||
OUT=$(PROJECT_ROOT="$D" bash "$SELF_DIR/create_change.sh" new_thing 2>&1)
|
||
if printf '%s\n' "$OUT" | grep -q 'chg_0010_new_thing'; then
|
||
echo "PASS S11 编号连archive扫(不复用chg_0009)"; PASS=$((PASS+1))
|
||
else
|
||
echo "FAIL S11 期望 chg_0010,实际:"; printf '%s\n' "$OUT" | sed 's/^/ |/'; FAIL=$((FAIL+1))
|
||
fi
|
||
|
||
# ---- S12: a1 monorepo 路径白名单真实生效(backend/src/foo/)------------------
|
||
# 旧实现 'git diff -- src/' 在 monorepo 下静默失效;新实现以 entries 为 pathspec。
|
||
D=$T/s12; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a
|
||
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios
|
||
|
||
### S1 add
|
||
$B_WHEN"
|
||
# 白名单指 backend/src/foo/,仓库根无 src/;应通过路径 pathspec 真实生效 + 提示 note
|
||
printf 'scenarios:\n - id: S1\n cases: [C1_a]\ngate:\n blocking_gap_tolerance: 0\n implementation_files:\n - backend/src/foo/\n' > "$D/openspec/changes/chg_0001_demo/testcases.map.yaml"
|
||
git -C "$D" init -q -b main && git -C "$D" add -A && git -C "$D" commit -qm init
|
||
mkdir -p "$D/backend/src/foo" && echo x > "$D/backend/src/foo/a.ts"
|
||
git -C "$D" add -A && git -C "$D" commit -qm impl
|
||
# 期望:通过(pathspec 真生效,无 out-of-scope WARN)。
|
||
# note 只在写歪成 src/ 但根无 src/ 目录时触发;本用例白名单已是 backend/src/foo/,无 note。
|
||
# 关键反向断言:out-of-scope WARN 不应出现(旧实现下会静默失效,新实现下能精准放行)。
|
||
ck_neg() { # ck_neg <name> <want_exit> <not_grep> <dir>
|
||
local name="$1" want_exit="$2" not_grep="$3" dir="$4"
|
||
local out rc; out=$(PROJECT_ROOT="$dir" bash "$GATE" 2>&1); rc=$?
|
||
if [ "$rc" = "$want_exit" ] && ! printf '%s\n' "$out" | grep -qF "$not_grep"; then
|
||
echo "PASS $name"; PASS=$((PASS+1))
|
||
else
|
||
echo "FAIL $name (rc=$rc want=$want_exit; should NOT contain '$not_grep')"; printf '%s\n' "$out" | sed 's/^/ |/'; FAIL=$((FAIL+1))
|
||
fi
|
||
}
|
||
ck_neg "S12 monorepo路径白名单真实生效" 0 "out-of-scope" "$D"
|
||
|
||
# ---- S13: a4 map.yaml '# spec:' 注释 capability 不一致 → FAIL -----------------
|
||
D=$T/s13; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a
|
||
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios
|
||
|
||
### S1 add
|
||
$B_WHEN"
|
||
# 注释里 capability 写歪(demo-cap 与 spec.md 一致,但这里故意写 typo)—— 应 FAIL
|
||
printf '# spec: wrong-cap@chg_0001\nscenarios:\n - id: S1\n cases: [C1_a]\ngate:\n blocking_gap_tolerance: 0\n implementation_files:\n - src/demo/\n' > "$D/openspec/changes/chg_0001_demo/testcases.map.yaml"
|
||
ck "S13 map.yaml capability注释不一致FAIL" 1 "capability 不一致" "$D"
|
||
|
||
# ---- S14: --json 模式 PASS(结构合法 + 无文本漂移)--------------------------
|
||
# ck_json <name> <want_exit> <jq_assertion> <not_grep_text> <dir>
|
||
# - 跑 gate --json,断言退码 + jq 表达式 + stdout 不含指定文本(防 echo 守卫漏改)
|
||
ck_json() {
|
||
local name="$1" want_exit="$2" jq_expr="$3" not_text="$4" dir="$5"
|
||
local out rc
|
||
out=$(PROJECT_ROOT="$dir" bash "$GATE" --json 2>&1); rc=$?
|
||
local ok=1
|
||
[ "$rc" = "$want_exit" ] || ok=0
|
||
if [ -n "$not_text" ]; then
|
||
printf '%s\n' "$out" | grep -qF "$not_text" && ok=0
|
||
fi
|
||
# JSON 合法性 + jq 表达式断言(jq 失败 → ok=0)
|
||
if ! printf '%s\n' "$out" | jq -e "$jq_expr" >/dev/null 2>&1; then ok=0; fi
|
||
if [ "$ok" = "1" ]; then echo "PASS $name"; PASS=$((PASS+1));
|
||
else echo "FAIL $name (rc=$rc want=$want_exit; jq='$jq_expr'; not_grep='$not_text')"; printf '%s\n' "$out" | sed 's/^/ |/'; FAIL=$((FAIL+1)); fi
|
||
}
|
||
|
||
# 复用 S3 的 delta 合法 fixture(ADDED + MODIFIED + REMOVED + 主 spec 对齐)
|
||
D=$T/s14; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a C3_d
|
||
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## ADDED Scenarios
|
||
|
||
### S3 new thing
|
||
$B_WHEN
|
||
|
||
## MODIFIED Scenarios
|
||
|
||
### S1 add
|
||
- **WHEN** a-modified
|
||
- **THEN** b-modified
|
||
|
||
## REMOVED Scenarios
|
||
|
||
### S2 del
|
||
|
||
- **Reason**: superseded by S3
|
||
- **Migration**: use S3 path"
|
||
mkmap "$D/openspec/changes/chg_0001_demo" "S3:C3_d;S1:C1_a"
|
||
mkdir -p "$D/openspec/specs/demo-cap"
|
||
mkspec "$D/openspec/specs/demo-cap" "demo-cap" "## Concrete Scenarios
|
||
|
||
### S1 add
|
||
$B_WHEN
|
||
### S2 del
|
||
$B_WHEN"
|
||
# PASS 路径:jq 断言 (status=PASS, summary.pass>=1, 无 status envelope, 无文本漂移)
|
||
# 注意 jq `and` 左结合后右侧表达式以左侧 input 为上下文;用括号分组避免 .summary
|
||
# 丢失(jq -e 退码 1 = false → ck_json 报 FAIL)。
|
||
ck_json "S14 --json PASS退0+合法+无文本漂移" 0 \
|
||
'(.summary.fail == 0) and ((.changes | length) >= 1) and (any(.changes[]; .status == "PASS")) and ((.status // null) == null)' \
|
||
"==[ PASS ]==" "$D"
|
||
|
||
# ---- S15: --json 模式 FAIL(信封含期望 code + 文本抑制)---------------------
|
||
# 复用 S5 fixture(主 spec 不存在却有 MODIFIED → DELTA_BUT_NO_MAIN_SPEC)
|
||
D=$T/s15; mkp "$D"; mkt "$D"; mkc "$D/openspec/changes/chg_0001_demo" C1_a
|
||
mkspec "$D/openspec/changes/chg_0001_demo" "demo-cap" "## MODIFIED Scenarios
|
||
|
||
### S1 add
|
||
$B_WHEN"
|
||
mkmap "$D/openspec/changes/chg_0001_demo" "S1:C1_a"
|
||
# FAIL 路径:jq 断言 (status envelope 含期望 code, summary.fail>=1, 无 ==[ FAIL ]== 文本)
|
||
ck_json "S15 --json FAIL退1+信封含code+文本抑制" 1 \
|
||
'(.status | type == "array") and (.status | any(.[]; .code == "DELTA_BUT_NO_MAIN_SPEC")) and (.summary.fail >= 1)' \
|
||
"==[ FAIL ]==" "$D"
|
||
|
||
echo "----"
|
||
echo "PASS=$PASS FAIL=$FAIL"
|
||
[ "$FAIL" = "0" ]
|