#!/usr/bin/env python3 # -*- coding: utf-8 -*- """check-citations-scan —— 文档引文(`路径:行号`)校验扫描器。 🔴 为什么存在: 仓根 + docs/ 的文档(技能向说明归技能仓 `ssw-jnpf-lite-rust`) 大量引用源码位置,语法有三种: `文件.ext:行号` 单行 `文件.ext:起-止` 区间 `文件.ext:行/行/行` 多个散行(斜杠 = 「这几行分别」) `文件.ext:行+` 该行及其后 这些行号是**会漂移**的:源码改一行 ⇒ 上游行号整体位移 ⇒ 文档里的引用 **静默指向别的行**——不报错,只是悄悄说错话。⇒ 必须有把行号核对清楚的尺子。 🔴 纯静态:只读文件,不起服务、不连库、不发请求。纯 `python3`(macOS 自带)。 用法: check-citations-scan.py --root <仓根> [--jnpf <官方源码根>] [--react ] check-citations-scan.py --list # 打印每条引文明细(排查用) check-citations-scan.py --self-test # 反自证:注入已知坏引用 ⇒ 必须被抓 输出协议(制表符分隔,逐行一条): CHECK\t\t\t<说明> WARN\t\t<说明> 退出码:0 = 扫完(看输出里的 DRIFT)/ 2 = 致命错误(参数错等) """ import argparse import os import re import shutil import sys import tempfile HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(HERE) # 外部参照根(可用环境变量 / 命令行覆盖) DEFAULT_JNPF = os.path.expanduser("~/Desktop/workspace/jnpf") DEFAULT_REACT = os.path.expanduser("~/Desktop/workspace/skills/ssw-jnpf-lite-react") DEFAULT_DDL = os.path.expanduser( "~/Desktop/workspace/jnpf/jnpf-database/PostgreSQL/JNPF-PostgreSQL.sql" ) # 依赖源码根(cargo registry)—— 文档里会引「某个 crate 的某一版源码的某一行」 DEFAULT_CARGO = os.path.expanduser("~/.cache/cargo/registry/src") CARGO_FALLBACK = os.path.expanduser("~/.cargo/registry/src") # 遍历时跳过的目录(构建产物 / 依赖树 / 运行期数据) SKIP_DIRS = { ".git", ".idea", ".vscode", "node_modules", "target", "dist", "build", ".next", ".nuxt", "data", "__pycache__", ".venv", "venv", "vendor", ".workbuddy", ".superpowers", "coverage", ".pytest_cache", ".mypy_cache", } # 文档语料:仓根 *.md + docs/**(技能资产文档已迁出本仓,归技能仓 ssw-jnpf-lite-rust) DOC_GLOBS = ( ("", "*.md"), ("docs", "**/*.md"), ) # 承认的文件后缀(决定什么算「一条引文」) EXTS = ( "rs|ts|tsx|jsx|js|vue|sql|sh|py|java|kt|xml|yml|yaml|json|md|html|" "css|scss|toml|properties|gradle|c|h|go|txt|conf|ini" ) # 引文本体:路径 + 行号部分 CITE_RE = re.compile( r"(?P(?:[A-Za-z0-9_.\-]+/)*[A-Za-z0-9_.\-]*\.(?:" + EXTS + r"))" r":(?P\d+(?:(?:\s*[/\-]\s*)\d+)*\+?)" ) # 「被提及的文件名」(不带行号)—— 用于给孤儿行号继承上文文件名 # 🔴 允许 `~/` 前缀(文档里常见 `~/Desktop/workspace/jnpf/…` 这种绝对坐标) FILE_RE = re.compile( r"(?P~?/?(?:[A-Za-z0-9_.\-]+/)*[A-Za-z0-9_.\-]*\.(?:" + EXTS + r"))" ) # 孤儿行号(`:4503` 这种没有路径的续行引用)—— 只报不判失败 ORPHAN_RE = re.compile(r"(? 0 and text[m.start() - 1] in "\"'": continue # 🔴 **端口号不是引文**:本仓的端口一律写成反引号包裹的 `` `:30010` ``。 # 判据 = 反引号包裹 **且** 数值 ≥ 1024(真引文的行号在同一写法下都 < 1024, # 而 DDL 那种大行号(812/2209/…)**不**用反引号包裹)。 # 🔴 **端口号不是引文**:本仓的端口写成 `` `:30010` `` 或行内的 `后端 :30010`。 # 判据 = 数字**后面还跟着别的内容**(行内写法)⇒ 端口 / 版本号 / 别的数值; # 数字**后面是行尾或表格竖线**(列写法)⇒ 才是行号引用。 # (两种写法的区别是结构性的,不是猜的。) tail = text[m.end():] stripped = tail.lstrip(" \t`") if stripped and not stripped.startswith("|"): continue # 表格单元格里的 `` `:30000` `` 同样是端口(数值 ≥ 1024 且被反引号包着) num = int(m.group(1)) backticked = (m.start() > 0 and text[m.start() - 1] == "`" and m.end() < len(text) and text[m.end()] == "`") if backticked and num >= 1024: continue inherited = None for pos, name in reversed(mentions): if pos < m.start(): inherited = name break orphans.append({ "doc": path, "doc_line": text.count("\n", 0, m.start()) + 1, "line": int(m.group(1)), "inherited": inherited, }) return cites, orphans, mentions_by_doc _LOOKUP_CACHE = {} def _lookups(indexes): """把 `{根: {相对路径: 绝对路径}}` 展开成两张倒排表(按后缀 / 按基名)。 🔴 **为什么需要这张表**(实测数据):原来的 `resolve` 对**每条引文**都要遍历 该根下的**全部文件**做 `rel.endswith("/" + 路径)`。本仓 551 条引文 × 两千多个文件 ⇒ 实测 **1008 万次** `endswith`(1.7s,占本闸门四成)。 ⇒ 建一次倒排表(每条路径按层级切出若干后缀),查询变成 O(1) 查表。 缓存键用 `id(indexes)`,但**同时把 indexes 本体存进缓存值** ⇒ 只要缓存项还活着, 那个 id 就不可能被别的对象复用 ⇒ 不会脏读(同 `find_fn_body` 的做法)。 """ key = id(indexes) hit = _LOOKUP_CACHE.get(key) if hit is not None and hit[0] is indexes: return hit[1], hit[2] suffix_map, base_map = {}, {} for name, files in indexes.items(): for rel, ap in files.items(): base_map.setdefault((name, os.path.basename(rel)), []).append((rel, ap)) parts = rel.split("/") for i in range(len(parts)): suffix_map.setdefault((name, "/" + "/".join(parts[i:])), []).append((rel, ap)) _LOOKUP_CACHE[key] = (indexes, suffix_map, base_map) return suffix_map, base_map def resolve(cite_path, roots, indexes): """把引文里的路径解析成候选文件列表 [(root名, 绝对路径), …]。 三级解析(先精确后模糊): ① 精确:<根>/<路径> 存在 ② 后缀:某根下存在相对路径以 `/<路径>` 结尾(覆盖 `db/schema.sql` → `src/backend/db/schema.sql`) ③ 基名:路径里没有 `/` 时,按文件名在全仓找(官方源码文档里大量 `XxxController.java:27` 这类写法) 🔴 `~/` 开头的按**绝对路径**处理(文档里直接写官方源码坐标)。 """ cands = [] expanded = os.path.expanduser(cite_path) if expanded.startswith("/"): if os.path.isfile(expanded): cands.append(("abs", expanded)) return cands, "abs" for name, base in roots.items(): p = os.path.join(base, cite_path) if os.path.isfile(p): cands.append((name, p)) if cands: return cands, "exact" suffix_map, base_map = _lookups(indexes) suffix = "/" + cite_path for name in roots: for rel, ap in suffix_map.get((name, suffix), ()): cands.append((name, ap)) if cands: return cands, "suffix" if "/" not in cite_path: for name in roots: for rel, ap in base_map.get((name, cite_path), ()): cands.append((name, ap)) if cands: return cands, "basename" return [], "none" def judge(citation, roots, indexes): """判定一条引文。返回 (状态, 解析结果, 说明)。 状态:OK / RANGE_BAD(文件在但行号越界)/ UNRESOLVED(文件找不到) """ cands, tier = resolve(citation["path"], roots, indexes) if not cands: return "UNRESOLVED", [], "找不到文件" fits = [] for name, ap in cands: lc = line_count(ap) if all(start <= lc and (end is None or end <= lc) for start, end in citation["spans"]): fits.append((name, ap)) if fits: return "OK", fits, tier detail = "、".join( "%s(%d 行)" % (os.path.relpath(ap, roots[name]), line_count(ap)) for name, ap in cands[:3] ) return "RANGE_BAD", cands, detail # ---------------------------------------------------------------- 检查项 def check_citations(cites, roots, indexes, missing_roots): """CIT1 可解析 / CIT2 行号有效。返回 (c1_status, c1_detail, c2_status, c2_detail, bad列表)。""" unresolved = [] bad_range = [] for c in cites: status, cands, detail = judge(c, roots, indexes) c["status"] = status c["detail"] = detail if status == "UNRESOLVED": unresolved.append(c) elif status == "RANGE_BAD": bad_range.append(c) # CIT1 —— 可解析 if not unresolved: c1_status, c1_detail = "OK", "%d 条引文全部定位到真实文件" % len(cites) elif missing_roots: c1_status = "SKIP" c1_detail = "%d 条引文定位不到(外部参照根缺失:%s ⇒ 跳过,不判失败)" % ( len(unresolved), "、".join(missing_roots)) else: c1_status = "DRIFT" c1_detail = "%d 条引文定位不到文件:%s" % ( len(unresolved), ";".join("%s:%d → %s" % (os.path.basename(c["doc"]), c["doc_line"], c["path"]) for c in unresolved[:6])) # CIT2 —— 行号有效 if not bad_range: c2_status, c2_detail = "OK", "%d 条引文的行号全部落在文件行数内" % len(cites) else: c2_status = "DRIFT" c2_detail = "%d 条引文行号越界:%s" % ( len(bad_range), ";".join("%s:%d → %s(文件最多 %s 行)" % ( os.path.basename(c["doc"]), c["doc_line"], c["path"], c["detail"]) for c in bad_range[:6])) return c1_status, c1_detail, c2_status, c2_detail, unresolved + bad_range # 孤儿行号的「上文窗口」:文件通常在同一个小节 / 同一张表里被点名 ORPHAN_WINDOW = 40 def check_orphans(orphans, roots, indexes, mentions_by_doc): """核对「孤儿行号」(`:NNN`)。 🔴 **为什么不能简单「继承上文最近的文件名」**:同一张表里**相邻两行可能指的是不同文件** (官方文档的表常在「官方出处」列里写 `前端 xxx.ts:14` 与 `XxxController.java:107-110` 混排) ⇒ 机械继承会**大量误报**。 ⇒ 折中口径:取**上文 40 行内被点名过的所有文件**,只要**其中任意一个**的行数 覆盖得住这个行号 ⇒ 视为「对得上」(弱判据、低误报)。 一个都对不上 ⇒ 报「疑似漂移」(**只报不判失败**)。 """ checked = 0 unresolved = 0 suspects = [] for o in orphans: near = mentions_by_doc.get(o["doc"], []) cands = [] for (line, name) in near: if not (o["doc_line"] - ORPHAN_WINDOW <= line <= o["doc_line"]): continue got, _ = resolve(name, roots, indexes) cands.extend(got) if not cands: unresolved += 1 continue checked += 1 if not any(o["line"] <= line_count(ap) for _, ap in cands): suspects.append("%s:%d → 引用了第 %d 行,但上文 40 行内点名的文件最多只有 %d 行" % ( os.path.basename(o["doc"]), o["doc_line"], o["line"], max(line_count(ap) for _, ap in cands))) return checked, unresolved, suspects def check_ddl_table(root, ddl_path): """CIT3 —— 文档里的 DDL 行号表 ↔ 官方 DDL 真实行。 文档写「`base_authorize` | 812」⇒ 官方 DDL 第 812 行必须真的有 `CREATE TABLE base_authorize` ⇒ 行号漂移(DDL 增删表)当场被抓。 """ claude = os.path.join(root, "CLAUDE.md") if not os.path.isfile(claude): return "SKIP", "仓根没有 CLAUDE.md" if not os.path.isfile(ddl_path): return "SKIP", "官方 DDL 不存在:%s" % ddl_path with open(claude, "r", encoding="utf-8", errors="replace") as fh: text = fh.read() rows = DDL_ROW_RE.findall(text) if not rows: return "SKIP", "CLAUDE.md 里没有 DDL 行号表" try: with open(ddl_path, "r", encoding="utf-8", errors="replace") as fh: ddl_lines = fh.readlines() except OSError as exc: return "SKIP", "读官方 DDL 失败:%s" % exc bad = [] for name, num in rows: n = int(num) if n < 1 or n > len(ddl_lines): bad.append("%s 标 %d 行(DDL 共 %d 行)" % (name, n, len(ddl_lines))) continue line = ddl_lines[n - 1] if not re.search(DDL_CREATE_RE.format(name=re.escape(name)), line, re.I): bad.append("%s 标 %d 行,但该行是「%s」" % (name, n, line.strip()[:60])) if bad: return "DRIFT", "DDL 行号漂移 %d 条:%s" % (len(bad), ";".join(bad)) return "OK", "%d 条 DDL 行号与官方 DDL 逐行对上" % len(rows) # ---------------------------------------------------------------- 反自证 def self_test(): """注入已知坏引文 ⇒ 必须被抓到。永远返回「干净」的尺子 = 没有尺子。""" failures = [] tmpdir = tempfile.mkdtemp(prefix="cite-selftest-") try: # 造一份「干净」的文档 + 一个假源码文件 src_dir = os.path.join(tmpdir, "src") os.makedirs(src_dir) code = os.path.join(src_dir, "demo.rs") with open(code, "w", encoding="utf-8") as fh: fh.write("\n".join("line%d" % i for i in range(1, 21)) + "\n") # 20 行 doc = os.path.join(tmpdir, "doc.md") with open(doc, "w", encoding="utf-8") as fh: fh.write("见 `src/demo.rs:5` 与 `src/demo.rs:12-18`。\n") roots = {"tmp": tmpdir} indexes = {"tmp": build_index(tmpdir)} cites, _, _ = extract_citations([doc]) c1, _, c2, _, bad = check_citations(cites, roots, indexes, []) if c1 != "OK" or c2 != "OK" or bad: failures.append("干净引文被误判(c1=%s c2=%s bad=%d)" % (c1, c2, len(bad))) # 注入 1:行号越界 with open(doc, "w", encoding="utf-8") as fh: fh.write("见 `src/demo.rs:999`。\n") cites, _, _ = extract_citations([doc]) _, _, c2, _, bad = check_citations(cites, roots, indexes, []) if c2 != "DRIFT": failures.append("行号越界没被抓到(c2=%s)" % c2) # 注入 2:文件不存在 with open(doc, "w", encoding="utf-8") as fh: fh.write("见 `src/nope.rs:3`。\n") cites, _, _ = extract_citations([doc]) c1, _, _, _, bad = check_citations(cites, roots, indexes, []) if c1 != "DRIFT": failures.append("文件不存在没被抓到(c1=%s)" % c1) # 注入 3:DDL 行号写错 ddl = os.path.join(tmpdir, "ddl.sql") with open(ddl, "w", encoding="utf-8") as fh: fh.write("CREATE TABLE base_alpha (\n a int\n);\n") claude = os.path.join(tmpdir, "CLAUDE.md") with open(claude, "w", encoding="utf-8") as fh: fh.write("| `base_alpha` | 1 | x |\n") st, _ = check_ddl_table(tmpdir, ddl) if st != "OK": failures.append("正确 DDL 行号被误判(%s)" % st) with open(claude, "w", encoding="utf-8") as fh: fh.write("| `base_alpha` | 2 | x |\n") st, _ = check_ddl_table(tmpdir, ddl) if st != "DRIFT": failures.append("DDL 行号错位没被抓到(%s)" % st) finally: shutil.rmtree(tmpdir, ignore_errors=True) if failures: for f in failures: print("SELFTEST-FAIL\t%s" % f) return 1 print("反自证通过:行号越界 / 文件缺失 / DDL 行号错位 三类坏引用都被抓到,干净引文不误报") return 0 # ---------------------------------------------------------------- 主流程 def main(argv): ap = argparse.ArgumentParser(description="文档引文(路径:行号)校验扫描器") ap.add_argument("--root", default=ROOT, help="仓根(默认:本脚本所在仓)") ap.add_argument("--jnpf", default=os.environ.get("JNPF_SRC_ROOT", DEFAULT_JNPF), help="官方 JNPF 源码根") ap.add_argument("--react", default=os.environ.get("REACT_SKILL_ROOT", DEFAULT_REACT), help="react 姊妹技能根") ap.add_argument("--ddl", default=os.environ.get("JNPF_DDL", DEFAULT_DDL), help="官方 DDL 文件") ap.add_argument("--cargo", default=os.environ.get("CARGO_SRC_ROOT", DEFAULT_CARGO), help="cargo registry 的 src 目录(引文指到依赖源码时用)") ap.add_argument("--list", action="store_true", help="打印每条引文明细") ap.add_argument("--self-test", action="store_true", help="反自证模式") args = ap.parse_args(argv) if args.self_test: return self_test() root = os.path.abspath(args.root) if not os.path.isdir(root): print("❌ 仓根不存在:%s" % root) return 2 roots = { "repo": root, "jnpf": os.path.abspath(os.path.expanduser(args.jnpf)), "react": os.path.abspath(os.path.expanduser(args.react)), } # 依赖源码根(pglite-rs 的各版本)—— 文档里会引「某依赖的某一行」 cargo_base = os.path.abspath(os.path.expanduser(args.cargo)) if not os.path.isdir(cargo_base) and os.path.isdir(CARGO_FALLBACK): cargo_base = CARGO_FALLBACK for i, crate_root in enumerate(cargo_crate_roots(cargo_base)): roots["cargo%d" % i] = crate_root missing_roots = [n for n, p in roots.items() if not os.path.isdir(p)] indexes = {n: build_index(p) for n, p in roots.items() if os.path.isdir(p)} docs = collect_docs(root) cites, orphans, mentions_by_doc = extract_citations(docs) c1, c1d, c2, c2d, bad = check_citations(cites, roots, indexes, missing_roots) emit_check("CIT1", c1, c1d) emit_check("CIT2", c2, c2d) c3, c3d = check_ddl_table(root, os.path.abspath(os.path.expanduser(args.ddl))) emit_check("CIT3", c3, c3d) o_checked, o_unresolved, o_suspects = check_orphans(orphans, roots, indexes, mentions_by_doc) if orphans: emit_warn("CIT-W1", "%d 处「无路径的续行行号」(`:NNN`,本仓的表格约定)—— " "按上文最近的文件名继承核对:%d 处对得上,%d 处无法定位" % ( len(orphans), o_checked, o_unresolved)) for s in o_suspects[:8]: emit_warn("CIT-W1", "疑似漂移:%s" % s) if missing_roots: emit_warn("CIT-W2", "外部参照根缺失:%s ⇒ 指向它们的引文按「跳过」处理" % "、".join("%s=%s" % (n, roots[n]) for n in missing_roots)) if args.list: print("---- 引文明细 ----") for c in cites: print("%s:%d\t%s:%s\t%s\t%s" % ( os.path.relpath(c["doc"], root), c["doc_line"], c["path"], c["body"], c.get("status", "?"), c.get("detail", ""))) return 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))