Files
ssw-ontology/scripts/dedup_cleanup.py
T
ssw f1b687c2d2 目录扁平化:子目录内容提升至仓库根,文档归入 docs/
- 将 shangshu-ontology/ 子目录内容移至仓库根一级
- 修复 run.sh 的 REPO_ROOT 计算(扁平化后不再需要 cd ..),ARCHIVE_DIR 正确指向仓库根/archive
- 文档(README/PLAN/USAGE/IMPORT/product-intro)移至 docs/,同步更新结构树与导入命令中的旧路径
- 新增根 README.md 索引指向 docs/
- 新增 .gitignore:忽略 bin/、web/node_modules/、web/dist/、server/webroot/、.env、.workbuddy/ 等噪音
- main.go 的 ARCHIVE_DIR fallback 基于可执行文件相邻上级,扁平化后仍正确,无需改动
2026-08-07 14:21:34 +08:00

104 lines
4.1 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
# 一次性去重:合并现有 objects 中「同类型 + 归一化名称相同/高度相似」的重复实体。
# 与 server/ai/ai.go 的 resolveObjectPK 逻辑保持一致(归一化 + 包含关系 + Jaccard≥0.85)。
# 用法:python3 dedup_cleanup.py # dry-run,只打印
# python3 dedup_cleanup.py --apply # 真正合并
import sys, json, psycopg2
DRY = "--apply" not in sys.argv
print("MODE:", "DRY-RUN" if DRY else "APPLY")
conn = psycopg2.connect(host="localhost", port=15432, user="dacli",
password="dacli123", dbname="shangshu")
cur = conn.cursor()
# ---------- 与 Go 一致的归一化键(确定性,不模糊、不括号剥离、不链式) ----------
def normalize_name(s: str) -> str:
s = (s or "").strip().lower()
out = []
for ch in s:
o = ord(ch)
if 0xFF10 <= o <= 0xFF19: # 0-9
ch = chr(o - 0xFF10 + 0x30)
elif 0xFF21 <= o <= 0xFF3A: # A-Z
ch = chr(o - 0xFF21 + 0x41)
elif 0xFF41 <= o <= 0xFF5A: # a-z
ch = chr(o - 0xFF41 + 0x61)
if ch.isalnum():
out.append(ch)
return "".join(out)
def dedup_key(type_name: str, name: str, pk: str) -> str:
key = name if name else pk # 空 name 回退 pk,避免塌缩
return type_name + "\x00" + normalize_name(key)
# ---------- 加载全部对象 ----------
cur.execute("SELECT id, type_api_name, pk, name, properties_json FROM objects")
rows = cur.fetchall()
print(f"加载对象 {len(rows)} 条")
by_type = {}
for r in rows:
by_type.setdefault(r[1], []).append(r)
# ---------- 按去重键分组(确定性,不会链式/括号误并) ----------
groups = []
for t, objs in by_type.items():
buckets = {}
for r in objs:
k = dedup_key(t, r[3], r[2])
buckets.setdefault(k, []).append(r)
for members in buckets.values():
if len(members) > 1:
groups.append((t, members))
print(f"\n发现重复组 {len(groups)} 个:")
for t, members in groups:
names = [m[3] or m[2] for m in members]
print(f" [{t}] {' | '.join(names)}")
if DRY:
print("\n(DRY-RUN 结束,未做任何修改。加 --apply 执行合并)")
sys.exit(0)
# ---------- 执行合并 ----------
total_del = 0
for t, members in groups:
# canonical:名称最短(可读性最佳),并列取最小 id
members_sorted = sorted(members, key=lambda m: (len(m[3] or m[2]), m[0]))
canon = members_sorted[0]
canon_id, canon_pk = canon[0], canon[2]
# 合并属性:canonical 缺失的键,从其他成员补齐
canon_props = dict(canon[4] or {})
for m in members_sorted[1:]:
for k, v in (m[4] or {}).items():
if k not in canon_props and v not in (None, "", []):
canon_props[k] = v
cur.execute("UPDATE objects SET properties_json=%s::jsonb WHERE id=%s",
(json.dumps(canon_props, ensure_ascii=False), canon_id))
for m in members_sorted[1:]:
mid, mpk = m[0], m[2]
# 关系重指向 canonical pk
cur.execute("UPDATE links SET source_pk=%s WHERE source_type=%s AND source_pk=%s",
(canon_pk, t, mpk))
cur.execute("UPDATE links SET target_pk=%s WHERE target_type=%s AND target_pk=%s",
(canon_pk, t, mpk))
# 访问标记随对象清理
cur.execute("DELETE FROM object_markings WHERE object_type=%s AND object_pk=%s",
(t, mpk))
cur.execute("DELETE FROM objects WHERE id=%s", (mid,))
total_del += 1
# ---------- 关系去重(同 5 元组只留一条) ----------
cur.execute("""
DELETE FROM links a USING links b
WHERE a.id > b.id
AND a.link_type=b.link_type AND a.source_type=b.source_type
AND a.source_pk=b.source_pk AND a.target_type=b.target_type AND a.target_pk=b.target_pk
""")
conn.commit()
print(f"\n已合并:删除重复对象 {total_del} 条,关系已按 5 元组去重。")
cur.execute("SELECT count(*) FROM objects"); print("剩余对象:", cur.fetchone()[0])
cur.execute("SELECT count(*) FROM links"); print("剩余关系:", cur.fetchone()[0])
cur.close(); conn.close()