- 将 shangshu-ontology/ 子目录内容移至仓库根一级 - 修复 run.sh 的 REPO_ROOT 计算(扁平化后不再需要 cd ..),ARCHIVE_DIR 正确指向仓库根/archive - 文档(README/PLAN/USAGE/IMPORT/product-intro)移至 docs/,同步更新结构树与导入命令中的旧路径 - 新增根 README.md 索引指向 docs/ - 新增 .gitignore:忽略 bin/、web/node_modules/、web/dist/、server/webroot/、.env、.workbuddy/ 等噪音 - main.go 的 ARCHIVE_DIR fallback 基于可执行文件相邻上级,扁平化后仍正确,无需改动
104 lines
4.1 KiB
Python
104 lines
4.1 KiB
Python
#!/usr/bin/env python3
|
||
# 一次性去重:合并现有 objects 中「同类型 + 归一化名称相同/高度相似」的重复实体。
|
||
# 与 server/ai/ai.go 的 resolveObjectPK 逻辑保持一致(归一化 + 包含关系 + Jaccard≥0.85)。
|
||
# 用法:python3 dedup_cleanup.py # dry-run,只打印
|
||
# python3 dedup_cleanup.py --apply # 真正合并
|
||
import sys, json, psycopg2
|
||
|
||
DRY = "--apply" not in sys.argv
|
||
print("MODE:", "DRY-RUN" if DRY else "APPLY")
|
||
|
||
conn = psycopg2.connect(host="localhost", port=15432, user="dacli",
|
||
password="dacli123", dbname="shangshu")
|
||
cur = conn.cursor()
|
||
|
||
# ---------- 与 Go 一致的归一化键(确定性,不模糊、不括号剥离、不链式) ----------
|
||
def normalize_name(s: str) -> str:
|
||
s = (s or "").strip().lower()
|
||
out = []
|
||
for ch in s:
|
||
o = ord(ch)
|
||
if 0xFF10 <= o <= 0xFF19: # 0-9
|
||
ch = chr(o - 0xFF10 + 0x30)
|
||
elif 0xFF21 <= o <= 0xFF3A: # A-Z
|
||
ch = chr(o - 0xFF21 + 0x41)
|
||
elif 0xFF41 <= o <= 0xFF5A: # a-z
|
||
ch = chr(o - 0xFF41 + 0x61)
|
||
if ch.isalnum():
|
||
out.append(ch)
|
||
return "".join(out)
|
||
|
||
def dedup_key(type_name: str, name: str, pk: str) -> str:
|
||
key = name if name else pk # 空 name 回退 pk,避免塌缩
|
||
return type_name + "\x00" + normalize_name(key)
|
||
|
||
# ---------- 加载全部对象 ----------
|
||
cur.execute("SELECT id, type_api_name, pk, name, properties_json FROM objects")
|
||
rows = cur.fetchall()
|
||
print(f"加载对象 {len(rows)} 条")
|
||
|
||
by_type = {}
|
||
for r in rows:
|
||
by_type.setdefault(r[1], []).append(r)
|
||
|
||
# ---------- 按去重键分组(确定性,不会链式/括号误并) ----------
|
||
groups = []
|
||
for t, objs in by_type.items():
|
||
buckets = {}
|
||
for r in objs:
|
||
k = dedup_key(t, r[3], r[2])
|
||
buckets.setdefault(k, []).append(r)
|
||
for members in buckets.values():
|
||
if len(members) > 1:
|
||
groups.append((t, members))
|
||
|
||
print(f"\n发现重复组 {len(groups)} 个:")
|
||
for t, members in groups:
|
||
names = [m[3] or m[2] for m in members]
|
||
print(f" [{t}] {' | '.join(names)}")
|
||
|
||
if DRY:
|
||
print("\n(DRY-RUN 结束,未做任何修改。加 --apply 执行合并)")
|
||
sys.exit(0)
|
||
|
||
# ---------- 执行合并 ----------
|
||
total_del = 0
|
||
for t, members in groups:
|
||
# canonical:名称最短(可读性最佳),并列取最小 id
|
||
members_sorted = sorted(members, key=lambda m: (len(m[3] or m[2]), m[0]))
|
||
canon = members_sorted[0]
|
||
canon_id, canon_pk = canon[0], canon[2]
|
||
# 合并属性:canonical 缺失的键,从其他成员补齐
|
||
canon_props = dict(canon[4] or {})
|
||
for m in members_sorted[1:]:
|
||
for k, v in (m[4] or {}).items():
|
||
if k not in canon_props and v not in (None, "", []):
|
||
canon_props[k] = v
|
||
cur.execute("UPDATE objects SET properties_json=%s::jsonb WHERE id=%s",
|
||
(json.dumps(canon_props, ensure_ascii=False), canon_id))
|
||
for m in members_sorted[1:]:
|
||
mid, mpk = m[0], m[2]
|
||
# 关系重指向 canonical pk
|
||
cur.execute("UPDATE links SET source_pk=%s WHERE source_type=%s AND source_pk=%s",
|
||
(canon_pk, t, mpk))
|
||
cur.execute("UPDATE links SET target_pk=%s WHERE target_type=%s AND target_pk=%s",
|
||
(canon_pk, t, mpk))
|
||
# 访问标记随对象清理
|
||
cur.execute("DELETE FROM object_markings WHERE object_type=%s AND object_pk=%s",
|
||
(t, mpk))
|
||
cur.execute("DELETE FROM objects WHERE id=%s", (mid,))
|
||
total_del += 1
|
||
|
||
# ---------- 关系去重(同 5 元组只留一条) ----------
|
||
cur.execute("""
|
||
DELETE FROM links a USING links b
|
||
WHERE a.id > b.id
|
||
AND a.link_type=b.link_type AND a.source_type=b.source_type
|
||
AND a.source_pk=b.source_pk AND a.target_type=b.target_type AND a.target_pk=b.target_pk
|
||
""")
|
||
conn.commit()
|
||
print(f"\n已合并:删除重复对象 {total_del} 条,关系已按 5 元组去重。")
|
||
cur.execute("SELECT count(*) FROM objects"); print("剩余对象:", cur.fetchone()[0])
|
||
cur.execute("SELECT count(*) FROM links"); print("剩余关系:", cur.fetchone()[0])
|
||
cur.close(); conn.close()
|