#!/usr/bin/env python3 # 一次性去重:合并现有 objects 中「同类型 + 归一化名称相同/高度相似」的重复实体。 # 与 server/ai/ai.go 的 resolveObjectPK 逻辑保持一致(归一化 + 包含关系 + Jaccard≥0.85)。 # 用法:python3 dedup_cleanup.py # dry-run,只打印 # python3 dedup_cleanup.py --apply # 真正合并 import sys, json, psycopg2 DRY = "--apply" not in sys.argv print("MODE:", "DRY-RUN" if DRY else "APPLY") conn = psycopg2.connect(host="localhost", port=15432, user="dacli", password="dacli123", dbname="shangshu") cur = conn.cursor() # ---------- 与 Go 一致的归一化键(确定性,不模糊、不括号剥离、不链式) ---------- def normalize_name(s: str) -> str: s = (s or "").strip().lower() out = [] for ch in s: o = ord(ch) if 0xFF10 <= o <= 0xFF19: # 0-9 ch = chr(o - 0xFF10 + 0x30) elif 0xFF21 <= o <= 0xFF3A: # A-Z ch = chr(o - 0xFF21 + 0x41) elif 0xFF41 <= o <= 0xFF5A: # a-z ch = chr(o - 0xFF41 + 0x61) if ch.isalnum(): out.append(ch) return "".join(out) def dedup_key(type_name: str, name: str, pk: str) -> str: key = name if name else pk # 空 name 回退 pk,避免塌缩 return type_name + "\x00" + normalize_name(key) # ---------- 加载全部对象 ---------- cur.execute("SELECT id, type_api_name, pk, name, properties_json FROM objects") rows = cur.fetchall() print(f"加载对象 {len(rows)} 条") by_type = {} for r in rows: by_type.setdefault(r[1], []).append(r) # ---------- 按去重键分组(确定性,不会链式/括号误并) ---------- groups = [] for t, objs in by_type.items(): buckets = {} for r in objs: k = dedup_key(t, r[3], r[2]) buckets.setdefault(k, []).append(r) for members in buckets.values(): if len(members) > 1: groups.append((t, members)) print(f"\n发现重复组 {len(groups)} 个:") for t, members in groups: names = [m[3] or m[2] for m in members] print(f" [{t}] {' | '.join(names)}") if DRY: print("\n(DRY-RUN 结束,未做任何修改。加 --apply 执行合并)") sys.exit(0) # ---------- 执行合并 ---------- total_del = 0 for t, members in groups: # canonical:名称最短(可读性最佳),并列取最小 id members_sorted = sorted(members, key=lambda m: (len(m[3] or m[2]), m[0])) canon = members_sorted[0] canon_id, canon_pk = canon[0], canon[2] # 合并属性:canonical 缺失的键,从其他成员补齐 canon_props = dict(canon[4] or {}) for m in members_sorted[1:]: for k, v in (m[4] or {}).items(): if k not in canon_props and v not in (None, "", []): canon_props[k] = v cur.execute("UPDATE objects SET properties_json=%s::jsonb WHERE id=%s", (json.dumps(canon_props, ensure_ascii=False), canon_id)) for m in members_sorted[1:]: mid, mpk = m[0], m[2] # 关系重指向 canonical pk cur.execute("UPDATE links SET source_pk=%s WHERE source_type=%s AND source_pk=%s", (canon_pk, t, mpk)) cur.execute("UPDATE links SET target_pk=%s WHERE target_type=%s AND target_pk=%s", (canon_pk, t, mpk)) # 访问标记随对象清理 cur.execute("DELETE FROM object_markings WHERE object_type=%s AND object_pk=%s", (t, mpk)) cur.execute("DELETE FROM objects WHERE id=%s", (mid,)) total_del += 1 # ---------- 关系去重(同 5 元组只留一条) ---------- cur.execute(""" DELETE FROM links a USING links b WHERE a.id > b.id AND a.link_type=b.link_type AND a.source_type=b.source_type AND a.source_pk=b.source_pk AND a.target_type=b.target_type AND a.target_pk=b.target_pk """) conn.commit() print(f"\n已合并:删除重复对象 {total_del} 条,关系已按 5 元组去重。") cur.execute("SELECT count(*) FROM objects"); print("剩余对象:", cur.fetchone()[0]) cur.execute("SELECT count(*) FROM links"); print("剩余关系:", cur.fetchone()[0]) cur.close(); conn.close()