「让客户端敢操作」的关键能力:写前/写后打一份快照 → 任意 GET 带 X-Snapshot-Id
头即可命中 backups_<id> schema 的同名表(业务 SQL 不带 schema 前缀照样切)。
设计要点:
- 备份 = 独立 schema backups_<id>(真表,不是单存 JSONB)—— 查询性能与活库一致
- snapshotRouter 中间件:BeginTx + SET LOCAL search_path → 把 a.pool 临时换
为事务视图 txPool → handler 结束 commit + 还原 a.pool。业务代码一行不改。
- a.pool 字段类型从 *pgxpool.Pool 升级为 dbRunner interface(中间件可换 txPool)。
- 11 个 helper 函数签名同步改为 dbExec(与现有 dbExec 共存,无破坏性)。
- 写方法(POST/PUT/PATCH/DELETE)带快照头 → 403 拒绝(mistake-safety)。
接口:
- POST /api/v1/backups 创建
- GET /api/v1/backups 列表
- GET /api/v1/backups/:id 元信息(含 schema_name)
- GET /api/v1/backups/:id/switch 切库提示
- DELETE /api/v1/backups/:id 级联删 schema
- 支持 X-Snapshot-Id (直接 run id) + X-Snapshot-Ts (秒级时间戳) 两种切换入口
MCP 工具 (5 个):backup_create / backup_list / backup_inspect / backup_switch / backup_delete
测试 (22 个全绿,~5.5s 集成):
- SnapshotRouter:改前建快照 → 改后读快照只见改前
- SnapshotSearchPathPersist:不污染下一个请求
- SnapshotByTimestamp:ts 头也能切库
- SnapshotDeleteCleansSchema:delete 真 DROP schema
- BackupLifecycle / BackupDiffAfterMutation:完整 create→list→inspect→switch→delete
文档:docs/BACKUP.md 写清用法、机制、客户端工作流;CHANGELOG.md [1.3.0]。
909 lines
30 KiB
Go
909 lines
30 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/gin-gonic/gin"
|
||
)
|
||
|
||
// ---------- 实例治理(2026-09-05) ----------
|
||
//
|
||
// 来源:数字孪生本体外部实操反馈(docs/INSTANCE_GOVERNANCE.md 原始需求)+ 本库扫描复核。
|
||
// 覆盖:级联删除 / 批量写入 / 字段级变更审计 / 历史回滚 / 重复实体提示。
|
||
//
|
||
// ⚠️ 关于"9-2 / 9-3 数据丢失":根因是容器 PGDATA 未持久化(容器重置即回退到初始库),
|
||
// 已在 docker/entrypoint.sh 修复(PGDATA 用户传值优先 + EXTERNAL_PG 模式)。
|
||
// create_object 的全量覆盖语义虽不是那次事故的原因,但确实是危险的默认值,
|
||
// 故仍按需求改为 merge 优先 —— 只是优先级从"事故根因"降为"隐患治理"。
|
||
|
||
// ---------- 字段级 diff ----------
|
||
|
||
type propDiff struct {
|
||
Added []string `json:"added"`
|
||
Removed []string `json:"removed"`
|
||
Changed map[string]map[string]interface{} `json:"changed"` // k -> {"from":..,"to":..}
|
||
}
|
||
|
||
func (d propDiff) empty() bool {
|
||
return len(d.Added) == 0 && len(d.Removed) == 0 && len(d.Changed) == 0
|
||
}
|
||
|
||
// diffProps 对比新旧属性,产出字段级变更明细。
|
||
func diffProps(old, cur map[string]interface{}) propDiff {
|
||
d := propDiff{Changed: map[string]map[string]interface{}{}}
|
||
if old == nil {
|
||
old = map[string]interface{}{}
|
||
}
|
||
if cur == nil {
|
||
cur = map[string]interface{}{}
|
||
}
|
||
for k, v := range cur {
|
||
ov, ok := old[k]
|
||
if !ok {
|
||
d.Added = append(d.Added, k)
|
||
continue
|
||
}
|
||
if mustJSON(ov) != mustJSON(v) {
|
||
d.Changed[k] = map[string]interface{}{"from": ov, "to": v}
|
||
}
|
||
}
|
||
for k, v := range old {
|
||
if _, ok := cur[k]; !ok {
|
||
_ = v
|
||
d.Removed = append(d.Removed, k)
|
||
}
|
||
}
|
||
sort.Strings(d.Added)
|
||
sort.Strings(d.Removed)
|
||
return d
|
||
}
|
||
|
||
// mergeProps 把 patch 合并进 base,返回合并结果与被删除的键。
|
||
// 约定:patch 中显式传 null 表示删除该字段(merge 模式下唯一能删字段的方式)。
|
||
func mergeProps(base, patch map[string]interface{}) (map[string]interface{}, []string) {
|
||
out := map[string]interface{}{}
|
||
for k, v := range base {
|
||
out[k] = v
|
||
}
|
||
var removed []string
|
||
for k, v := range patch {
|
||
if v == nil {
|
||
if _, existed := out[k]; existed {
|
||
delete(out, k)
|
||
removed = append(removed, k)
|
||
}
|
||
continue
|
||
}
|
||
out[k] = v
|
||
}
|
||
sort.Strings(removed)
|
||
return out, removed
|
||
}
|
||
|
||
// recordChange 写一条对象级变更审计。失败只记日志,不阻断主流程。
|
||
func recordChange(ctx context.Context, p dbExec, ch objectChange) {
|
||
if p == nil {
|
||
return
|
||
}
|
||
_, err := p.Exec(ctx,
|
||
`INSERT INTO object_changes(object_type,object_pk,action,actor,via,diff,snapshot,source,links_snapshot)
|
||
VALUES($1,$2,$3,$4,$5,$6::jsonb,$7::jsonb,$8,$9::jsonb)`,
|
||
ch.ObjectType, ch.ObjectPK, ch.Action, ch.Actor, ch.Via,
|
||
mustJSON(ch.Diff), snapshotJSON(ch.Snapshot), ch.Source, linksJSON(ch.LinksSnapshot))
|
||
if err != nil {
|
||
fmt.Printf("[audit] 记录变更失败 %s/%s %s: %v\n", ch.ObjectType, ch.ObjectPK, ch.Action, err)
|
||
}
|
||
}
|
||
|
||
type objectChange struct {
|
||
ObjectType string
|
||
ObjectPK string
|
||
Action string // create | update | delete | restore
|
||
Actor string // principal
|
||
Via string // api | mcp | km_extract | batch | propagate | classification_recompute
|
||
Diff propDiff
|
||
Snapshot map[string]interface{} // delete 时的整份属性快照
|
||
Source string
|
||
// LinksSnapshot delete 时该实例的全部关系边快照(issue #4)。
|
||
// 与 deleteObjectCascade 的级联删除对称:删除存边 → restore 可按 restore_edges 重建。
|
||
LinksSnapshot []linkSnapshotRow
|
||
}
|
||
|
||
// linkSnapshotRow 关系边快照行(links 表的可重建子集)。
|
||
type linkSnapshotRow struct {
|
||
LinkType string `json:"link_type"`
|
||
SourceType string `json:"source_type"`
|
||
SourcePK string `json:"source_pk"`
|
||
TargetType string `json:"target_type"`
|
||
TargetPK string `json:"target_pk"`
|
||
Properties map[string]interface{} `json:"properties,omitempty"`
|
||
Source string `json:"source,omitempty"`
|
||
}
|
||
|
||
func snapshotJSON(s map[string]interface{}) string {
|
||
if s == nil {
|
||
return "null"
|
||
}
|
||
return mustJSON(s)
|
||
}
|
||
|
||
func linksJSON(s []linkSnapshotRow) string {
|
||
if len(s) == 0 {
|
||
return "null"
|
||
}
|
||
return mustJSON(s)
|
||
}
|
||
|
||
// viaOf 判定调用来源,区分「平台端手动改」与「MCP 抽取写入」。
|
||
func viaOf(c *gin.Context) string {
|
||
if v := c.GetHeader("X-Via"); v != "" {
|
||
return v
|
||
}
|
||
return "api"
|
||
}
|
||
|
||
// ---------- 对象行读取 ----------
|
||
|
||
type objectRow struct {
|
||
ID int64
|
||
TypeAPIName string
|
||
PK string
|
||
Name string
|
||
Properties map[string]interface{}
|
||
Source string
|
||
Deleted bool
|
||
SameAs string
|
||
SourceTS int64 // 业务源时间戳(毫秒);0=未提供
|
||
// Classification 生效分级(issue #5);空 = 未分级(视为最低)
|
||
Classification string
|
||
}
|
||
|
||
func loadObjectRow(ctx context.Context, p dbExec, typ, pk string) (*objectRow, error) {
|
||
var r objectRow
|
||
var props []byte
|
||
err := p.QueryRow(ctx,
|
||
`SELECT id, type_api_name, pk, name, properties_json, COALESCE(source,''), COALESCE(deleted,false), COALESCE(same_as,''), COALESCE(source_ts,0), COALESCE(classification,'')
|
||
FROM objects WHERE type_api_name=$1 AND pk=$2`, typ, pk).
|
||
Scan(&r.ID, &r.TypeAPIName, &r.PK, &r.Name, &props, &r.Source, &r.Deleted, &r.SameAs, &r.SourceTS, &r.Classification)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
r.Properties = decodeProps(props)
|
||
return &r, nil
|
||
}
|
||
|
||
// ---------- 级联删除 ----------
|
||
|
||
// deleteObjectCascade 删除实例并级联回滚其产生的所有关系边。
|
||
// 幂等:对象不存在时返回 deleted=false,不报错。快照写入 object_changes,可 restore_object 回滚。
|
||
//
|
||
// 血统传播(对齐 Palantir Lineage-Aware Deletion):
|
||
// - propagate=false(默认):只删实例+边,响应回显 source_ref 提示来源(文档/连接),
|
||
// 由调用方决定是否连带清理——治理动作(去重、清测试数据)不应误伤同文档其他实体。
|
||
// - propagate=true:沿 source 找到 KM 文件/连接,把该来源的全部抽取产物+摄取记录一并清理,
|
||
// 相当于「这份文档产出的知识全部遗忘」。
|
||
func deleteObjectCascade(ctx context.Context, a *app, typ, pk, actor, via string, propagate bool) (gin.H, error) {
|
||
row, err := loadObjectRow(ctx, a.pool, typ, pk)
|
||
if err != nil {
|
||
return gin.H{"ok": true, "type": typ, "pk": pk, "deleted": false, "reason": "not_found",
|
||
"message": "实例不存在(幂等:重复删除不报错)"}, nil
|
||
}
|
||
|
||
tx, err := a.pool.Begin(ctx)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer tx.Rollback(ctx)
|
||
|
||
// 级联清理:该实例作为 source 或 target 的所有边
|
||
// ⚠️ 先快照再删(issue #4):restore_object 才能把关系一并恢复,
|
||
// 否则「删一次再回滚」会把实例变成图上的孤岛。
|
||
linkSnap := loadLinkSnapshot(ctx, a.pool, typ, pk)
|
||
var linksDeleted int64
|
||
if tag, e := tx.Exec(ctx,
|
||
`DELETE FROM links WHERE (source_type=$1 AND source_pk=$2) OR (target_type=$1 AND target_pk=$2)`,
|
||
typ, pk); e == nil {
|
||
linksDeleted = tag.RowsAffected()
|
||
}
|
||
|
||
if _, err := tx.Exec(ctx, `DELETE FROM objects WHERE type_api_name=$1 AND pk=$2`, typ, pk); err != nil {
|
||
return nil, err
|
||
}
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
recordChange(ctx, a.pool, objectChange{
|
||
ObjectType: typ, ObjectPK: pk, Action: "delete", Actor: actor, Via: via,
|
||
Snapshot: row.Properties, Source: row.Source, LinksSnapshot: linkSnap,
|
||
})
|
||
|
||
resp := gin.H{
|
||
"ok": true, "type": typ, "pk": pk, "deleted": true,
|
||
"objects_deleted": 1, "links_deleted": linksDeleted,
|
||
"snapshot_saved": true,
|
||
"links_snapshot_saved": len(linkSnap),
|
||
"message": fmt.Sprintf("已删除实例并级联清理 %d 条关系边(可用 restore_object 连同边一起回滚)", linksDeleted),
|
||
}
|
||
|
||
// 血统回显 / 传播
|
||
if row.Source != "" {
|
||
remaining, _ := countObjectsBySource(ctx, a.pool, row.Source)
|
||
if propagate {
|
||
cascadeDeleteSourceCtx(ctx, a.pool, row.Source)
|
||
recordChange(ctx, a.pool, objectChange{
|
||
ObjectType: typ, ObjectPK: pk, Action: "delete", Actor: actor, Via: "propagate",
|
||
Snapshot: map[string]interface{}{"propagated_source": row.Source},
|
||
Source: row.Source,
|
||
})
|
||
resp["propagated"] = true
|
||
resp["message"] = fmt.Sprintf("已删除实例 + %d 条边,并沿血统清理来源 %s 的全部抽取产物(含摄取记录)", linksDeleted, row.Source)
|
||
} else {
|
||
resp["source_ref"] = row.Source
|
||
resp["source_remaining_objects"] = remaining
|
||
if remaining > 0 {
|
||
resp["warning"] = fmt.Sprintf("该实例来自来源 %s,该来源下仍有 %d 个实例。如需整体遗忘请加 propagate=true", row.Source, remaining)
|
||
}
|
||
}
|
||
}
|
||
return resp, nil
|
||
}
|
||
|
||
// countObjectsBySource 统计某 source 溯源下剩余的实例数(0 表示最后一个)。
|
||
func countObjectsBySource(ctx context.Context, p dbExec, source string) (int64, error) {
|
||
var n int64
|
||
err := p.QueryRow(ctx, `SELECT count(*) FROM objects WHERE source=$1`, source).Scan(&n)
|
||
return n, err
|
||
}
|
||
|
||
// loadLinkSnapshot 读取某实例作为 source 或 target 的全部边(含边属性与溯源)。
|
||
// 用于 delete 前存快照,供 restore_object 重建。
|
||
func loadLinkSnapshot(ctx context.Context, p dbExec, typ, pk string) []linkSnapshotRow {
|
||
rows, err := p.Query(ctx,
|
||
`SELECT link_type, source_type, source_pk, target_type, target_pk,
|
||
COALESCE(properties_json,'{}'::jsonb), COALESCE(source,'')
|
||
FROM links WHERE (source_type=$1 AND source_pk=$2) OR (target_type=$1 AND target_pk=$2)
|
||
ORDER BY id`, typ, pk)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
defer rows.Close()
|
||
out := []linkSnapshotRow{}
|
||
for rows.Next() {
|
||
var r linkSnapshotRow
|
||
var props []byte
|
||
if err := rows.Scan(&r.LinkType, &r.SourceType, &r.SourcePK, &r.TargetType, &r.TargetPK, &props, &r.Source); err != nil {
|
||
continue
|
||
}
|
||
r.Properties = decodeProps(props)
|
||
out = append(out, r)
|
||
}
|
||
return out
|
||
}
|
||
|
||
// restoreObject 从最近一条 delete 快照恢复实例。
|
||
//
|
||
// issue #4(2026-09-07):默认**连同关系边一起恢复**(restore_edges 默认 true)。
|
||
// 删除是级联的(实例 + 它的全部边),回滚却只恢复属性 —— 这是不对称的,
|
||
// 一次误删 + 回滚会留下一个在图上游离的孤岛实例。现在删前存边快照,回滚时重建。
|
||
//
|
||
// 边重建规则:
|
||
// - 边两端实例必须都存在,否则跳过(对端可能也被删了,凭空建边会造出悬空引用);
|
||
// - 已存在的边不重复建(ON CONFLICT DO NOTHING),重复回滚幂等;
|
||
// - restore_edges=false 可显式只要属性回滚。
|
||
func (a *app) restoreObject(c *gin.Context) {
|
||
typ := c.Param("type")
|
||
pk := c.Param("pk")
|
||
authz, _ := a.authzOf(c)
|
||
if !authz.CanWrite("write", typ, pk) {
|
||
a.abortForbidden(c, "无写权限")
|
||
return
|
||
}
|
||
restoreEdges := c.Query("restore_edges") != "false"
|
||
var body struct {
|
||
RestoreEdges *bool `json:"restore_edges"`
|
||
}
|
||
if c.ShouldBindJSON(&body) == nil && body.RestoreEdges != nil {
|
||
restoreEdges = *body.RestoreEdges
|
||
}
|
||
var props, links []byte
|
||
var source string
|
||
err := a.pool.QueryRow(c,
|
||
`SELECT snapshot, COALESCE(links_snapshot,'[]'::jsonb), COALESCE(source,'') FROM object_changes
|
||
WHERE object_type=$1 AND object_pk=$2 AND action='delete' AND snapshot IS NOT NULL
|
||
ORDER BY id DESC LIMIT 1`, typ, pk).Scan(&props, &links, &source)
|
||
if err != nil {
|
||
c.JSON(404, gin.H{"error": "没有可回滚的删除快照"})
|
||
return
|
||
}
|
||
var oid int64
|
||
if err := a.pool.QueryRow(c, `SELECT id FROM object_types WHERE api_name=$1`, typ).Scan(&oid); err != nil {
|
||
c.JSON(400, gin.H{"error": "对象类型不存在: " + typ})
|
||
return
|
||
}
|
||
restored := decodeProps(props)
|
||
nm := resolveDisplayName(a.titlePropOf(c, typ), restored)
|
||
// 分级物化(issue #5):回滚出来的实例同样要带上生效分级,否则一回滚就绕过分级过滤
|
||
cls := materializeClassification(a.classConfigOf(c, typ), restored, "")
|
||
if _, err := a.pool.Exec(c,
|
||
`INSERT INTO objects(object_type_id,type_api_name,pk,name,properties_json,source,deleted,classification)
|
||
VALUES($1,$2,$3,$4,$5::jsonb,$6,false,$7)
|
||
ON CONFLICT(object_type_id,pk) DO UPDATE SET
|
||
name=EXCLUDED.name, properties_json=EXCLUDED.properties_json,
|
||
source=COALESCE(NULLIF(EXCLUDED.source,''), objects.source),
|
||
classification=EXCLUDED.classification,
|
||
deleted=false, updated_at=now()`,
|
||
oid, typ, pk, nm, mustJSON(restored), source, cls); err != nil {
|
||
c.JSON(500, gin.H{"error": err.Error()})
|
||
return
|
||
}
|
||
|
||
// 关系边重建(issue #4)
|
||
edgesCreated, edgesSkipped, edgesDangling := 0, 0, 0
|
||
var snap []linkSnapshotRow
|
||
if restoreEdges && len(links) > 0 {
|
||
_ = json.Unmarshal(links, &snap)
|
||
for _, e := range snap {
|
||
// 两端都还在才建边,避免造出悬空引用
|
||
var n int
|
||
if err := a.pool.QueryRow(c,
|
||
`SELECT count(*) FROM objects WHERE (type_api_name=$1 AND pk=$2) OR (type_api_name=$3 AND pk=$4)`,
|
||
e.SourceType, e.SourcePK, e.TargetType, e.TargetPK).Scan(&n); err != nil || n < 2 {
|
||
edgesDangling++
|
||
continue
|
||
}
|
||
props := "{}"
|
||
if len(e.Properties) > 0 {
|
||
props = mustJSON(e.Properties)
|
||
}
|
||
tag, err := a.pool.Exec(c,
|
||
`INSERT INTO links(link_type,source_type,source_pk,target_type,target_pk,properties_json,source)
|
||
VALUES($1,$2,$3,$4,$5,$6::jsonb,$7) ON CONFLICT DO NOTHING`,
|
||
e.LinkType, e.SourceType, e.SourcePK, e.TargetType, e.TargetPK, props, e.Source)
|
||
if err != nil {
|
||
edgesSkipped++
|
||
continue
|
||
}
|
||
if tag.RowsAffected() > 0 {
|
||
edgesCreated++
|
||
} else {
|
||
edgesSkipped++
|
||
}
|
||
}
|
||
}
|
||
|
||
recordChange(c, a.pool, objectChange{
|
||
ObjectType: typ, ObjectPK: pk, Action: "restore",
|
||
Actor: a.principalFromRequest(c), Via: viaOf(c), Snapshot: restored, Source: source,
|
||
LinksSnapshot: snap,
|
||
})
|
||
msg := fmt.Sprintf("已从删除快照恢复实例(%d 个属性)", len(restored))
|
||
if restoreEdges {
|
||
msg = fmt.Sprintf("已从删除快照恢复实例并重建 %d 条关系边(跳过 %d 条已存在,%d 条因对端缺失未建)",
|
||
edgesCreated, edgesSkipped, edgesDangling)
|
||
}
|
||
c.JSON(200, gin.H{"ok": true, "type": typ, "pk": pk, "restored_fields": len(restored),
|
||
"restore_edges": restoreEdges, "links_snapshot": len(snap),
|
||
"edges_created": edgesCreated, "edges_skipped": edgesSkipped, "edges_dangling": edgesDangling,
|
||
"classification": cls,
|
||
"message": msg})
|
||
}
|
||
|
||
// ---------- 变更历史 ----------
|
||
|
||
func (a *app) getObjectHistory(c *gin.Context) {
|
||
typ := c.Param("type")
|
||
pk := c.Param("pk")
|
||
limit := 50
|
||
rows, err := a.pool.Query(c,
|
||
`SELECT id, created_at, action, COALESCE(actor,''), COALESCE(via,''), diff, COALESCE(source,'')
|
||
FROM object_changes WHERE object_type=$1 AND object_pk=$2 ORDER BY id DESC LIMIT $3`,
|
||
typ, pk, limit)
|
||
if err != nil {
|
||
c.JSON(500, gin.H{"error": err.Error()})
|
||
return
|
||
}
|
||
defer rows.Close()
|
||
type item struct {
|
||
ID int64 `json:"id"`
|
||
CreatedAt time.Time `json:"created_at"`
|
||
Action string `json:"action"`
|
||
Actor string `json:"actor"`
|
||
Via string `json:"via"`
|
||
Diff interface{} `json:"diff"`
|
||
Source string `json:"source"`
|
||
}
|
||
out := []item{}
|
||
for rows.Next() {
|
||
var it item
|
||
var diff []byte
|
||
if err := rows.Scan(&it.ID, &it.CreatedAt, &it.Action, &it.Actor, &it.Via, &diff, &it.Source); err != nil {
|
||
continue
|
||
}
|
||
it.Diff = jsonRaw(diff)
|
||
out = append(out, it)
|
||
}
|
||
c.JSON(200, gin.H{"type": typ, "pk": pk, "items": out, "count": len(out)})
|
||
}
|
||
|
||
// listRecentChanges 全库最近变更(管理端审计视图用)。
|
||
func (a *app) listRecentChanges(c *gin.Context) {
|
||
limit := 100
|
||
if l := c.Query("limit"); l != "" {
|
||
fmt.Sscan(l, &limit)
|
||
}
|
||
if limit <= 0 || limit > 500 {
|
||
limit = 100
|
||
}
|
||
args := []interface{}{limit}
|
||
where := "1=1"
|
||
if t := c.Query("type"); t != "" {
|
||
args = append(args, t)
|
||
where += fmt.Sprintf(" AND object_type=$%d", len(args))
|
||
}
|
||
if ac := c.Query("action"); ac != "" {
|
||
args = append(args, ac)
|
||
where += fmt.Sprintf(" AND action=$%d", len(args))
|
||
}
|
||
if at := c.Query("actor"); at != "" {
|
||
args = append(args, at)
|
||
where += fmt.Sprintf(" AND actor=$%d", len(args))
|
||
}
|
||
// 按属性查变更(issue #5-⑨):分级属性的升降、某敏感字段的每次改动,
|
||
// 都落在这张表的 diff 里,给一个入口就能审计「谁把什么从哪级改到哪级」。
|
||
propFilter := c.Query("property")
|
||
if propFilter != "" {
|
||
args = append(args, propFilter)
|
||
where += fmt.Sprintf(` AND (diff->'changed' ? $%d OR diff->'added' ? $%d OR diff->'removed' ? $%d)`,
|
||
len(args), len(args), len(args))
|
||
}
|
||
rows, err := a.pool.Query(c,
|
||
`SELECT id, created_at, object_type, object_pk, action, COALESCE(actor,''), COALESCE(via,''), diff
|
||
FROM object_changes WHERE `+where+` ORDER BY id DESC LIMIT $1`, args...)
|
||
if err != nil {
|
||
c.JSON(500, gin.H{"error": err.Error()})
|
||
return
|
||
}
|
||
defer rows.Close()
|
||
type item struct {
|
||
ID int64 `json:"id"`
|
||
CreatedAt time.Time `json:"created_at"`
|
||
ObjectType string `json:"object_type"`
|
||
ObjectPK string `json:"object_pk"`
|
||
Action string `json:"action"`
|
||
Actor string `json:"actor"`
|
||
Via string `json:"via"`
|
||
Diff interface{} `json:"diff"`
|
||
}
|
||
out := []item{}
|
||
for rows.Next() {
|
||
var it item
|
||
var diff []byte
|
||
if err := rows.Scan(&it.ID, &it.CreatedAt, &it.ObjectType, &it.ObjectPK, &it.Action, &it.Actor, &it.Via, &diff); err != nil {
|
||
continue
|
||
}
|
||
it.Diff = jsonRaw(diff)
|
||
out = append(out, it)
|
||
}
|
||
res := gin.H{"items": out, "count": len(out)}
|
||
if propFilter != "" {
|
||
res["property_filter"] = propFilter
|
||
}
|
||
c.JSON(200, res)
|
||
}
|
||
|
||
// ---------- 批量写入 ----------
|
||
|
||
type batchObjectItem struct {
|
||
Type string `json:"type"`
|
||
PK string `json:"pk"`
|
||
Properties map[string]interface{} `json:"properties"`
|
||
Source string `json:"source"`
|
||
SourceTS int64 `json:"source_ts"` // 业务源时间戳(毫秒);比已有值旧则该条 stale_write 失败
|
||
SameAs string `json:"same_as"`
|
||
Replace bool `json:"replace"`
|
||
Deleted *bool `json:"deleted"`
|
||
// 数据分级(issue #5):min_classification = 分级下界(如 KM 源文件级);
|
||
// validate_enum=false 可跳过该条的枚举校验(历史脏数据导入用)
|
||
MinClassification string `json:"min_classification"`
|
||
ValidateEnum *bool `json:"validate_enum"`
|
||
}
|
||
|
||
type batchObjectsInput struct {
|
||
Items []batchObjectItem `json:"items"`
|
||
StopOnError bool `json:"stop_on_error"`
|
||
}
|
||
|
||
// batchCreateObjects 单事务批量写入实例,返回逐条明细。
|
||
// 设计:默认全部成功才提交(stop_on_error=true 时遇到第一条错误即回滚)。
|
||
func (a *app) batchCreateObjects(c *gin.Context) {
|
||
var in batchObjectsInput
|
||
if err := c.ShouldBindJSON(&in); err != nil {
|
||
c.JSON(400, gin.H{"error": err.Error()})
|
||
return
|
||
}
|
||
if len(in.Items) == 0 {
|
||
c.JSON(400, gin.H{"error": "items 必填"})
|
||
return
|
||
}
|
||
if len(in.Items) > 500 {
|
||
c.JSON(400, gin.H{"error": "单次批量上限 500 条"})
|
||
return
|
||
}
|
||
authz, _ := a.authzOf(c)
|
||
actor := a.principalFromRequest(c)
|
||
via := viaOf(c)
|
||
if via == "api" {
|
||
via = "batch"
|
||
}
|
||
|
||
tx, err := a.pool.Begin(c)
|
||
if err != nil {
|
||
c.JSON(500, gin.H{"error": err.Error()})
|
||
return
|
||
}
|
||
defer tx.Rollback(c)
|
||
|
||
type itemOut struct {
|
||
Index int `json:"index"`
|
||
Type string `json:"type"`
|
||
PK string `json:"pk"`
|
||
Status string `json:"status"` // created | updated | unchanged | failed
|
||
Error string `json:"error,omitempty"`
|
||
Added []string `json:"added,omitempty"`
|
||
Changed []string `json:"changed,omitempty"`
|
||
Removed []string `json:"removed,omitempty"`
|
||
}
|
||
results := make([]itemOut, 0, len(in.Items))
|
||
created, updated, failed := 0, 0, 0
|
||
|
||
for i, it := range in.Items {
|
||
if it.Type == "" || it.PK == "" {
|
||
failed++
|
||
results = append(results, itemOut{Index: i, Type: it.Type, PK: it.PK, Status: "failed", Error: "type 与 pk 必填"})
|
||
if in.StopOnError {
|
||
c.JSON(400, gin.H{"error": fmt.Sprintf("第 %d 条参数错误,已回滚", i), "results": results})
|
||
return
|
||
}
|
||
continue
|
||
}
|
||
if !authz.CanWrite("write", it.Type, it.PK) {
|
||
failed++
|
||
results = append(results, itemOut{Index: i, Type: it.Type, PK: it.PK, Status: "failed", Error: "无写权限"})
|
||
if in.StopOnError {
|
||
c.JSON(403, gin.H{"error": fmt.Sprintf("第 %d 条无写权限,已回滚", i), "results": results})
|
||
return
|
||
}
|
||
continue
|
||
}
|
||
res, err := applyObjectUpsert(c, tx, a, objectUpsert{
|
||
Type: it.Type, PK: it.PK, Properties: it.Properties, Source: it.Source,
|
||
SameAs: it.SameAs, Replace: it.Replace, Deleted: it.Deleted, SourceTS: it.SourceTS,
|
||
Actor: actor, Via: via,
|
||
MinClassification: it.MinClassification,
|
||
SkipEnumCheck: it.ValidateEnum != nil && !*it.ValidateEnum,
|
||
})
|
||
if err != nil {
|
||
failed++
|
||
status := "failed"
|
||
if errors.Is(err, ErrStaleWrite) {
|
||
status = "stale_skipped"
|
||
}
|
||
results = append(results, itemOut{Index: i, Type: it.Type, PK: it.PK, Status: status, Error: err.Error()})
|
||
if in.StopOnError {
|
||
c.JSON(500, gin.H{"error": fmt.Sprintf("第 %d 条失败,已回滚:%v", i, err), "results": results})
|
||
return
|
||
}
|
||
continue
|
||
}
|
||
status := "updated"
|
||
if res.Created {
|
||
created++
|
||
status = "created"
|
||
} else if res.Diff.empty() {
|
||
status = "unchanged"
|
||
} else {
|
||
updated++
|
||
}
|
||
results = append(results, itemOut{
|
||
Index: i, Type: it.Type, PK: it.PK, Status: status,
|
||
Added: res.Diff.Added, Changed: keysOf(res.Diff.Changed), Removed: res.Diff.Removed,
|
||
})
|
||
}
|
||
|
||
if err := tx.Commit(c); err != nil {
|
||
c.JSON(500, gin.H{"error": "提交失败: " + err.Error()})
|
||
return
|
||
}
|
||
// 事务提交后再补显示名(依赖 title_property,放在事务外避免长事务)
|
||
for _, it := range in.Items {
|
||
if len(it.Properties) > 0 {
|
||
a.refreshDisplayName(c, it.Type, it.PK, it.Properties)
|
||
}
|
||
}
|
||
invalidateTitlePropCache()
|
||
c.JSON(200, gin.H{
|
||
"ok": true, "total": len(in.Items),
|
||
"created": created, "updated": updated, "failed": failed,
|
||
"results": results,
|
||
})
|
||
}
|
||
|
||
func keysOf(m map[string]map[string]interface{}) []string {
|
||
out := make([]string, 0, len(m))
|
||
for k := range m {
|
||
out = append(out, k)
|
||
}
|
||
sort.Strings(out)
|
||
return out
|
||
}
|
||
|
||
type batchLinkItem struct {
|
||
Type string `json:"type"`
|
||
SourceType string `json:"source_type"`
|
||
SourcePK string `json:"source_pk"`
|
||
TargetType string `json:"target_type"`
|
||
TargetPK string `json:"target_pk"`
|
||
Properties map[string]interface{} `json:"properties"`
|
||
Source string `json:"source"`
|
||
}
|
||
|
||
func (a *app) batchCreateLinks(c *gin.Context) {
|
||
var in struct {
|
||
Items []batchLinkItem `json:"items"`
|
||
StopOnError bool `json:"stop_on_error"`
|
||
}
|
||
if err := c.ShouldBindJSON(&in); err != nil || len(in.Items) == 0 {
|
||
c.JSON(400, gin.H{"error": "items 必填"})
|
||
return
|
||
}
|
||
if len(in.Items) > 1000 {
|
||
c.JSON(400, gin.H{"error": "单次批量上限 1000 条"})
|
||
return
|
||
}
|
||
authz, _ := a.authzOf(c)
|
||
tx, err := a.pool.Begin(c)
|
||
if err != nil {
|
||
c.JSON(500, gin.H{"error": err.Error()})
|
||
return
|
||
}
|
||
defer tx.Rollback(c)
|
||
|
||
type itemOut struct {
|
||
Index int `json:"index"`
|
||
Type string `json:"type"`
|
||
Status string `json:"status"` // created | duplicated | failed
|
||
Error string `json:"error,omitempty"`
|
||
}
|
||
results := make([]itemOut, 0, len(in.Items))
|
||
created, dup, failed := 0, 0, 0
|
||
|
||
for i, it := range in.Items {
|
||
if it.Type == "" || it.SourceType == "" || it.SourcePK == "" || it.TargetType == "" || it.TargetPK == "" {
|
||
failed++
|
||
results = append(results, itemOut{Index: i, Type: it.Type, Status: "failed", Error: "type/source/target 必填"})
|
||
if in.StopOnError {
|
||
c.JSON(400, gin.H{"error": fmt.Sprintf("第 %d 条参数错误,已回滚", i), "results": results})
|
||
return
|
||
}
|
||
continue
|
||
}
|
||
if !authz.CanWrite("write", it.SourceType, it.SourcePK) || !authz.CanWrite("write", it.TargetType, it.TargetPK) {
|
||
failed++
|
||
results = append(results, itemOut{Index: i, Type: it.Type, Status: "failed", Error: "无写权限"})
|
||
if in.StopOnError {
|
||
c.JSON(403, gin.H{"error": fmt.Sprintf("第 %d 条无写权限,已回滚", i), "results": results})
|
||
return
|
||
}
|
||
continue
|
||
}
|
||
var cnt int
|
||
if err := tx.QueryRow(c, `SELECT count(*) FROM link_types WHERE api_name=$1`, it.Type).Scan(&cnt); err != nil || cnt == 0 {
|
||
failed++
|
||
results = append(results, itemOut{Index: i, Type: it.Type, Status: "failed", Error: "链接类型不存在: " + it.Type})
|
||
if in.StopOnError {
|
||
c.JSON(400, gin.H{"error": fmt.Sprintf("第 %d 条链接类型不存在,已回滚", i), "results": results})
|
||
return
|
||
}
|
||
continue
|
||
}
|
||
props := "{}"
|
||
if len(it.Properties) > 0 {
|
||
props = mustJSON(it.Properties)
|
||
}
|
||
tag, err := tx.Exec(c,
|
||
`INSERT INTO links(link_type,source_type,source_pk,target_type,target_pk,properties_json,source)
|
||
VALUES($1,$2,$3,$4,$5,$6::jsonb,$7) ON CONFLICT DO NOTHING`,
|
||
it.Type, it.SourceType, it.SourcePK, it.TargetType, it.TargetPK, props, it.Source)
|
||
if err != nil {
|
||
failed++
|
||
results = append(results, itemOut{Index: i, Type: it.Type, Status: "failed", Error: err.Error()})
|
||
if in.StopOnError {
|
||
c.JSON(500, gin.H{"error": fmt.Sprintf("第 %d 条失败,已回滚:%v", i, err), "results": results})
|
||
return
|
||
}
|
||
continue
|
||
}
|
||
if tag.RowsAffected() == 0 {
|
||
dup++
|
||
results = append(results, itemOut{Index: i, Type: it.Type, Status: "duplicated"})
|
||
continue
|
||
}
|
||
created++
|
||
results = append(results, itemOut{Index: i, Type: it.Type, Status: "created"})
|
||
}
|
||
if err := tx.Commit(c); err != nil {
|
||
c.JSON(500, gin.H{"error": "提交失败: " + err.Error()})
|
||
return
|
||
}
|
||
c.JSON(200, gin.H{
|
||
"ok": true, "total": len(in.Items),
|
||
"created": created, "duplicated": dup, "failed": failed, "results": results,
|
||
})
|
||
}
|
||
|
||
// ---------- 重复实体检测 ----------
|
||
|
||
// normalizeName 归一化名称用于相似比较:去空白/标点、统一小写。
|
||
func normalizeName(s string) string {
|
||
s = strings.ToLower(strings.TrimSpace(s))
|
||
repl := strings.NewReplacer(
|
||
" ", "", "\t", "", " ", "",
|
||
"-", "", "_", "", ".", "", "·", "",
|
||
"(", "", ")", "", "(", "", ")", "",
|
||
"、", "", ",", "", ",", "", "。", "",
|
||
":", "", ":", "", "/", "", "\\", "",
|
||
)
|
||
return repl.Replace(s)
|
||
}
|
||
|
||
// similarity 简易相似度:归一化后相等=1,包含=按长度比,否则用编辑距离比。
|
||
func similarity(a, b string) float64 {
|
||
na, nb := normalizeName(a), normalizeName(b)
|
||
if na == "" || nb == "" {
|
||
return 0
|
||
}
|
||
if na == nb {
|
||
return 1
|
||
}
|
||
if strings.Contains(na, nb) || strings.Contains(nb, na) {
|
||
short, long := len(na), len(nb)
|
||
if short > long {
|
||
short, long = long, short
|
||
}
|
||
return 0.7 + 0.29*float64(short)/float64(long)
|
||
}
|
||
d := levenshtein(na, nb)
|
||
long := len(na)
|
||
if len(nb) > long {
|
||
long = len(nb)
|
||
}
|
||
if long == 0 {
|
||
return 0
|
||
}
|
||
return 1 - float64(d)/float64(long)
|
||
}
|
||
|
||
type duplicateHit struct {
|
||
Type string `json:"type"`
|
||
PK string `json:"pk"`
|
||
Name string `json:"name"`
|
||
Similarity float64 `json:"similarity"`
|
||
Reason string `json:"reason"`
|
||
}
|
||
|
||
// findDuplicatesOf 在指定类型内查找与给定名称相似的实例。
|
||
func (a *app) findDuplicatesOf(ctx context.Context, db dbExec, typ, name, excludePK string, threshold float64, limit int) ([]duplicateHit, error) {
|
||
if threshold <= 0 {
|
||
threshold = 0.85
|
||
}
|
||
if limit <= 0 {
|
||
limit = 10
|
||
}
|
||
rows, err := db.Query(ctx,
|
||
`SELECT pk, name, COALESCE(same_as,''), COALESCE(deleted,false) FROM objects WHERE type_api_name=$1`, typ)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
hits := []duplicateHit{}
|
||
for rows.Next() {
|
||
var pk, nm, sameAs string
|
||
var deleted bool
|
||
if err := rows.Scan(&pk, &nm, &sameAs, &deleted); err != nil {
|
||
continue
|
||
}
|
||
if pk == excludePK {
|
||
continue
|
||
}
|
||
sim := similarity(name, nm)
|
||
if sim < threshold {
|
||
continue
|
||
}
|
||
reason := "名称相似"
|
||
if normalizeName(name) == normalizeName(nm) {
|
||
reason = "归一化后同名"
|
||
}
|
||
if sameAs != "" {
|
||
reason += "(已是别名,canonical=" + sameAs + ")"
|
||
}
|
||
if deleted {
|
||
reason += "(已软删除)"
|
||
}
|
||
hits = append(hits, duplicateHit{Type: typ, PK: pk, Name: nm, Similarity: sim, Reason: reason})
|
||
}
|
||
sort.Slice(hits, func(i, j int) bool { return hits[i].Similarity > hits[j].Similarity })
|
||
if len(hits) > limit {
|
||
hits = hits[:limit]
|
||
}
|
||
return hits, nil
|
||
}
|
||
|
||
// findDuplicates 扫描某类型(或全库)内的重复实体分组。
|
||
func (a *app) findDuplicates(c *gin.Context) {
|
||
typ := c.Query("type")
|
||
threshold := 0.9
|
||
if v := c.Query("threshold"); v != "" {
|
||
fmt.Sscan(v, &threshold)
|
||
}
|
||
if typ == "" {
|
||
c.JSON(200, gin.H{"groups": []interface{}{}, "count": 0,
|
||
"hint": "请传 type 限定对象类型(全库两两比对代价过高)"})
|
||
return
|
||
}
|
||
rows, err := a.pool.Query(c,
|
||
`SELECT pk, name, COALESCE(same_as,'') FROM objects
|
||
WHERE type_api_name=$1 AND COALESCE(name,'') <> '' ORDER BY id`, typ)
|
||
if err != nil {
|
||
c.JSON(500, gin.H{"error": err.Error()})
|
||
return
|
||
}
|
||
defer rows.Close()
|
||
type cand struct {
|
||
pk, name, sameAs string
|
||
}
|
||
var list []cand
|
||
for rows.Next() {
|
||
var x cand
|
||
if err := rows.Scan(&x.pk, &x.name, &x.sameAs); err == nil {
|
||
list = append(list, x)
|
||
}
|
||
}
|
||
type group struct {
|
||
Members []duplicateHit `json:"members"`
|
||
Reason string `json:"reason"`
|
||
}
|
||
var groups []group
|
||
used := map[string]bool{}
|
||
for i := 0; i < len(list); i++ {
|
||
if used[list[i].pk] {
|
||
continue
|
||
}
|
||
mem := []duplicateHit{{Type: typ, PK: list[i].pk, Name: list[i].name, Similarity: 1}}
|
||
for j := i + 1; j < len(list); j++ {
|
||
if used[list[j].pk] {
|
||
continue
|
||
}
|
||
if s := similarity(list[i].name, list[j].name); s >= threshold {
|
||
mem = append(mem, duplicateHit{Type: typ, PK: list[j].pk, Name: list[j].name, Similarity: s})
|
||
used[list[j].pk] = true
|
||
}
|
||
}
|
||
if len(mem) > 1 {
|
||
used[list[i].pk] = true
|
||
groups = append(groups, group{Members: mem,
|
||
Reason: "同一类型内名称高度相似,建议用 same_as 归并或物理删除"})
|
||
}
|
||
}
|
||
if groups == nil {
|
||
groups = []group{}
|
||
}
|
||
c.JSON(200, gin.H{"type": typ, "threshold": threshold, "groups": groups, "count": len(groups)})
|
||
}
|