含旁路创建位点回填治愈(seed/merge/AI 摄取):migrate 的三条 UPDATE 抽成 backfillMissingIRI,seedIfEmpty 灌入后、mergeExecute 提交后、ai.ApplySuggestion 落库后(runIngest 单点,覆盖 AI 摄取/km 抽取/连接批量抽取)各调一次治愈空值。 Co-Authored-By: Claude Opus 4.8 <[email protected]>
667 lines
31 KiB
Go
667 lines
31 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"log"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/jackc/pgx/v5/pgxpool"
|
||
)
|
||
|
||
var pool dbRunner
|
||
|
||
// initDB 连接 Postgres 并执行 schema 迁移。
|
||
func initDB(ctx context.Context, dsn string) (*pgxpool.Pool, error) {
|
||
cfg, err := pgxpool.ParseConfig(dsn)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("解析 DSN 失败: %w", err)
|
||
}
|
||
cfg.MaxConns = 20
|
||
cfg.MaxConnIdleTime = 5 * time.Minute
|
||
|
||
p, err := pgxpool.NewWithConfig(ctx, cfg)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("连接 Postgres 失败: %w", err)
|
||
}
|
||
if err := p.Ping(ctx); err != nil {
|
||
return nil, fmt.Errorf("Postgres 不可达: %w", err)
|
||
}
|
||
if err := migrate(ctx, p); err != nil {
|
||
return nil, fmt.Errorf("schema 迁移失败: %w", err)
|
||
}
|
||
return p, nil
|
||
}
|
||
|
||
func migrate(ctx context.Context, p dbExec) error {
|
||
ddl := `
|
||
CREATE TABLE IF NOT EXISTS object_types (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
api_name TEXT UNIQUE NOT NULL,
|
||
display_name TEXT NOT NULL,
|
||
description TEXT DEFAULT '',
|
||
icon TEXT DEFAULT '',
|
||
group_name TEXT DEFAULT '未分类',
|
||
-- Palantir Ontology 对标(P1-1 2026-08-24):
|
||
-- title_property 列表/详情头部主标题对应的 property.api_name
|
||
-- subtitle_property 副标题对应的 property.api_name
|
||
-- color UI 标识色(如 #1677ff)
|
||
-- primary_key 该对象类型的主键字段(默认 'pk' 列)
|
||
title_property TEXT DEFAULT '',
|
||
subtitle_property TEXT DEFAULT '',
|
||
color TEXT DEFAULT '',
|
||
primary_key TEXT DEFAULT 'pk',
|
||
created_at TIMESTAMPTZ DEFAULT now(),
|
||
updated_at TIMESTAMPTZ DEFAULT now()
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS properties (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
object_type_id BIGINT NOT NULL REFERENCES object_types(id) ON DELETE CASCADE,
|
||
api_name TEXT NOT NULL,
|
||
display_name TEXT NOT NULL,
|
||
data_type TEXT NOT NULL DEFAULT 'string',
|
||
required BOOLEAN DEFAULT FALSE,
|
||
enum_values JSONB DEFAULT '[]'::jsonb,
|
||
description TEXT DEFAULT '',
|
||
-- Palantir Ontology 对标(P1-2 2026-08-24):
|
||
-- format 字段格式提示(date: YYYY-MM-DD / number: 0.00 / string: email/url/regex)
|
||
-- default_value 默认值(JSONB 任意类型)
|
||
-- indexable 是否可被关键字检索(properties_json::text ILIKE 路径)
|
||
-- vector_index 是否向量索引(语义检索前置,配合 P2 Materialization 落地)
|
||
-- backing_column 该属性底层来源列(物化层用,P2 准备)
|
||
format TEXT DEFAULT '',
|
||
default_value JSONB,
|
||
indexable BOOLEAN DEFAULT FALSE,
|
||
vector_index BOOLEAN DEFAULT FALSE,
|
||
backing_column TEXT DEFAULT '',
|
||
UNIQUE(object_type_id, api_name)
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS link_types (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
api_name TEXT UNIQUE NOT NULL,
|
||
display_name TEXT NOT NULL,
|
||
source_type TEXT NOT NULL,
|
||
target_type TEXT NOT NULL,
|
||
cardinality TEXT DEFAULT 'many_to_many',
|
||
description TEXT DEFAULT '',
|
||
-- Palantir Ontology 对标(P1-3 2026-08-24):
|
||
-- directional true 表示单向边(source→target),false 表示双向(默认)
|
||
-- source_property source 端 key(可选,通常指 objects.pk 的字段名,默认 pk)
|
||
-- target_property target 端 key(可选)
|
||
directional BOOLEAN DEFAULT FALSE,
|
||
source_property TEXT DEFAULT '',
|
||
target_property TEXT DEFAULT ''
|
||
);
|
||
|
||
-- 链接本身可以带属性(如"雇佣关系"有 start_date、role、contract_type)
|
||
CREATE TABLE IF NOT EXISTS link_properties (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
link_type_id BIGINT NOT NULL REFERENCES link_types(id) ON DELETE CASCADE,
|
||
api_name TEXT NOT NULL,
|
||
display_name TEXT NOT NULL,
|
||
data_type TEXT NOT NULL DEFAULT 'string',
|
||
required BOOLEAN DEFAULT FALSE,
|
||
enum_values JSONB DEFAULT '[]'::jsonb,
|
||
description TEXT DEFAULT '',
|
||
UNIQUE(link_type_id, api_name)
|
||
);
|
||
|
||
-- 函数类型(Kinetic 层 ⑤Function):可被 Action / Function / MCP 调用的纯计算或派生
|
||
-- 必须在 action_types 之前建(action_types.impl_function_id 是 FK 引用 function_types.id)
|
||
CREATE TABLE IF NOT EXISTS function_types (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
api_name TEXT UNIQUE NOT NULL,
|
||
display_name TEXT NOT NULL,
|
||
description TEXT DEFAULT '',
|
||
inputs JSONB DEFAULT '[]'::jsonb,
|
||
outputs JSONB DEFAULT '[]'::jsonb,
|
||
impl_kind TEXT DEFAULT 'builtin',
|
||
impl_ref TEXT DEFAULT '',
|
||
is_pure BOOLEAN DEFAULT TRUE,
|
||
builtin_fn TEXT DEFAULT '',
|
||
-- 提示调度(Palantir Ontology 对标 2026-08-24):
|
||
-- when_to_suggest 触发上下文标签,JSONB 数组
|
||
-- llm_tool_description 给 LLM 看的自然语言描述
|
||
when_to_suggest JSONB DEFAULT '[]'::jsonb,
|
||
llm_tool_description TEXT DEFAULT '',
|
||
created_at TIMESTAMPTZ DEFAULT now(),
|
||
updated_at TIMESTAMPTZ DEFAULT now()
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS action_types (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
api_name TEXT UNIQUE NOT NULL,
|
||
display_name TEXT NOT NULL,
|
||
description TEXT DEFAULT '',
|
||
-- params 形式约定:JSONB 数组,每条 = {name, type, required, default, enum, validation, description}
|
||
-- type 取值:string / integer / number / boolean / date / datetime / enum / object_ref
|
||
params JSONB DEFAULT '[]'::jsonb,
|
||
-- 提交条件:执行前校验;key 可包含 object_status_in (string[]) / required_grant / required_clearance 等
|
||
submission_criteria JSONB DEFAULT '{}'::jsonb,
|
||
-- 副作用:执行后触发;key 可包含 webhooks (object[]) / cascade_actions (object[]) / notifications (object[])
|
||
side_effects JSONB DEFAULT '{}'::jsonb,
|
||
-- 函数绑定 (FBA):非空时,Action 执行链路完全由该 Function 实现,不再走声明式 params → properties 改写
|
||
impl_function_id BIGINT REFERENCES function_types(id) ON DELETE SET NULL,
|
||
-- 返回类型描述(string / object_ref / array 等,便于前端 UI 提示)
|
||
return_type TEXT DEFAULT '',
|
||
-- 提示调度(Palantir Ontology 对标 2026-08-24):
|
||
when_to_suggest JSONB DEFAULT '[]'::jsonb,
|
||
llm_tool_description TEXT DEFAULT ''
|
||
);
|
||
|
||
-- 接口类型(Semantic 层 ⑦Interface):声明式接口契约(API/MCP tool 签名 + 授权)
|
||
CREATE TABLE IF NOT EXISTS interface_types (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
api_name TEXT UNIQUE NOT NULL,
|
||
display_name TEXT NOT NULL,
|
||
description TEXT DEFAULT '',
|
||
protocol TEXT DEFAULT 'rest',
|
||
method TEXT DEFAULT 'GET',
|
||
path TEXT DEFAULT '',
|
||
inputs JSONB DEFAULT '[]'::jsonb,
|
||
outputs JSONB DEFAULT '{}'::jsonb,
|
||
requires_auth BOOLEAN DEFAULT TRUE,
|
||
expose_to_mcp BOOLEAN DEFAULT FALSE,
|
||
backing_ref TEXT DEFAULT '',
|
||
builtin BOOLEAN DEFAULT FALSE,
|
||
-- Palantir Ontology 对标(P1-4 2026-08-24):
|
||
-- shared_properties 声明该 Interface "强制" 共享的 properties(api_name 数组)
|
||
-- parent_interface_id 父 Interface(多态继承)
|
||
shared_properties JSONB DEFAULT '[]'::jsonb,
|
||
parent_interface_id BIGINT REFERENCES interface_types(id) ON DELETE SET NULL,
|
||
created_at TIMESTAMPTZ DEFAULT now(),
|
||
updated_at TIMESTAMPTZ DEFAULT now()
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS objects (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
object_type_id BIGINT NOT NULL REFERENCES object_types(id) ON DELETE CASCADE,
|
||
type_api_name TEXT NOT NULL,
|
||
pk TEXT NOT NULL,
|
||
name TEXT DEFAULT '',
|
||
properties_json JSONB DEFAULT '{}'::jsonb,
|
||
created_at TIMESTAMPTZ DEFAULT now(),
|
||
updated_at TIMESTAMPTZ DEFAULT now(),
|
||
UNIQUE(object_type_id, pk)
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS links (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
link_type TEXT NOT NULL,
|
||
source_type TEXT NOT NULL,
|
||
source_pk TEXT NOT NULL,
|
||
target_type TEXT NOT NULL,
|
||
target_pk TEXT NOT NULL,
|
||
created_at TIMESTAMPTZ DEFAULT now(),
|
||
UNIQUE(link_type, source_type, source_pk, target_type, target_pk)
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS ingestions (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
title TEXT NOT NULL,
|
||
raw_text TEXT DEFAULT '',
|
||
directive TEXT DEFAULT '',
|
||
mode TEXT DEFAULT 'auto',
|
||
status TEXT DEFAULT 'pending',
|
||
provider TEXT DEFAULT '',
|
||
result_json JSONB DEFAULT '{}'::jsonb,
|
||
summary TEXT DEFAULT '',
|
||
created_at TIMESTAMPTZ DEFAULT now(),
|
||
applied_at TIMESTAMPTZ
|
||
);
|
||
|
||
-- 知识文件:archive 目录下的文件元数据,关联提取产生的 ingestion 记录
|
||
CREATE TABLE IF NOT EXISTS km_files (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
rel_path TEXT NOT NULL UNIQUE, -- 相对 archive 根的路径,如 sub/foo.md
|
||
name TEXT NOT NULL,
|
||
folder TEXT NOT NULL DEFAULT '', -- 所属文件夹(相对根),空串表示根
|
||
size BIGINT DEFAULT 0,
|
||
mime TEXT DEFAULT '',
|
||
ingested BOOLEAN DEFAULT FALSE,
|
||
ingestion_id BIGINT, -- 关联 ingestions.id
|
||
summary TEXT DEFAULT '',
|
||
created_at TIMESTAMPTZ DEFAULT now(),
|
||
updated_at TIMESTAMPTZ DEFAULT now()
|
||
);
|
||
|
||
-- 数据连接资源:外部数据源定义(可导入本体成为「数据资源」实例)
|
||
CREATE TABLE IF NOT EXISTS data_connections (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
name TEXT NOT NULL,
|
||
conn_type TEXT NOT NULL DEFAULT 'file', -- db / api / file / other
|
||
config_json JSONB DEFAULT '{}'::jsonb,
|
||
description TEXT DEFAULT '',
|
||
ingested BOOLEAN DEFAULT FALSE,
|
||
ingestion_id BIGINT,
|
||
summary TEXT DEFAULT '',
|
||
created_at TIMESTAMPTZ DEFAULT now(),
|
||
updated_at TIMESTAMPTZ DEFAULT now()
|
||
);
|
||
|
||
-- 兼容存量库:补 name 列并从 properties_json->>'name' 回填,保障连接器名称检索精准
|
||
-- (必须在建索引前执行,否则索引引用未存在的列会报错)
|
||
ALTER TABLE objects ADD COLUMN IF NOT EXISTS name TEXT DEFAULT '';
|
||
UPDATE objects SET name = COALESCE(properties_json->>'name','') WHERE name IS NULL OR name = '';
|
||
ALTER TABLE objects ADD COLUMN IF NOT EXISTS source TEXT DEFAULT '';
|
||
ALTER TABLE links ADD COLUMN IF NOT EXISTS source TEXT DEFAULT '';
|
||
|
||
-- 兼容存量库:补 P0-2 提示调度字段 (2026-08-24)
|
||
ALTER TABLE function_types ADD COLUMN IF NOT EXISTS when_to_suggest JSONB DEFAULT '[]'::jsonb;
|
||
ALTER TABLE function_types ADD COLUMN IF NOT EXISTS llm_tool_description TEXT DEFAULT '';
|
||
ALTER TABLE action_types ADD COLUMN IF NOT EXISTS when_to_suggest JSONB DEFAULT '[]'::jsonb;
|
||
ALTER TABLE action_types ADD COLUMN IF NOT EXISTS llm_tool_description TEXT DEFAULT '';
|
||
|
||
-- 兼容存量库:补 P0-1 Action 完整契约字段 (2026-08-24)
|
||
ALTER TABLE action_types ADD COLUMN IF NOT EXISTS submission_criteria JSONB DEFAULT '{}'::jsonb;
|
||
ALTER TABLE action_types ADD COLUMN IF NOT EXISTS side_effects JSONB DEFAULT '{}'::jsonb;
|
||
ALTER TABLE action_types ADD COLUMN IF NOT EXISTS impl_function_id BIGINT REFERENCES function_types(id) ON DELETE SET NULL;
|
||
ALTER TABLE action_types ADD COLUMN IF NOT EXISTS return_type TEXT DEFAULT '';
|
||
|
||
-- 兼容存量库:补 P1-1 Object Type UX 字段 (2026-08-24)
|
||
ALTER TABLE object_types ADD COLUMN IF NOT EXISTS title_property TEXT DEFAULT '';
|
||
ALTER TABLE object_types ADD COLUMN IF NOT EXISTS subtitle_property TEXT DEFAULT '';
|
||
ALTER TABLE object_types ADD COLUMN IF NOT EXISTS color TEXT DEFAULT '';
|
||
ALTER TABLE object_types ADD COLUMN IF NOT EXISTS primary_key TEXT DEFAULT 'pk';
|
||
|
||
-- 兼容存量库:补 P1-2 Property 字段级元数据 (2026-08-24)
|
||
ALTER TABLE properties ADD COLUMN IF NOT EXISTS format TEXT DEFAULT '';
|
||
ALTER TABLE properties ADD COLUMN IF NOT EXISTS default_value JSONB;
|
||
ALTER TABLE properties ADD COLUMN IF NOT EXISTS indexable BOOLEAN DEFAULT FALSE;
|
||
ALTER TABLE properties ADD COLUMN IF NOT EXISTS vector_index BOOLEAN DEFAULT FALSE;
|
||
ALTER TABLE properties ADD COLUMN IF NOT EXISTS backing_column TEXT DEFAULT '';
|
||
|
||
-- 兼容存量库:补 P1-3 Link Type 元数据 (2026-08-24)
|
||
ALTER TABLE link_types ADD COLUMN IF NOT EXISTS directional BOOLEAN DEFAULT FALSE;
|
||
ALTER TABLE link_types ADD COLUMN IF NOT EXISTS source_property TEXT DEFAULT '';
|
||
ALTER TABLE link_types ADD COLUMN IF NOT EXISTS target_property TEXT DEFAULT '';
|
||
|
||
CREATE TABLE IF NOT EXISTS link_properties (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
link_type_id BIGINT NOT NULL REFERENCES link_types(id) ON DELETE CASCADE,
|
||
api_name TEXT NOT NULL,
|
||
display_name TEXT NOT NULL,
|
||
data_type TEXT NOT NULL DEFAULT 'string',
|
||
required BOOLEAN DEFAULT FALSE,
|
||
enum_values JSONB DEFAULT '[]'::jsonb,
|
||
description TEXT DEFAULT '',
|
||
UNIQUE(link_type_id, api_name)
|
||
);
|
||
|
||
CREATE INDEX IF NOT EXISTS idx_objects_type ON objects(type_api_name);
|
||
CREATE INDEX IF NOT EXISTS idx_objects_name ON objects(name);
|
||
CREATE INDEX IF NOT EXISTS idx_objects_type_name ON objects(type_api_name, name);
|
||
CREATE INDEX IF NOT EXISTS idx_objects_source ON objects(source);
|
||
CREATE INDEX IF NOT EXISTS idx_links_source ON links(source_type, source_pk);
|
||
CREATE INDEX IF NOT EXISTS idx_links_target ON links(target_type, target_pk);
|
||
CREATE INDEX IF NOT EXISTS idx_links_source_tag ON links(source);
|
||
|
||
-- Dynamic Security 审计日志:每次访问决策的轨迹
|
||
CREATE TABLE IF NOT EXISTS access_log (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
created_at TIMESTAMPTZ DEFAULT now(),
|
||
principal TEXT DEFAULT '',
|
||
method TEXT DEFAULT '',
|
||
path TEXT DEFAULT '',
|
||
target_type TEXT DEFAULT '',
|
||
target_pk TEXT DEFAULT '',
|
||
action TEXT DEFAULT '',
|
||
decision TEXT DEFAULT '', -- allow | deny | redact
|
||
matched_grants JSONB DEFAULT '[]'::jsonb,
|
||
matched_markings JSONB DEFAULT '[]'::jsonb,
|
||
duration_ms INT DEFAULT 0
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_access_log_principal ON access_log(principal);
|
||
CREATE INDEX IF NOT EXISTS idx_access_log_created ON access_log(created_at);
|
||
|
||
CREATE TABLE IF NOT EXISTS ontology_audit_log (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
created_at TIMESTAMPTZ DEFAULT now(),
|
||
actor TEXT DEFAULT '', -- 触发操作的主体(manager/connector/...)
|
||
action TEXT DEFAULT '', -- analyze | merge_preview | merge_execute | delete_zero_inst
|
||
target_type TEXT DEFAULT '', -- 主目标类型
|
||
details JSONB DEFAULT '{}'::jsonb, -- 操作细节 (源类型列表 / 影响行数 / 合并对等)
|
||
affected_rows INT DEFAULT 0
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_audit_log_created ON ontology_audit_log(created_at);
|
||
CREATE INDEX IF NOT EXISTS idx_audit_log_actor ON ontology_audit_log(actor);
|
||
-- 兼容存量库(2026-09-11):schema 审计按 target_pk 精确落库
|
||
ALTER TABLE ontology_audit_log ADD COLUMN IF NOT EXISTS target_pk TEXT DEFAULT '';
|
||
|
||
-- 兼容存量库:补 P1-4 Interface 共享 properties + 多态 (2026-08-24)
|
||
ALTER TABLE interface_types ADD COLUMN IF NOT EXISTS shared_properties JSONB DEFAULT '[]'::jsonb;
|
||
ALTER TABLE interface_types ADD COLUMN IF NOT EXISTS parent_interface_id BIGINT REFERENCES interface_types(id) ON DELETE SET NULL;
|
||
|
||
-- 物化层(P2-1 2026-08-24):把指定 ObjectType 的 objects 投影到 materialization_rows 便于快读
|
||
CREATE TABLE IF NOT EXISTS materializations (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
api_name TEXT UNIQUE NOT NULL,
|
||
display_name TEXT NOT NULL,
|
||
description TEXT DEFAULT '',
|
||
source_object_type TEXT NOT NULL,
|
||
-- projection: JSONB 数组,每条 = {property, alias, transform?}
|
||
-- transform 取值(简化版):upper / lower / trim / as_string / as_int
|
||
projection JSONB DEFAULT '[]'::jsonb,
|
||
refresh_strategy TEXT DEFAULT 'manual', -- manual / interval / on_commit
|
||
refresh_interval_seconds INT DEFAULT 0,
|
||
is_active BOOLEAN DEFAULT TRUE,
|
||
last_refresh_at TIMESTAMPTZ,
|
||
last_refresh_rows INT DEFAULT 0,
|
||
last_refresh_duration_ms INT DEFAULT 0,
|
||
created_at TIMESTAMPTZ DEFAULT now(),
|
||
updated_at TIMESTAMPTZ DEFAULT now()
|
||
);
|
||
|
||
-- 物化行(结构按 projection 动态;用 jsonb 兜底)
|
||
CREATE TABLE IF NOT EXISTS materialization_rows (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
materialization_id BIGINT NOT NULL REFERENCES materializations(id) ON DELETE CASCADE,
|
||
source_pk TEXT NOT NULL,
|
||
source_type TEXT NOT NULL,
|
||
row_json JSONB DEFAULT '{}'::jsonb,
|
||
refreshed_at TIMESTAMPTZ DEFAULT now(),
|
||
UNIQUE(materialization_id, source_pk)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_mat_rows_materialization ON materialization_rows(materialization_id);
|
||
CREATE INDEX IF NOT EXISTS idx_mat_rows_source ON materialization_rows(source_type, source_pk);
|
||
|
||
-- 外部数据源 / Backing Datasets(P2-2 2026-08-24):物化层的源配置
|
||
-- 当前精简版:仅 metadata + 字段映射(不做实际 ETL 执行,由 P2-1 materializations 接管)
|
||
CREATE TABLE IF NOT EXISTS backing_sources (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
api_name TEXT UNIQUE NOT NULL,
|
||
display_name TEXT NOT NULL,
|
||
description TEXT DEFAULT '',
|
||
-- 类型:postgres / mysql / csv / api / s3 / other
|
||
source_kind TEXT NOT NULL DEFAULT 'other',
|
||
-- 源连接配置:JSONB(URL / connection_string / path / headers 等,按 source_kind 不同)
|
||
config_json JSONB DEFAULT '{}'::jsonb,
|
||
-- 字段映射:JSONB 数组,每条 = {source_column, target_property, transform?}
|
||
-- source_column 源数据列名
|
||
-- target_property 物化后写入 objects.properties_json 的 key
|
||
-- transform upper/lower/trim/as_string/as_int/as_bool
|
||
field_mapping JSONB DEFAULT '[]'::jsonb,
|
||
-- 关联的物化(可选)
|
||
materialization_id BIGINT REFERENCES materializations(id) ON DELETE SET NULL,
|
||
is_active BOOLEAN DEFAULT TRUE,
|
||
created_at TIMESTAMPTZ DEFAULT now(),
|
||
updated_at TIMESTAMPTZ DEFAULT now()
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_backing_sources_mat ON backing_sources(materialization_id);
|
||
|
||
-- ============================================================================
|
||
-- 实例治理(2026-09-05):软删除 / 别名 / 边属性 / 变更审计
|
||
-- 来源:数字孪生本体外部实操反馈 + 本库扫描(见 docs/INSTANCE_GOVERNANCE.md)
|
||
-- ============================================================================
|
||
|
||
-- 软删除:真实列而非塞进 properties_json,避免与业务属性冲突、且可走索引。
|
||
-- 平台约定:deleted=true 的实例是「墓碑」,消费方默认不可见,管理端可见。
|
||
ALTER TABLE objects ADD COLUMN IF NOT EXISTS deleted BOOLEAN DEFAULT false;
|
||
-- 别名跳转:非空表示该实例是重复实体,canonical 指向同一类型的另一个 pk。
|
||
ALTER TABLE objects ADD COLUMN IF NOT EXISTS same_as TEXT DEFAULT '';
|
||
CREATE INDEX IF NOT EXISTS idx_objects_deleted ON objects(type_api_name, deleted) WHERE deleted;
|
||
CREATE INDEX IF NOT EXISTS idx_objects_same_as ON objects(type_api_name, same_as) WHERE same_as <> '';
|
||
|
||
-- 源时间戳(对齐 Palantir direct datasource 的 source timestamp 语义):
|
||
-- 写入方可携带业务侧时间戳(毫秒);比已有值更旧的写入直接拒绝(stale_write),
|
||
-- 旧信息不覆盖新信息。0 表示未提供,不做乱序检查(完全向后兼容)。
|
||
ALTER TABLE objects ADD COLUMN IF NOT EXISTS source_ts BIGINT DEFAULT 0;
|
||
CREATE INDEX IF NOT EXISTS idx_objects_source_ts ON objects(source_ts) WHERE source_ts > 0;
|
||
|
||
-- 边属性:create_link 支持 properties(任职起止时间、source 溯源等)
|
||
ALTER TABLE links ADD COLUMN IF NOT EXISTS properties_json JSONB DEFAULT '{}'::jsonb;
|
||
|
||
-- KM 文件级排除标记:被标记的文件默认不参与批量抽取
|
||
ALTER TABLE km_files ADD COLUMN IF NOT EXISTS excluded BOOLEAN DEFAULT false;
|
||
|
||
-- 对象级变更审计:记录谁在什么时候改了哪个字段(不含全量版本快照,成本可控)
|
||
CREATE TABLE IF NOT EXISTS object_changes (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
created_at TIMESTAMPTZ DEFAULT now(),
|
||
object_type TEXT NOT NULL,
|
||
object_pk TEXT NOT NULL,
|
||
action TEXT NOT NULL, -- create | update | delete | restore
|
||
actor TEXT DEFAULT '', -- 主体:manager / ingest / connector
|
||
via TEXT DEFAULT '', -- 调用来源:api | mcp | km_extract | batch
|
||
-- diff 结构:{"added":["k"], "removed":["k"], "changed":{"k":{"from":..,"to":..}}}
|
||
diff JSONB DEFAULT '{}'::jsonb,
|
||
-- 删除时的整份属性快照,供 restore_object 回滚
|
||
snapshot JSONB,
|
||
source TEXT DEFAULT '' -- 溯源标签(KM 文件路径 / conn:<id>)
|
||
);
|
||
-- 删除时的关系边快照,供 restore_object 回滚(issue #4,2026-09-07)
|
||
-- 结构:[{link_type,source_type,source_pk,target_type,target_pk,properties,source}, ...]
|
||
-- 存的是「该实例作为 source 或 target 的全部边」,与 deleteObjectCascade 级联删除对称。
|
||
ALTER TABLE object_changes ADD COLUMN IF NOT EXISTS links_snapshot JSONB;
|
||
|
||
-- ============================================================================
|
||
-- 数据分级(2026-09-07,issue #5)
|
||
--
|
||
-- 设计红线:**平台不硬编码 data_level 之类的字段名**。分级是「某类型的某个属性
|
||
-- 承载一组有序级别」的通用能力,data_level=L0~L3 只是其中一份配置。
|
||
-- object_types.classification_property 哪个属性承载分级值(空=该类型不启用分级)
|
||
-- object_types.classification_levels 有序级别(低→高),用于比较与排序
|
||
-- object_types.classification_default 写入时该属性缺失的默认级别(空=不设)
|
||
-- properties.sensitivity 字段级敏感度(值取自同类型的 levels 词汇)
|
||
-- objects.classification 物化后的「生效分级」,供 SQL 直接过滤
|
||
-- km_files.classification 源文件级分级,抽取产物继承其下界
|
||
--
|
||
-- 生效分级 = max(属性分级值, 来源文件分级),重算见 POST /objects/classification/recompute
|
||
-- ============================================================================
|
||
ALTER TABLE object_types ADD COLUMN IF NOT EXISTS classification_property TEXT DEFAULT '';
|
||
ALTER TABLE object_types ADD COLUMN IF NOT EXISTS classification_levels JSONB DEFAULT '[]'::jsonb;
|
||
ALTER TABLE object_types ADD COLUMN IF NOT EXISTS classification_default TEXT DEFAULT '';
|
||
ALTER TABLE properties ADD COLUMN IF NOT EXISTS sensitivity TEXT DEFAULT '';
|
||
ALTER TABLE objects ADD COLUMN IF NOT EXISTS classification TEXT DEFAULT '';
|
||
ALTER TABLE km_files ADD COLUMN IF NOT EXISTS classification TEXT DEFAULT '';
|
||
CREATE INDEX IF NOT EXISTS idx_objects_classification ON objects(type_api_name, classification) WHERE classification <> '';
|
||
|
||
CREATE INDEX IF NOT EXISTS idx_objchg_time ON object_changes(created_at DESC);
|
||
CREATE INDEX IF NOT EXISTS idx_objchg_actor ON object_changes(actor);
|
||
|
||
-- ============================================================================
|
||
-- 数据备份 (v1.3.0,按 schema 镜像):
|
||
-- backups schema 仅存元信息(snapshot_runs + snapshot_tables);实际数据按
|
||
-- schema 名 backups_<id> 镜像:每个快照对应一个独立 schema,包含全部业务
|
||
-- 表的同结构副本 + INSERT...SELECT 数据。请求带 X-Snapshot-Id 头时,
|
||
-- snapshotRouter 中间件开事务 + SET LOCAL search_path → 业务 SQL 不带 schema
|
||
-- 前缀也能命中快照库,写路径则被 403 拒绝。
|
||
-- 仅 manager 主体可访问,详见 server/backup.go + server/snapshot_router.go。
|
||
-- ============================================================================
|
||
CREATE SCHEMA IF NOT EXISTS backups;
|
||
CREATE TABLE IF NOT EXISTS backups.snapshot_runs (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
name TEXT DEFAULT '', -- 可选人读名(备份前后对比时一眼可见)
|
||
principal TEXT NOT NULL DEFAULT '', -- 创建主体
|
||
schema_name TEXT NOT NULL DEFAULT '', -- 实际数据 schema 名(如 backups_42)
|
||
format_version INT NOT NULL DEFAULT 1, -- 备份格式版本
|
||
table_count INT NOT NULL DEFAULT 0,
|
||
row_count BIGINT NOT NULL DEFAULT 0,
|
||
size_bytes BIGINT NOT NULL DEFAULT 0,
|
||
duration_ms INT NOT NULL DEFAULT 0,
|
||
notes TEXT DEFAULT '',
|
||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||
);
|
||
CREATE INDEX IF NOT EXISTS idx_snaprun_created ON backups.snapshot_runs(created_at DESC);
|
||
CREATE INDEX IF NOT EXISTS idx_snaprun_principal ON backups.snapshot_runs(principal);
|
||
|
||
-- ============================================================================
|
||
-- 全局 IRI(GB/T 48000.3-2026 命名要求):iri = 命名空间 + "#" + api_name。
|
||
-- 创建时物化写入;为空时启动回填;base 后续变更不重写已有 iri(标识持久化原则)。
|
||
-- properties 的 iri 按属性名全局拼接(同名属性跨类型共享 IRI,OWL 语义成立)。
|
||
-- ============================================================================
|
||
ALTER TABLE object_types ADD COLUMN IF NOT EXISTS iri TEXT DEFAULT '';
|
||
ALTER TABLE properties ADD COLUMN IF NOT EXISTS iri TEXT DEFAULT '';
|
||
ALTER TABLE link_types ADD COLUMN IF NOT EXISTS iri TEXT DEFAULT '';
|
||
`
|
||
_, err := p.Exec(ctx, ddl)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
// 存量回填 iri:只补空值(规则见 backfillMissingIRI)。
|
||
if err := backfillMissingIRI(ctx, p); err != nil {
|
||
return err
|
||
}
|
||
// 存量修复:objects.name 只从 properties.name 取过值,导致「显示名是 demand_name/title 等
|
||
// 非 name 属性」的实例 name 为空 —— 默认检索(走 name 列)对它们完全失效。
|
||
// object_types.title_property 已定义但从未接通,这里统一回填一次。
|
||
if err := backfillDisplayNames(ctx, p); err != nil {
|
||
return err
|
||
}
|
||
// 存量修复:分级物化列(objects.classification)。新增列的历史数据为空,
|
||
// 不回填会导致「刚启用分级 → 全部实例被视为未分级 → 分级过滤形同虚设」。
|
||
return backfillClassification(ctx, p)
|
||
}
|
||
|
||
// backfillMissingIRI 给 object_types / properties / link_types 三表只补空值 iri
|
||
// (iri 一经写入不随 ONTOLOGY_BASE_IRI 配置变更重写;properties 的 iri 按属性名
|
||
// 全局拼接,同名属性跨类型共享 IRI,OWL 语义成立)。
|
||
// migrate 启动时回填存量;seedIfEmpty / mergeExecute / AI 摄取等旁路创建位点
|
||
// (不走主创建路径、写入行不带 iri)在写入后调用治愈。幂等,开销为三表空值 UPDATE
|
||
// (schema 规模小),请求内联调用可接受;baseIRI 为空时整体跳过,不产出残缺 IRI。
|
||
func backfillMissingIRI(ctx context.Context, p dbExec) error {
|
||
if baseIRI() == "" { // 未配置命名空间时不产出残缺 IRI
|
||
return nil
|
||
}
|
||
prefix := strings.TrimSuffix(baseIRI(), "#") + "#"
|
||
for _, q := range []string{
|
||
`UPDATE object_types SET iri=$1||api_name WHERE iri IS NULL OR iri=''`,
|
||
`UPDATE properties SET iri=$1||api_name WHERE iri IS NULL OR iri=''`,
|
||
`UPDATE link_types SET iri=$1||api_name WHERE iri IS NULL OR iri=''`,
|
||
} {
|
||
if _, err := p.Exec(ctx, q, prefix); err != nil {
|
||
return fmt.Errorf("iri 回填失败: %w", err)
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// seedIfEmpty 首次启动且本体为空时灌入极简种子数据(1 个对象类型 + 3 条示例)。
|
||
// 设计:让别人 docker run 一开箱就能看到 "合作企业" + 三条占位示例,
|
||
// 也能在管理界面一键清掉,不污染业务本体。
|
||
func seedIfEmpty(ctx context.Context, p dbExec) error {
|
||
var n int
|
||
if err := p.QueryRow(ctx, "SELECT count(*) FROM object_types").Scan(&n); err != nil {
|
||
return err
|
||
}
|
||
if n > 0 {
|
||
log.Println("[seed] 本体已存在,跳过种子数据")
|
||
return nil
|
||
}
|
||
log.Println("[seed] 灌入极简占位种子 ...")
|
||
|
||
type seedProp struct {
|
||
APIName, DisplayName, DataType, Description string
|
||
Required bool
|
||
Enum []string
|
||
}
|
||
type seedType struct {
|
||
APIName, DisplayName, Description, Group, Icon string
|
||
Props []seedProp
|
||
}
|
||
types := []seedType{
|
||
// 极简种子 —— 仅 1 个对象类型 + 3 条示例,方便开箱即用又便于清理
|
||
{"MemberOrg", "合作企业", "组织机构(合作企业、会员单位、生态伙伴等)。", "组织", "🤝", []seedProp{
|
||
{"name", "名称", "string", "", true, nil},
|
||
{"role", "角色", "enum", "", false, []string{"自营", "合作伙伴", "客户", "供应商"}},
|
||
{"region", "区域", "string", "", false, nil},
|
||
}},
|
||
}
|
||
|
||
for _, t := range types {
|
||
var oid int64
|
||
if err := p.QueryRow(ctx,
|
||
"INSERT INTO object_types(api_name,display_name,description,icon,group_name) VALUES($1,$2,$3,$4,$5) RETURNING id",
|
||
t.APIName, t.DisplayName, t.Description, t.Icon, t.Group).Scan(&oid); err != nil {
|
||
return fmt.Errorf("插入对象类型 %s 失败: %w", t.APIName, err)
|
||
}
|
||
for _, pr := range t.Props {
|
||
enumJSON := "[]"
|
||
if len(pr.Enum) > 0 {
|
||
enumJSON = mustJSON(pr.Enum)
|
||
}
|
||
if _, err := p.Exec(ctx,
|
||
"INSERT INTO properties(object_type_id,api_name,display_name,data_type,required,enum_values,description) VALUES($1,$2,$3,$4,$5,$6,$7)",
|
||
oid, pr.APIName, pr.DisplayName, pr.DataType, pr.Required, enumJSON, pr.Description); err != nil {
|
||
return fmt.Errorf("插入属性 %s.%s 失败: %w", t.APIName, pr.APIName, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
// 链接类型 — 极简种子不内置,留空让用户按需创建
|
||
linkTypes := []struct {
|
||
api, disp, src, tgt, card, desc string
|
||
}{}
|
||
for _, l := range linkTypes {
|
||
if _, err := p.Exec(ctx,
|
||
"INSERT INTO link_types(api_name,display_name,source_type,target_type,cardinality,description) VALUES($1,$2,$3,$4,$5,$6) ON CONFLICT(api_name) DO NOTHING",
|
||
l.api, l.disp, l.src, l.tgt, l.card, l.desc); err != nil {
|
||
return fmt.Errorf("插入链接类型 %s 失败: %w", l.api, err)
|
||
}
|
||
}
|
||
|
||
// 动作类型 — 极简种子不内置,留空让用户按需创建
|
||
actionTypes := []struct{ api, disp, desc, params string }{}
|
||
for _, a := range actionTypes {
|
||
if _, err := p.Exec(ctx,
|
||
"INSERT INTO action_types(api_name,display_name,description,params) VALUES($1,$2,$3,$4::jsonb) ON CONFLICT(api_name) DO NOTHING",
|
||
a.api, a.disp, a.desc, a.params); err != nil {
|
||
return fmt.Errorf("插入动作类型 %s 失败: %w", a.api, err)
|
||
}
|
||
}
|
||
|
||
// 对象实例 — 极简种子:3 条占位示例(与具体业务无关,仅用于开箱演示),部署后可一键删干净
|
||
objs := []struct {
|
||
typ, pk string
|
||
props map[string]interface{}
|
||
}{
|
||
{"MemberOrg", "demo_self", m("name", "示例·自营组织", "role", "自营", "region", "本地")},
|
||
{"MemberOrg", "demo_partner_a", m("name", "示例·合作伙伴 A", "role", "合作伙伴", "region", "本地")},
|
||
{"MemberOrg", "demo_partner_b", m("name", "示例·合作伙伴 B", "role", "合作伙伴", "region", "异地")},
|
||
}
|
||
for _, o := range objs {
|
||
if err := insertObjectRaw(ctx, p, o.typ, o.pk, o.props); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
|
||
// 链接实例 — 极简种子无内置链接
|
||
links := []struct {
|
||
typ, st, spk, tt, tpk string
|
||
}{}
|
||
for _, l := range links {
|
||
if _, err := p.Exec(ctx,
|
||
"INSERT INTO links(link_type,source_type,source_pk,target_type,target_pk) VALUES($1,$2,$3,$4,$5) ON CONFLICT DO NOTHING",
|
||
l.typ, l.st, l.spk, l.tt, l.tpk); err != nil {
|
||
return fmt.Errorf("插入链接 %s 失败: %w", l.typ, err)
|
||
}
|
||
}
|
||
|
||
// 种子行走旁路插入(不带 iri),灌入后立即治愈
|
||
if err := backfillMissingIRI(ctx, p); err != nil {
|
||
return err
|
||
}
|
||
log.Println("[seed] 种子数据完成")
|
||
return nil
|
||
}
|
||
|
||
// insertObjectRaw 在 seed 阶段直接插入对象实例(按类型名+主键)。
|
||
func insertObjectRaw(ctx context.Context, p dbExec, typeName, pk string, props map[string]interface{}) error {
|
||
var oid int64
|
||
if err := p.QueryRow(ctx, "SELECT id FROM object_types WHERE api_name=$1", typeName).Scan(&oid); err != nil {
|
||
return fmt.Errorf("找不到对象类型 %s: %w", typeName, err)
|
||
}
|
||
propsJSON := mustJSON(props)
|
||
nameVal, _ := props["name"].(string)
|
||
_, err := p.Exec(ctx,
|
||
"INSERT INTO objects(object_type_id,type_api_name,pk,name,properties_json) VALUES($1,$2,$3,$4,$5::jsonb) ON CONFLICT(object_type_id,pk) DO UPDATE SET name=EXCLUDED.name, properties_json=EXCLUDED.properties_json, updated_at=now()",
|
||
oid, typeName, pk, nameVal, propsJSON)
|
||
if err != nil {
|
||
return fmt.Errorf("插入对象 %s/%s 失败: %w", typeName, pk, err)
|
||
}
|
||
return nil
|
||
}
|