Files
ssw-ontology/server/handlers_materialization.go
T
admin 86730bd52d feat: 对标 Palantir Ontology + P0/P1/P2 全落地 + xeo 端到端导入
## Palantir Ontology 对标(10 项)

P0 核心契约:
- when_to_suggest JSONB + llm_tool_description TEXT 加到 function_types/action_types
- Action 完整契约: submission_criteria / side_effects / impl_function_id / return_type
- FBA (Function-backed Action) 执行链: /api/v1/objects/:type/:pk/actions/:action
- /api/v1/ontology/suggestions 提示调度端点

P1 元原语补全:
- Object Type UX: title_property / subtitle_property / color / primary_key
- Property 字段元数据: format / default_value / indexable / vector_index / backing_column
- Link Type: directional + 独立 link_properties 子表
- Interface Type: shared_properties + parent_interface_id 多态
- Property extended types: geoshape / geopoint / media / timeseries / timestamp

P2 数据管道层:
- materializations / materialization_rows 物化层 + 60s ticker scheduler
- backing_sources 同步源元数据 + 投影字段映射

AI 提示词升级:
- ai/prompt.go 抽出 250+ 行静态 systemPrompt
- 动态上下文注入: 每次 Suggest 前查 db 拉已注册类型清单
- Suggestion struct 扩展 22 字段

## ON CONFLICT 漏 source 修复(4 处)
- handlers_objects createObject/updateObject: COALESCE(NULLIF(EXCLUDED.source,''), objects.source)
- handlers_km upsertObject + ai.ApplySuggestion: SET source=EXCLUDED.source
- 表名必须限定 objects.source, 否则报 ambiguous

## xeo mobileApi 端到端导入
- /api/v1/ai/ingest raw_text 改用 json.dumps 转义, 修复 ingestion_id=56 failed
- 1179 实例 + 1672 links + 1806 物化行
- 占位: 5 TrustedDataSpace + 158 MemberOrg
- 5 类 Materialization 配好(mat_xeo_dataproduct 等)

## 文档
- docs/update/2026-08-24.md: 10 节迁移 + 增量 11 节
- .gitignore: 加 web/vite.config.js.timestamp-*.mjs

Refs: docs/update/2026-08-24.md
2026-08-24 13:54:49 +08:00

359 lines
11 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
// 物化层(P2-1 2026-08-24)
//
// 设计目标:把指定 ObjectType 的 objects 按 projection 投影到 materialization_rows,
// 让大查询走"快读表"而非实时 join properties_json。
//
// 本期落地(精简版):
// - 定义 CRUD:list / create / delete / get
// - 手动刷新:POST /:api_name/refresh
// - 自动刷新:startMaterializationScheduler 启 goroutine,每 60s 扫一次 is_active + interval 到期的物化
// - 物化行查询:GET /:api_name/rows(直接返 materialization_rows)
// - 投影规则(简化版):projection[].property 读 source.properties_json[property],按 transform 转
// 当前支持的 transform:upper / lower / trim / as_string / as_int / as_bool
import (
"context"
"encoding/json"
"fmt"
"log"
"strconv"
"strings"
"time"
"github.com/gin-gonic/gin"
)
type Materialization struct {
ID int64 `json:"id"`
APIName string `json:"api_name"`
DisplayName string `json:"display_name"`
Description string `json:"description"`
SourceObjectType string `json:"source_object_type"`
Projection []map[string]interface{} `json:"projection"`
RefreshStrategy string `json:"refresh_strategy"`
RefreshIntervalSeconds int `json:"refresh_interval_seconds"`
IsActive bool `json:"is_active"`
LastRefreshAt *time.Time `json:"last_refresh_at"`
LastRefreshRows int `json:"last_refresh_rows"`
LastRefreshDurationMs int `json:"last_refresh_duration_ms"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
func scanMatRow(scan func(...any) error) (Materialization, error) {
var m Materialization
var projection []byte
var lastRefresh *time.Time
if err := scan(&m.ID, &m.APIName, &m.DisplayName, &m.Description, &m.SourceObjectType,
&projection, &m.RefreshStrategy, &m.RefreshIntervalSeconds, &m.IsActive,
&lastRefresh, &m.LastRefreshRows, &m.LastRefreshDurationMs,
&m.CreatedAt, &m.UpdatedAt); err != nil {
return m, err
}
m.Projection = []map[string]interface{}{}
_ = json.Unmarshal(projection, &m.Projection)
m.LastRefreshAt = lastRefresh
return m, nil
}
func (a *app) listMaterializations(c *gin.Context) {
rows, err := a.pool.Query(c,
`SELECT id, api_name, display_name, description, source_object_type, projection,
refresh_strategy, refresh_interval_seconds, is_active,
last_refresh_at, last_refresh_rows, last_refresh_duration_ms,
created_at, updated_at
FROM materializations ORDER BY id`)
if err != nil {
c.JSON(500, gin.H{"error": err.Error()})
return
}
defer rows.Close()
var out []Materialization
for rows.Next() {
m, err := scanMatRow(rows.Scan)
if err == nil {
out = append(out, m)
}
}
c.JSON(200, gin.H{"materializations": out, "count": len(out)})
}
type materializationInput struct {
APIName string `json:"api_name"`
DisplayName string `json:"display_name"`
Description string `json:"description"`
SourceObjectType string `json:"source_object_type"`
Projection []map[string]interface{} `json:"projection"`
RefreshStrategy string `json:"refresh_strategy"`
RefreshIntervalSeconds int `json:"refresh_interval_seconds"`
IsActive bool `json:"is_active"`
}
func (a *app) createMaterialization(c *gin.Context) {
var in materializationInput
if err := c.ShouldBindJSON(&in); err != nil {
c.JSON(400, gin.H{"error": err.Error()})
return
}
if in.APIName == "" || in.DisplayName == "" || in.SourceObjectType == "" {
c.JSON(400, gin.H{"error": "api_name/display_name/source_object_type 必填"})
return
}
// 校验 source_object_type 真实存在
var cnt int
if err := a.pool.QueryRow(c, `SELECT count(*) FROM object_types WHERE api_name=$1`, in.SourceObjectType).Scan(&cnt); err != nil || cnt == 0 {
c.JSON(400, gin.H{"error": "source_object_type 不存在: " + in.SourceObjectType})
return
}
if in.RefreshStrategy == "" {
in.RefreshStrategy = "manual"
}
if in.Projection == nil {
in.Projection = []map[string]interface{}{}
}
projJSON := mustJSON(in.Projection)
_, err := a.pool.Exec(c,
`INSERT INTO materializations(api_name,display_name,description,source_object_type,projection,
refresh_strategy,refresh_interval_seconds,is_active)
VALUES($1,$2,$3,$4,$5::jsonb,$6,$7,$8)
ON CONFLICT(api_name) DO UPDATE SET
display_name=EXCLUDED.display_name,
description=EXCLUDED.description,
source_object_type=EXCLUDED.source_object_type,
projection=EXCLUDED.projection,
refresh_strategy=EXCLUDED.refresh_strategy,
refresh_interval_seconds=EXCLUDED.refresh_interval_seconds,
is_active=EXCLUDED.is_active,
updated_at=now()`,
in.APIName, in.DisplayName, in.Description, in.SourceObjectType, projJSON,
in.RefreshStrategy, in.RefreshIntervalSeconds, in.IsActive)
if err != nil {
c.JSON(409, gin.H{"error": "创建失败: " + err.Error()})
return
}
c.JSON(201, gin.H{"ok": true, "api_name": in.APIName})
}
func (a *app) deleteMaterialization(c *gin.Context) {
name := c.Param("name")
tag, err := a.pool.Exec(c, `DELETE FROM materializations WHERE api_name=$1`, name)
if err != nil {
c.JSON(500, gin.H{"error": err.Error()})
return
}
if tag.RowsAffected() == 0 {
c.JSON(404, gin.H{"error": "物化不存在"})
return
}
c.JSON(200, gin.H{"ok": true})
}
// applyTransform 按 projection.transform 简化版规则转换值
func applyTransform(v interface{}, transform string) interface{} {
if v == nil {
return nil
}
s, ok := v.(string)
if !ok {
return v
}
switch transform {
case "upper":
return strings.ToUpper(s)
case "lower":
return strings.ToLower(s)
case "trim":
return strings.TrimSpace(s)
case "as_int":
var i int
_, _ = fmt.Sscanf(s, "%d", &i)
return i
case "as_bool":
return s == "true" || s == "1" || s == "yes"
case "as_string":
fallthrough
default:
return s
}
}
// refreshMaterialization 按 projection 从 source objects 投影到 materialization_rows
// 返回受影响行数 + 耗时。
func refreshMaterialization(ctx context.Context, a *app, matID int64, apiName string) (int, int64, error) {
t0 := time.Now()
// 加载物化定义
var mat Materialization
var projection []byte
err := a.pool.QueryRow(ctx,
`SELECT id, api_name, display_name, description, source_object_type, projection,
refresh_strategy, refresh_interval_seconds, is_active
FROM materializations WHERE id=$1`, matID).
Scan(&mat.ID, &mat.APIName, &mat.DisplayName, &mat.Description, &mat.SourceObjectType,
&projection, &mat.RefreshStrategy, &mat.RefreshIntervalSeconds, &mat.IsActive)
if err != nil {
return 0, 0, fmt.Errorf("加载物化定义失败: %w", err)
}
_ = json.Unmarshal(projection, &mat.Projection)
// 拉源数据
rows, err := a.pool.Query(ctx,
`SELECT pk, properties_json FROM objects WHERE type_api_name=$1`, mat.SourceObjectType)
if err != nil {
return 0, 0, fmt.Errorf("拉源数据失败: %w", err)
}
defer rows.Close()
tx, err := a.pool.Begin(ctx)
if err != nil {
return 0, 0, err
}
defer tx.Rollback(ctx)
count := 0
for rows.Next() {
var pk string
var propsRaw []byte
if err := rows.Scan(&pk, &propsRaw); err != nil {
continue
}
var props map[string]interface{}
_ = json.Unmarshal(propsRaw, &props)
// 按 projection 拼 row_json
rowJSON := map[string]interface{}{}
for _, col := range mat.Projection {
prop, _ := col["property"].(string)
alias, _ := col["alias"].(string)
transform, _ := col["transform"].(string)
if prop == "" {
continue
}
if alias == "" {
alias = prop
}
rowJSON[alias] = applyTransform(props[prop], transform)
}
rowJSON["pk"] = pk
// upsert 到 materialization_rows
_, err := tx.Exec(ctx,
`INSERT INTO materialization_rows(materialization_id,source_pk,source_type,row_json,refreshed_at)
VALUES($1,$2,$3,$4::jsonb,now())
ON CONFLICT(materialization_id,source_pk) DO UPDATE SET
row_json=EXCLUDED.row_json, refreshed_at=now()`,
matID, pk, mat.SourceObjectType, mustJSON(rowJSON))
if err != nil {
return 0, 0, fmt.Errorf("写入物化行失败: %w", err)
}
count++
}
duration := time.Since(t0).Milliseconds()
// 更新物化的 last_refresh_*
_, _ = tx.Exec(ctx,
`UPDATE materializations SET last_refresh_at=now(), last_refresh_rows=$1, last_refresh_duration_ms=$2, updated_at=now() WHERE id=$3`,
count, duration, matID)
if err := tx.Commit(ctx); err != nil {
return 0, 0, err
}
log.Printf("[mat] %s 刷新完成:%d 行,%d ms", apiName, count, duration)
return count, duration, nil
}
// POST /api/v1/materializations/:name/refresh
func (a *app) refreshMaterializationHandler(c *gin.Context) {
name := c.Param("name")
var mid int64
if err := a.pool.QueryRow(c, `SELECT id FROM materializations WHERE api_name=$1`, name).Scan(&mid); err != nil {
c.JSON(404, gin.H{"error": "物化不存在"})
return
}
count, duration, err := refreshMaterialization(c.Request.Context(), a, mid, name)
if err != nil {
c.JSON(500, gin.H{"error": err.Error()})
return
}
c.JSON(200, gin.H{"ok": true, "rows": count, "duration_ms": duration})
}
// GET /api/v1/materializations/:name/rows
func (a *app) listMaterializationRows(c *gin.Context) {
name := c.Param("name")
limit, _ := strconv.Atoi(c.Query("limit"))
if limit <= 0 || limit > 1000 {
limit = 100
}
rows, err := a.pool.Query(c,
`SELECT row_json FROM materialization_rows mr
JOIN materializations m ON m.id = mr.materialization_id
WHERE m.api_name=$1 ORDER BY mr.source_pk LIMIT $2`, name, limit)
if err != nil {
c.JSON(500, gin.H{"error": err.Error()})
return
}
defer rows.Close()
var out []map[string]interface{}
for rows.Next() {
var j []byte
if err := rows.Scan(&j); err != nil {
continue
}
var r map[string]interface{}
_ = json.Unmarshal(j, &r)
out = append(out, r)
}
c.JSON(200, gin.H{"rows": out, "count": len(out)})
}
// startMaterializationScheduler 启后台 ticker,每 60 秒扫一次 interval 到期的物化
func startMaterializationScheduler(ctx context.Context, a *app) {
go func() {
tick := time.NewTicker(60 * time.Second)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
rows, err := a.pool.Query(ctx,
`SELECT id, api_name, refresh_interval_seconds FROM materializations
WHERE is_active=TRUE AND refresh_strategy='interval' AND refresh_interval_seconds>0`)
if err != nil {
log.Printf("[mat scheduler] 扫描失败: %v", err)
continue
}
type cand struct {
id int64
name string
every int
}
var cands []cand
for rows.Next() {
var c cand
if err := rows.Scan(&c.id, &c.name, &c.every); err != nil {
continue
}
cands = append(cands, c)
}
rows.Close()
for _, c := range cands {
// 简单判定:距 last_refresh_at 超过 refresh_interval_seconds 才刷
var lastRefresh *time.Time
if err := a.pool.QueryRow(ctx, `SELECT last_refresh_at FROM materializations WHERE id=$1`, c.id).Scan(&lastRefresh); err != nil {
continue
}
if lastRefresh == nil || time.Since(*lastRefresh) > time.Duration(c.every)*time.Second {
if _, _, err := refreshMaterialization(ctx, a, c.id, c.name); err != nil {
log.Printf("[mat scheduler] 刷新 %s 失败: %v", c.name, err)
}
}
}
}
}
}()
}