## Palantir Ontology 对标(10 项) P0 核心契约: - when_to_suggest JSONB + llm_tool_description TEXT 加到 function_types/action_types - Action 完整契约: submission_criteria / side_effects / impl_function_id / return_type - FBA (Function-backed Action) 执行链: /api/v1/objects/:type/:pk/actions/:action - /api/v1/ontology/suggestions 提示调度端点 P1 元原语补全: - Object Type UX: title_property / subtitle_property / color / primary_key - Property 字段元数据: format / default_value / indexable / vector_index / backing_column - Link Type: directional + 独立 link_properties 子表 - Interface Type: shared_properties + parent_interface_id 多态 - Property extended types: geoshape / geopoint / media / timeseries / timestamp P2 数据管道层: - materializations / materialization_rows 物化层 + 60s ticker scheduler - backing_sources 同步源元数据 + 投影字段映射 AI 提示词升级: - ai/prompt.go 抽出 250+ 行静态 systemPrompt - 动态上下文注入: 每次 Suggest 前查 db 拉已注册类型清单 - Suggestion struct 扩展 22 字段 ## ON CONFLICT 漏 source 修复(4 处) - handlers_objects createObject/updateObject: COALESCE(NULLIF(EXCLUDED.source,''), objects.source) - handlers_km upsertObject + ai.ApplySuggestion: SET source=EXCLUDED.source - 表名必须限定 objects.source, 否则报 ambiguous ## xeo mobileApi 端到端导入 - /api/v1/ai/ingest raw_text 改用 json.dumps 转义, 修复 ingestion_id=56 failed - 1179 实例 + 1672 links + 1806 物化行 - 占位: 5 TrustedDataSpace + 158 MemberOrg - 5 类 Materialization 配好(mat_xeo_dataproduct 等) ## 文档 - docs/update/2026-08-24.md: 10 节迁移 + 增量 11 节 - .gitignore: 加 web/vite.config.js.timestamp-*.mjs Refs: docs/update/2026-08-24.md
359 lines
11 KiB
Go
359 lines
11 KiB
Go
package main
|
||
|
||
// 物化层(P2-1 2026-08-24)
|
||
//
|
||
// 设计目标:把指定 ObjectType 的 objects 按 projection 投影到 materialization_rows,
|
||
// 让大查询走"快读表"而非实时 join properties_json。
|
||
//
|
||
// 本期落地(精简版):
|
||
// - 定义 CRUD:list / create / delete / get
|
||
// - 手动刷新:POST /:api_name/refresh
|
||
// - 自动刷新:startMaterializationScheduler 启 goroutine,每 60s 扫一次 is_active + interval 到期的物化
|
||
// - 物化行查询:GET /:api_name/rows(直接返 materialization_rows)
|
||
// - 投影规则(简化版):projection[].property 读 source.properties_json[property],按 transform 转
|
||
// 当前支持的 transform:upper / lower / trim / as_string / as_int / as_bool
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"log"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/gin-gonic/gin"
|
||
)
|
||
|
||
type Materialization struct {
|
||
ID int64 `json:"id"`
|
||
APIName string `json:"api_name"`
|
||
DisplayName string `json:"display_name"`
|
||
Description string `json:"description"`
|
||
SourceObjectType string `json:"source_object_type"`
|
||
Projection []map[string]interface{} `json:"projection"`
|
||
RefreshStrategy string `json:"refresh_strategy"`
|
||
RefreshIntervalSeconds int `json:"refresh_interval_seconds"`
|
||
IsActive bool `json:"is_active"`
|
||
LastRefreshAt *time.Time `json:"last_refresh_at"`
|
||
LastRefreshRows int `json:"last_refresh_rows"`
|
||
LastRefreshDurationMs int `json:"last_refresh_duration_ms"`
|
||
CreatedAt time.Time `json:"created_at"`
|
||
UpdatedAt time.Time `json:"updated_at"`
|
||
}
|
||
|
||
func scanMatRow(scan func(...any) error) (Materialization, error) {
|
||
var m Materialization
|
||
var projection []byte
|
||
var lastRefresh *time.Time
|
||
if err := scan(&m.ID, &m.APIName, &m.DisplayName, &m.Description, &m.SourceObjectType,
|
||
&projection, &m.RefreshStrategy, &m.RefreshIntervalSeconds, &m.IsActive,
|
||
&lastRefresh, &m.LastRefreshRows, &m.LastRefreshDurationMs,
|
||
&m.CreatedAt, &m.UpdatedAt); err != nil {
|
||
return m, err
|
||
}
|
||
m.Projection = []map[string]interface{}{}
|
||
_ = json.Unmarshal(projection, &m.Projection)
|
||
m.LastRefreshAt = lastRefresh
|
||
return m, nil
|
||
}
|
||
|
||
func (a *app) listMaterializations(c *gin.Context) {
|
||
rows, err := a.pool.Query(c,
|
||
`SELECT id, api_name, display_name, description, source_object_type, projection,
|
||
refresh_strategy, refresh_interval_seconds, is_active,
|
||
last_refresh_at, last_refresh_rows, last_refresh_duration_ms,
|
||
created_at, updated_at
|
||
FROM materializations ORDER BY id`)
|
||
if err != nil {
|
||
c.JSON(500, gin.H{"error": err.Error()})
|
||
return
|
||
}
|
||
defer rows.Close()
|
||
var out []Materialization
|
||
for rows.Next() {
|
||
m, err := scanMatRow(rows.Scan)
|
||
if err == nil {
|
||
out = append(out, m)
|
||
}
|
||
}
|
||
c.JSON(200, gin.H{"materializations": out, "count": len(out)})
|
||
}
|
||
|
||
type materializationInput struct {
|
||
APIName string `json:"api_name"`
|
||
DisplayName string `json:"display_name"`
|
||
Description string `json:"description"`
|
||
SourceObjectType string `json:"source_object_type"`
|
||
Projection []map[string]interface{} `json:"projection"`
|
||
RefreshStrategy string `json:"refresh_strategy"`
|
||
RefreshIntervalSeconds int `json:"refresh_interval_seconds"`
|
||
IsActive bool `json:"is_active"`
|
||
}
|
||
|
||
func (a *app) createMaterialization(c *gin.Context) {
|
||
var in materializationInput
|
||
if err := c.ShouldBindJSON(&in); err != nil {
|
||
c.JSON(400, gin.H{"error": err.Error()})
|
||
return
|
||
}
|
||
if in.APIName == "" || in.DisplayName == "" || in.SourceObjectType == "" {
|
||
c.JSON(400, gin.H{"error": "api_name/display_name/source_object_type 必填"})
|
||
return
|
||
}
|
||
// 校验 source_object_type 真实存在
|
||
var cnt int
|
||
if err := a.pool.QueryRow(c, `SELECT count(*) FROM object_types WHERE api_name=$1`, in.SourceObjectType).Scan(&cnt); err != nil || cnt == 0 {
|
||
c.JSON(400, gin.H{"error": "source_object_type 不存在: " + in.SourceObjectType})
|
||
return
|
||
}
|
||
if in.RefreshStrategy == "" {
|
||
in.RefreshStrategy = "manual"
|
||
}
|
||
if in.Projection == nil {
|
||
in.Projection = []map[string]interface{}{}
|
||
}
|
||
projJSON := mustJSON(in.Projection)
|
||
|
||
_, err := a.pool.Exec(c,
|
||
`INSERT INTO materializations(api_name,display_name,description,source_object_type,projection,
|
||
refresh_strategy,refresh_interval_seconds,is_active)
|
||
VALUES($1,$2,$3,$4,$5::jsonb,$6,$7,$8)
|
||
ON CONFLICT(api_name) DO UPDATE SET
|
||
display_name=EXCLUDED.display_name,
|
||
description=EXCLUDED.description,
|
||
source_object_type=EXCLUDED.source_object_type,
|
||
projection=EXCLUDED.projection,
|
||
refresh_strategy=EXCLUDED.refresh_strategy,
|
||
refresh_interval_seconds=EXCLUDED.refresh_interval_seconds,
|
||
is_active=EXCLUDED.is_active,
|
||
updated_at=now()`,
|
||
in.APIName, in.DisplayName, in.Description, in.SourceObjectType, projJSON,
|
||
in.RefreshStrategy, in.RefreshIntervalSeconds, in.IsActive)
|
||
if err != nil {
|
||
c.JSON(409, gin.H{"error": "创建失败: " + err.Error()})
|
||
return
|
||
}
|
||
c.JSON(201, gin.H{"ok": true, "api_name": in.APIName})
|
||
}
|
||
|
||
func (a *app) deleteMaterialization(c *gin.Context) {
|
||
name := c.Param("name")
|
||
tag, err := a.pool.Exec(c, `DELETE FROM materializations WHERE api_name=$1`, name)
|
||
if err != nil {
|
||
c.JSON(500, gin.H{"error": err.Error()})
|
||
return
|
||
}
|
||
if tag.RowsAffected() == 0 {
|
||
c.JSON(404, gin.H{"error": "物化不存在"})
|
||
return
|
||
}
|
||
c.JSON(200, gin.H{"ok": true})
|
||
}
|
||
|
||
// applyTransform 按 projection.transform 简化版规则转换值
|
||
func applyTransform(v interface{}, transform string) interface{} {
|
||
if v == nil {
|
||
return nil
|
||
}
|
||
s, ok := v.(string)
|
||
if !ok {
|
||
return v
|
||
}
|
||
switch transform {
|
||
case "upper":
|
||
return strings.ToUpper(s)
|
||
case "lower":
|
||
return strings.ToLower(s)
|
||
case "trim":
|
||
return strings.TrimSpace(s)
|
||
case "as_int":
|
||
var i int
|
||
_, _ = fmt.Sscanf(s, "%d", &i)
|
||
return i
|
||
case "as_bool":
|
||
return s == "true" || s == "1" || s == "yes"
|
||
case "as_string":
|
||
fallthrough
|
||
default:
|
||
return s
|
||
}
|
||
}
|
||
|
||
// refreshMaterialization 按 projection 从 source objects 投影到 materialization_rows
|
||
// 返回受影响行数 + 耗时。
|
||
func refreshMaterialization(ctx context.Context, a *app, matID int64, apiName string) (int, int64, error) {
|
||
t0 := time.Now()
|
||
|
||
// 加载物化定义
|
||
var mat Materialization
|
||
var projection []byte
|
||
err := a.pool.QueryRow(ctx,
|
||
`SELECT id, api_name, display_name, description, source_object_type, projection,
|
||
refresh_strategy, refresh_interval_seconds, is_active
|
||
FROM materializations WHERE id=$1`, matID).
|
||
Scan(&mat.ID, &mat.APIName, &mat.DisplayName, &mat.Description, &mat.SourceObjectType,
|
||
&projection, &mat.RefreshStrategy, &mat.RefreshIntervalSeconds, &mat.IsActive)
|
||
if err != nil {
|
||
return 0, 0, fmt.Errorf("加载物化定义失败: %w", err)
|
||
}
|
||
_ = json.Unmarshal(projection, &mat.Projection)
|
||
|
||
// 拉源数据
|
||
rows, err := a.pool.Query(ctx,
|
||
`SELECT pk, properties_json FROM objects WHERE type_api_name=$1`, mat.SourceObjectType)
|
||
if err != nil {
|
||
return 0, 0, fmt.Errorf("拉源数据失败: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
|
||
tx, err := a.pool.Begin(ctx)
|
||
if err != nil {
|
||
return 0, 0, err
|
||
}
|
||
defer tx.Rollback(ctx)
|
||
|
||
count := 0
|
||
for rows.Next() {
|
||
var pk string
|
||
var propsRaw []byte
|
||
if err := rows.Scan(&pk, &propsRaw); err != nil {
|
||
continue
|
||
}
|
||
var props map[string]interface{}
|
||
_ = json.Unmarshal(propsRaw, &props)
|
||
// 按 projection 拼 row_json
|
||
rowJSON := map[string]interface{}{}
|
||
for _, col := range mat.Projection {
|
||
prop, _ := col["property"].(string)
|
||
alias, _ := col["alias"].(string)
|
||
transform, _ := col["transform"].(string)
|
||
if prop == "" {
|
||
continue
|
||
}
|
||
if alias == "" {
|
||
alias = prop
|
||
}
|
||
rowJSON[alias] = applyTransform(props[prop], transform)
|
||
}
|
||
rowJSON["pk"] = pk
|
||
// upsert 到 materialization_rows
|
||
_, err := tx.Exec(ctx,
|
||
`INSERT INTO materialization_rows(materialization_id,source_pk,source_type,row_json,refreshed_at)
|
||
VALUES($1,$2,$3,$4::jsonb,now())
|
||
ON CONFLICT(materialization_id,source_pk) DO UPDATE SET
|
||
row_json=EXCLUDED.row_json, refreshed_at=now()`,
|
||
matID, pk, mat.SourceObjectType, mustJSON(rowJSON))
|
||
if err != nil {
|
||
return 0, 0, fmt.Errorf("写入物化行失败: %w", err)
|
||
}
|
||
count++
|
||
}
|
||
|
||
duration := time.Since(t0).Milliseconds()
|
||
|
||
// 更新物化的 last_refresh_*
|
||
_, _ = tx.Exec(ctx,
|
||
`UPDATE materializations SET last_refresh_at=now(), last_refresh_rows=$1, last_refresh_duration_ms=$2, updated_at=now() WHERE id=$3`,
|
||
count, duration, matID)
|
||
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return 0, 0, err
|
||
}
|
||
|
||
log.Printf("[mat] %s 刷新完成:%d 行,%d ms", apiName, count, duration)
|
||
return count, duration, nil
|
||
}
|
||
|
||
// POST /api/v1/materializations/:name/refresh
|
||
func (a *app) refreshMaterializationHandler(c *gin.Context) {
|
||
name := c.Param("name")
|
||
var mid int64
|
||
if err := a.pool.QueryRow(c, `SELECT id FROM materializations WHERE api_name=$1`, name).Scan(&mid); err != nil {
|
||
c.JSON(404, gin.H{"error": "物化不存在"})
|
||
return
|
||
}
|
||
count, duration, err := refreshMaterialization(c.Request.Context(), a, mid, name)
|
||
if err != nil {
|
||
c.JSON(500, gin.H{"error": err.Error()})
|
||
return
|
||
}
|
||
c.JSON(200, gin.H{"ok": true, "rows": count, "duration_ms": duration})
|
||
}
|
||
|
||
// GET /api/v1/materializations/:name/rows
|
||
func (a *app) listMaterializationRows(c *gin.Context) {
|
||
name := c.Param("name")
|
||
limit, _ := strconv.Atoi(c.Query("limit"))
|
||
if limit <= 0 || limit > 1000 {
|
||
limit = 100
|
||
}
|
||
rows, err := a.pool.Query(c,
|
||
`SELECT row_json FROM materialization_rows mr
|
||
JOIN materializations m ON m.id = mr.materialization_id
|
||
WHERE m.api_name=$1 ORDER BY mr.source_pk LIMIT $2`, name, limit)
|
||
if err != nil {
|
||
c.JSON(500, gin.H{"error": err.Error()})
|
||
return
|
||
}
|
||
defer rows.Close()
|
||
var out []map[string]interface{}
|
||
for rows.Next() {
|
||
var j []byte
|
||
if err := rows.Scan(&j); err != nil {
|
||
continue
|
||
}
|
||
var r map[string]interface{}
|
||
_ = json.Unmarshal(j, &r)
|
||
out = append(out, r)
|
||
}
|
||
c.JSON(200, gin.H{"rows": out, "count": len(out)})
|
||
}
|
||
|
||
// startMaterializationScheduler 启后台 ticker,每 60 秒扫一次 interval 到期的物化
|
||
func startMaterializationScheduler(ctx context.Context, a *app) {
|
||
go func() {
|
||
tick := time.NewTicker(60 * time.Second)
|
||
defer tick.Stop()
|
||
for {
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-tick.C:
|
||
rows, err := a.pool.Query(ctx,
|
||
`SELECT id, api_name, refresh_interval_seconds FROM materializations
|
||
WHERE is_active=TRUE AND refresh_strategy='interval' AND refresh_interval_seconds>0`)
|
||
if err != nil {
|
||
log.Printf("[mat scheduler] 扫描失败: %v", err)
|
||
continue
|
||
}
|
||
type cand struct {
|
||
id int64
|
||
name string
|
||
every int
|
||
}
|
||
var cands []cand
|
||
for rows.Next() {
|
||
var c cand
|
||
if err := rows.Scan(&c.id, &c.name, &c.every); err != nil {
|
||
continue
|
||
}
|
||
cands = append(cands, c)
|
||
}
|
||
rows.Close()
|
||
for _, c := range cands {
|
||
// 简单判定:距 last_refresh_at 超过 refresh_interval_seconds 才刷
|
||
var lastRefresh *time.Time
|
||
if err := a.pool.QueryRow(ctx, `SELECT last_refresh_at FROM materializations WHERE id=$1`, c.id).Scan(&lastRefresh); err != nil {
|
||
continue
|
||
}
|
||
if lastRefresh == nil || time.Since(*lastRefresh) > time.Duration(c.every)*time.Second {
|
||
if _, _, err := refreshMaterialization(ctx, a, c.id, c.name); err != nil {
|
||
log.Printf("[mat scheduler] 刷新 %s 失败: %v", c.name, err)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}()
|
||
} |