Files
ssw-ontology/server/handlers_materialization.go
T

359 lines
11 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
// 物化层(P2-1 2026-08-24)
//
// 设计目标:把指定 ObjectType 的 objects 按 projection 投影到 materialization_rows,
// 让大查询走"快读表"而非实时 join properties_json。
//
// 本期落地(精简版):
// - 定义 CRUD:list / create / delete / get
// - 手动刷新:POST /:api_name/refresh
// - 自动刷新:startMaterializationScheduler 启 goroutine,每 60s 扫一次 is_active + interval 到期的物化
// - 物化行查询:GET /:api_name/rows(直接返 materialization_rows)
// - 投影规则(简化版):projection[].property 读 source.properties_json[property],按 transform 转
// 当前支持的 transform:upper / lower / trim / as_string / as_int / as_bool
import (
"context"
"encoding/json"
"fmt"
"log"
"strconv"
"strings"
"time"
"github.com/gin-gonic/gin"
)
type Materialization struct {
ID int64 `json:"id"`
APIName string `json:"api_name"`
DisplayName string `json:"display_name"`
Description string `json:"description"`
SourceObjectType string `json:"source_object_type"`
Projection []map[string]interface{} `json:"projection"`
RefreshStrategy string `json:"refresh_strategy"`
RefreshIntervalSeconds int `json:"refresh_interval_seconds"`
IsActive bool `json:"is_active"`
LastRefreshAt *time.Time `json:"last_refresh_at"`
LastRefreshRows int `json:"last_refresh_rows"`
LastRefreshDurationMs int `json:"last_refresh_duration_ms"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
func scanMatRow(scan func(...any) error) (Materialization, error) {
var m Materialization
var projection []byte
var lastRefresh *time.Time
if err := scan(&m.ID, &m.APIName, &m.DisplayName, &m.Description, &m.SourceObjectType,
&projection, &m.RefreshStrategy, &m.RefreshIntervalSeconds, &m.IsActive,
&lastRefresh, &m.LastRefreshRows, &m.LastRefreshDurationMs,
&m.CreatedAt, &m.UpdatedAt); err != nil {
return m, err
}
m.Projection = []map[string]interface{}{}
_ = json.Unmarshal(projection, &m.Projection)
m.LastRefreshAt = lastRefresh
return m, nil
}
func (a *app) listMaterializations(c *gin.Context) {
rows, err := a.pool.Query(c,
`SELECT id, api_name, display_name, description, source_object_type, projection,
refresh_strategy, refresh_interval_seconds, is_active,
last_refresh_at, last_refresh_rows, last_refresh_duration_ms,
created_at, updated_at
FROM materializations ORDER BY id`)
if err != nil {
c.JSON(500, gin.H{"error": err.Error()})
return
}
defer rows.Close()
out := make([]Materialization, 0)
for rows.Next() {
m, err := scanMatRow(rows.Scan)
if err == nil {
out = append(out, m)
}
}
c.JSON(200, gin.H{"materializations": out, "count": len(out)})
}
type materializationInput struct {
APIName string `json:"api_name"`
DisplayName string `json:"display_name"`
Description string `json:"description"`
SourceObjectType string `json:"source_object_type"`
Projection []map[string]interface{} `json:"projection"`
RefreshStrategy string `json:"refresh_strategy"`
RefreshIntervalSeconds int `json:"refresh_interval_seconds"`
IsActive bool `json:"is_active"`
}
func (a *app) createMaterialization(c *gin.Context) {
var in materializationInput
if err := c.ShouldBindJSON(&in); err != nil {
c.JSON(400, gin.H{"error": err.Error()})
return
}
if in.APIName == "" || in.DisplayName == "" || in.SourceObjectType == "" {
c.JSON(400, gin.H{"error": "api_name/display_name/source_object_type 必填"})
return
}
// 校验 source_object_type 真实存在
var cnt int
if err := a.pool.QueryRow(c, `SELECT count(*) FROM object_types WHERE api_name=$1`, in.SourceObjectType).Scan(&cnt); err != nil || cnt == 0 {
c.JSON(400, gin.H{"error": "source_object_type 不存在: " + in.SourceObjectType})
return
}
if in.RefreshStrategy == "" {
in.RefreshStrategy = "manual"
}
if in.Projection == nil {
in.Projection = []map[string]interface{}{}
}
projJSON := mustJSON(in.Projection)
_, err := a.pool.Exec(c,
`INSERT INTO materializations(api_name,display_name,description,source_object_type,projection,
refresh_strategy,refresh_interval_seconds,is_active)
VALUES($1,$2,$3,$4,$5::jsonb,$6,$7,$8)
ON CONFLICT(api_name) DO UPDATE SET
display_name=EXCLUDED.display_name,
description=EXCLUDED.description,
source_object_type=EXCLUDED.source_object_type,
projection=EXCLUDED.projection,
refresh_strategy=EXCLUDED.refresh_strategy,
refresh_interval_seconds=EXCLUDED.refresh_interval_seconds,
is_active=EXCLUDED.is_active,
updated_at=now()`,
in.APIName, in.DisplayName, in.Description, in.SourceObjectType, projJSON,
in.RefreshStrategy, in.RefreshIntervalSeconds, in.IsActive)
if err != nil {
c.JSON(409, gin.H{"error": "创建失败: " + err.Error()})
return
}
c.JSON(201, gin.H{"ok": true, "api_name": in.APIName})
}
func (a *app) deleteMaterialization(c *gin.Context) {
name := c.Param("name")
tag, err := a.pool.Exec(c, `DELETE FROM materializations WHERE api_name=$1`, name)
if err != nil {
c.JSON(500, gin.H{"error": err.Error()})
return
}
if tag.RowsAffected() == 0 {
c.JSON(404, gin.H{"error": "物化不存在"})
return
}
c.JSON(200, gin.H{"ok": true})
}
// applyTransform 按 projection.transform 简化版规则转换值
func applyTransform(v interface{}, transform string) interface{} {
if v == nil {
return nil
}
s, ok := v.(string)
if !ok {
return v
}
switch transform {
case "upper":
return strings.ToUpper(s)
case "lower":
return strings.ToLower(s)
case "trim":
return strings.TrimSpace(s)
case "as_int":
var i int
_, _ = fmt.Sscanf(s, "%d", &i)
return i
case "as_bool":
return s == "true" || s == "1" || s == "yes"
case "as_string":
fallthrough
default:
return s
}
}
// refreshMaterialization 按 projection 从 source objects 投影到 materialization_rows
// 返回受影响行数 + 耗时。
func refreshMaterialization(ctx context.Context, a *app, matID int64, apiName string) (int, int64, error) {
t0 := time.Now()
// 加载物化定义
var mat Materialization
var projection []byte
err := a.pool.QueryRow(ctx,
`SELECT id, api_name, display_name, description, source_object_type, projection,
refresh_strategy, refresh_interval_seconds, is_active
FROM materializations WHERE id=$1`, matID).
Scan(&mat.ID, &mat.APIName, &mat.DisplayName, &mat.Description, &mat.SourceObjectType,
&projection, &mat.RefreshStrategy, &mat.RefreshIntervalSeconds, &mat.IsActive)
if err != nil {
return 0, 0, fmt.Errorf("加载物化定义失败: %w", err)
}
_ = json.Unmarshal(projection, &mat.Projection)
// 拉源数据
rows, err := a.pool.Query(ctx,
`SELECT pk, properties_json FROM objects WHERE type_api_name=$1`, mat.SourceObjectType)
if err != nil {
return 0, 0, fmt.Errorf("拉源数据失败: %w", err)
}
defer rows.Close()
tx, err := a.pool.Begin(ctx)
if err != nil {
return 0, 0, err
}
defer tx.Rollback(ctx)
count := 0
for rows.Next() {
var pk string
var propsRaw []byte
if err := rows.Scan(&pk, &propsRaw); err != nil {
continue
}
var props map[string]interface{}
_ = json.Unmarshal(propsRaw, &props)
// 按 projection 拼 row_json
rowJSON := map[string]interface{}{}
for _, col := range mat.Projection {
prop, _ := col["property"].(string)
alias, _ := col["alias"].(string)
transform, _ := col["transform"].(string)
if prop == "" {
continue
}
if alias == "" {
alias = prop
}
rowJSON[alias] = applyTransform(props[prop], transform)
}
rowJSON["pk"] = pk
// upsert 到 materialization_rows
_, err := tx.Exec(ctx,
`INSERT INTO materialization_rows(materialization_id,source_pk,source_type,row_json,refreshed_at)
VALUES($1,$2,$3,$4::jsonb,now())
ON CONFLICT(materialization_id,source_pk) DO UPDATE SET
row_json=EXCLUDED.row_json, refreshed_at=now()`,
matID, pk, mat.SourceObjectType, mustJSON(rowJSON))
if err != nil {
return 0, 0, fmt.Errorf("写入物化行失败: %w", err)
}
count++
}
duration := time.Since(t0).Milliseconds()
// 更新物化的 last_refresh_*
_, _ = tx.Exec(ctx,
`UPDATE materializations SET last_refresh_at=now(), last_refresh_rows=$1, last_refresh_duration_ms=$2, updated_at=now() WHERE id=$3`,
count, duration, matID)
if err := tx.Commit(ctx); err != nil {
return 0, 0, err
}
log.Printf("[mat] %s 刷新完成:%d 行,%d ms", apiName, count, duration)
return count, duration, nil
}
// POST /api/v1/materializations/:name/refresh
func (a *app) refreshMaterializationHandler(c *gin.Context) {
name := c.Param("name")
var mid int64
if err := a.pool.QueryRow(c, `SELECT id FROM materializations WHERE api_name=$1`, name).Scan(&mid); err != nil {
c.JSON(404, gin.H{"error": "物化不存在"})
return
}
count, duration, err := refreshMaterialization(c.Request.Context(), a, mid, name)
if err != nil {
c.JSON(500, gin.H{"error": err.Error()})
return
}
c.JSON(200, gin.H{"ok": true, "rows": count, "duration_ms": duration})
}
// GET /api/v1/materializations/:name/rows
func (a *app) listMaterializationRows(c *gin.Context) {
name := c.Param("name")
limit, _ := strconv.Atoi(c.Query("limit"))
if limit <= 0 || limit > 1000 {
limit = 100
}
rows, err := a.pool.Query(c,
`SELECT row_json FROM materialization_rows mr
JOIN materializations m ON m.id = mr.materialization_id
WHERE m.api_name=$1 ORDER BY mr.source_pk LIMIT $2`, name, limit)
if err != nil {
c.JSON(500, gin.H{"error": err.Error()})
return
}
defer rows.Close()
out := make([]map[string]interface{}, 0)
for rows.Next() {
var j []byte
if err := rows.Scan(&j); err != nil {
continue
}
var r map[string]interface{}
_ = json.Unmarshal(j, &r)
out = append(out, r)
}
c.JSON(200, gin.H{"rows": out, "count": len(out)})
}
// startMaterializationScheduler 启后台 ticker,每 60 秒扫一次 interval 到期的物化
func startMaterializationScheduler(ctx context.Context, a *app) {
go func() {
tick := time.NewTicker(60 * time.Second)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
rows, err := a.pool.Query(ctx,
`SELECT id, api_name, refresh_interval_seconds FROM materializations
WHERE is_active=TRUE AND refresh_strategy='interval' AND refresh_interval_seconds>0`)
if err != nil {
log.Printf("[mat scheduler] 扫描失败: %v", err)
continue
}
type cand struct {
id int64
name string
every int
}
var cands []cand
for rows.Next() {
var c cand
if err := rows.Scan(&c.id, &c.name, &c.every); err != nil {
continue
}
cands = append(cands, c)
}
rows.Close()
for _, c := range cands {
// 简单判定:距 last_refresh_at 超过 refresh_interval_seconds 才刷
var lastRefresh *time.Time
if err := a.pool.QueryRow(ctx, `SELECT last_refresh_at FROM materializations WHERE id=$1`, c.id).Scan(&lastRefresh); err != nil {
continue
}
if lastRefresh == nil || time.Since(*lastRefresh) > time.Duration(c.every)*time.Second {
if _, _, err := refreshMaterialization(ctx, a, c.id, c.name); err != nil {
log.Printf("[mat scheduler] 刷新 %s 失败: %v", c.name, err)
}
}
}
}
}
}()
}