一次收口 2026-11-06 的九项能力(CHANGELOG 1.5.4 → 1.8.0): - 自检探针(readiness.go):check_readiness / check_answerable 两个只读探针,把 「本体里没有」「我没权限看」「模型没抽全」三种空结果原因拆开 - 读路径语义钉死(scope.go):标记与分级的可见范围统一由 Scope 产出,实例分级 为空时继承类型默认级(fail-closed),列表与详情口径一致 - 行级可改性轴(object_owner.go):归属只收窄不宽于 grants,所有对象写路径统一 走 canWriteObject,归属查询失败即拒绝 - 公理第 6 类 certification:认证位保护人工策展内容,AI 重摄取不再覆盖已认证条目 - 跨版本结构 diff(schema_diff.go):schema_snapshots 全量快照 + diff 端点,机器 事实与人工策展分组呈现 - 分区维度 zone:分区绑类型、命中任一即可见,读路径一律带 zone 条件 - 统一分页信封(pagination.go):列表回包统一 count/total/limit/offset/has_more/ truncated,total 与 items 同源 - MCP 长任务进度(mcp_progress.go):客户端带 progressToken 才走 SSE 进度帧 - 本体 schema 导入(ontology_import.go):Turtle 导入 + 四道 fail-fast 闸门 + 整事务 + (source, external_id) 幂等 + IRI 往返不变式 MCP 工具清单并入单一注册表(mcp_registry.go),对外声明与实际可调不再各说各话; 工具数 42 → 54(只读 27 / 写入 27),注册表与分发 case 的双向漂移由 AST 单测拦截。 CHANGELOG 编号整理:新「认证位 + 行级归属」条目由 1.6.0 改为 1.5.6(1.6.x 留给 分区与结构 diff),2026-09-20 的 GB/T 规划稿改标「规划稿·未发布」——该能力已并入 v1.5.1,避免与正式发布的版本号重复。 Co-Authored-By: Claude Code <[email protected]>
346 lines
12 KiB
Go
346 lines
12 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/gin-gonic/gin"
|
||
)
|
||
|
||
// ============================================================================
|
||
// 数据备份(v1.3.0,按 schema 镜像)
|
||
//
|
||
// 设计要点(取代旧的 JSONB 方案):
|
||
// 1) 每次备份 = 一个独立 schema `backups_<run_id>`,包含全部 public 业务表
|
||
// 的同结构副本 + 一致的 INSERT…SELECT 数据。
|
||
// 2) 业务 SQL 完全不动:snapshotRouter 给请求开事务 + SET LOCAL search_path,
|
||
// 业务代码里 `SELECT * FROM objects` 会自然命中 `backups_42.objects`。
|
||
// 3) 写路径检测 X-Snapshot-Id + method 非 GET/HEAD → 403 拒绝,保证快照库只读。
|
||
// 4) 不提供 restore 到线上:MCP 客户端拿到 schema 名后,可在另一个 DB 容器恢复
|
||
// 验证;切换 + 比较是更稳的工作流。
|
||
// 5) 不自动备份:每次 create/update/delete 不触发,让客户端掌握节奏。
|
||
// ============================================================================
|
||
|
||
const backupFormatVersion = 1
|
||
|
||
// backupRun 备份元信息(API 响应 / MCP 工具返回结构)。
|
||
type backupRun struct {
|
||
ID int64 `json:"id"`
|
||
Name string `json:"name"`
|
||
Principal string `json:"principal"`
|
||
SchemaName string `json:"schema_name"` // 数据所在的 schema(backups_<id>)
|
||
FormatVersion int `json:"format_version"`
|
||
TableCount int `json:"table_count"`
|
||
RowCount int64 `json:"row_count"`
|
||
SizeBytes int64 `json:"size_bytes"`
|
||
DurationMS int `json:"duration_ms"`
|
||
Notes string `json:"notes"`
|
||
CreatedAt time.Time `json:"created_at"`
|
||
}
|
||
|
||
// snapshotColumnDef 描述单列(CREATE TABLE 用)。
|
||
type snapshotColumnDef struct {
|
||
Name string
|
||
Type string
|
||
Nullable bool
|
||
Default string // 含 DEFAULT 子句("now()" / "'L0'" 等)
|
||
}
|
||
|
||
// collectPublicSchema 读取 public schema 中所有表 + 列结构,用于动态生成备份 DDL。
|
||
//
|
||
// 实现要点:
|
||
// - 从 information_schema.columns 取列定义(类型、可空性、默认值);
|
||
// - data_type 含数组 / 自定义类型时直接原样照搬,PG 接受;
|
||
// - 列顺序 ordinal_position,保证 CREATE TABLE 顺序与原表一致。
|
||
func collectPublicSchema(ctx context.Context, p dbExec) (map[string][]snapshotColumnDef, error) {
|
||
rows, err := p.Query(ctx,
|
||
`SELECT table_name, column_name, data_type, is_nullable, COALESCE(column_default,'')
|
||
FROM information_schema.columns
|
||
WHERE table_schema='public'
|
||
ORDER BY table_name, ordinal_position`)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
out := map[string][]snapshotColumnDef{}
|
||
for rows.Next() {
|
||
var tbl, col, typ, nullable, defv string
|
||
if err := rows.Scan(&tbl, &col, &typ, &nullable, &defv); err != nil {
|
||
return nil, err
|
||
}
|
||
out[tbl] = append(out[tbl], snapshotColumnDef{
|
||
Name: col, Type: typ, Nullable: nullable == "YES", Default: defv,
|
||
})
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// listBackupableTables 返回 public schema 中所有表名(按字母序)。
|
||
func listBackupableTables(ctx context.Context, p dbExec) ([]string, error) {
|
||
rows, err := p.Query(ctx,
|
||
`SELECT table_name FROM information_schema.tables
|
||
WHERE table_schema='public' AND table_type='BASE TABLE'
|
||
ORDER BY table_name`)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []string
|
||
for rows.Next() {
|
||
var name string
|
||
if err := rows.Scan(&name); err == nil {
|
||
out = append(out, name)
|
||
}
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// snapshotDialect data_type → CREATE TABLE 子句的转换。
|
||
//
|
||
// PG 的 information_schema.columns.data_type 是规范化名(integer/text/jsonb 等),
|
||
// 直接拿来当 CREATE TABLE 类型在 90% 场景可用。复杂类型(数组、域)会有惊喜,
|
||
// 备份方案本身不追求 100% 类型完美——快照是只读恢复用,少数边界退回发现再补。
|
||
func snapshotColumnClause(c snapshotColumnDef) string {
|
||
var sb strings.Builder
|
||
sb.WriteString(quoteIdent(c.Name))
|
||
sb.WriteByte(' ')
|
||
sb.WriteString(c.Type)
|
||
if ! c.Nullable {
|
||
sb.WriteString(" NOT NULL")
|
||
}
|
||
if c.Default != "" {
|
||
sb.WriteString(" DEFAULT ")
|
||
sb.WriteString(c.Default)
|
||
}
|
||
return sb.String()
|
||
}
|
||
|
||
// quoteIdent 双引号包裹 identifier,保留大小写与 PG 关键字安全。
|
||
// information_schema 返回的列名都是规范形式(已转小写),引号包裹无副作用。
|
||
func quoteIdent(s string) string {
|
||
return `"` + strings.ReplaceAll(s, `"`, `""`) + `"`
|
||
}
|
||
|
||
// createBackup 创建一份全库 schema 镜像快照。
|
||
//
|
||
// 流程:
|
||
// 1) BEGIN 事务
|
||
// 2) 预分配 run_id(INSERT snapshot_runs RETURNING id),schema 名 = backups_<id>
|
||
// 3) CREATE SCHEMA backups_<id>
|
||
// 4) 对每张业务表:CREATE TABLE backups_<id>.<tbl> (… 同结构 …) +
|
||
// INSERT INTO backups_<id>.<tbl> SELECT * FROM public.<tbl>
|
||
// 5) UPDATE snapshot_runs SET table_count / size_bytes / duration_ms
|
||
// 6) COMMIT
|
||
//
|
||
// 失败时事务回滚 → schema 与元信息都不会留下垃圾。
|
||
func (a *app) createBackup(ctx context.Context, principal, name, notes string) (*backupRun, error) {
|
||
start := time.Now()
|
||
tx, err := a.pool.Begin(ctx)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("开启事务失败: %w", err)
|
||
}
|
||
defer tx.Rollback(ctx)
|
||
|
||
tables, err := listBackupableTables(ctx, a.pool)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("列备份表失败: %w", err)
|
||
}
|
||
schemaCols, err := collectPublicSchema(ctx, a.pool)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("读取列结构失败: %w", err)
|
||
}
|
||
if name == "" {
|
||
name = fmt.Sprintf("backup-%s", start.Format("20060102-150405"))
|
||
}
|
||
|
||
// 预分配 run id → schema 名稳定且可预测
|
||
var runID int64
|
||
if err := tx.QueryRow(ctx,
|
||
`INSERT INTO backups.snapshot_runs(name, principal, schema_name, format_version, table_count, duration_ms, notes)
|
||
VALUES($1, $2, '', $3, $4, 0, $5) RETURNING id`,
|
||
name, principal, backupFormatVersion, len(tables), notes).Scan(&runID); err != nil {
|
||
return nil, fmt.Errorf("创建快照元信息失败: %w", err)
|
||
}
|
||
schemaName := fmt.Sprintf("backups_%d", runID)
|
||
|
||
if _, err := tx.Exec(ctx,
|
||
fmt.Sprintf(`CREATE SCHEMA %s`, quoteIdent(schemaName))); err != nil {
|
||
return nil, fmt.Errorf("建 schema %s 失败: %w", schemaName, err)
|
||
}
|
||
|
||
var totalBytes int64
|
||
var rowCount int64
|
||
for i, tbl := range tables {
|
||
// 进度回报:逐表复制,大表可能很久,客户端不会把「还在跑」当卡死。
|
||
reportProgress(ctx, float64(i), float64(len(tables)), "备份表 "+tbl)
|
||
cols := schemaCols[tbl]
|
||
if len(cols) == 0 {
|
||
continue // 视图或非真实表,跳过
|
||
}
|
||
clauses := make([]string, 0, len(cols))
|
||
for _, c := range cols {
|
||
clauses = append(clauses, snapshotColumnClause(c))
|
||
}
|
||
// CREATE TABLE backups_<id>.<tbl> (col1 type, col2 type ...)
|
||
createSQL := fmt.Sprintf(`CREATE TABLE %s.%s (%s)`,
|
||
quoteIdent(schemaName), quoteIdent(tbl), strings.Join(clauses, ", "))
|
||
if _, err := tx.Exec(ctx, createSQL); err != nil {
|
||
return nil, fmt.Errorf("CREATE %s.%s 失败: %w", schemaName, tbl, err)
|
||
}
|
||
// INSERT…SELECT(同一 schema 内),走大表的 COPY 也行;这里 INSERT 兼容更好
|
||
insertSQL := fmt.Sprintf(`INSERT INTO %s.%s SELECT * FROM public.%s`,
|
||
quoteIdent(schemaName), quoteIdent(tbl), quoteIdent(tbl))
|
||
tag, err := tx.Exec(ctx, insertSQL)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("复制 %s → %s.%s 失败: %w", tbl, schemaName, tbl, err)
|
||
}
|
||
rowsCopied := tag.RowsAffected()
|
||
rowCount += rowsCopied
|
||
|
||
// 表字节数(pg_total_relation_size 包含 TOAST,但用于估算足够)
|
||
var sizeBytes int64
|
||
_ = tx.QueryRow(ctx,
|
||
`SELECT pg_total_relation_size($1::regclass)`,
|
||
fmt.Sprintf(`%s.%s`, schemaName, tbl)).Scan(&sizeBytes)
|
||
totalBytes += sizeBytes
|
||
}
|
||
|
||
durationMS := int(time.Since(start) / time.Millisecond)
|
||
if _, err := tx.Exec(ctx,
|
||
`UPDATE backups.snapshot_runs
|
||
SET schema_name=$1, table_count=$2, row_count=$3, size_bytes=$4, duration_ms=$5
|
||
WHERE id=$6`,
|
||
schemaName, len(tables), rowCount, totalBytes, durationMS, runID); err != nil {
|
||
return nil, fmt.Errorf("更新快照统计失败: %w", err)
|
||
}
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return nil, fmt.Errorf("提交事务失败: %w", err)
|
||
}
|
||
return a.loadBackupRun(ctx, runID)
|
||
}
|
||
|
||
// loadBackupRun 读取一次快照的元信息。
|
||
func (a *app) loadBackupRun(ctx context.Context, runID int64) (*backupRun, error) {
|
||
r := &backupRun{}
|
||
err := a.pool.QueryRow(ctx,
|
||
`SELECT id, name, principal, COALESCE(schema_name,''), format_version, table_count, row_count,
|
||
size_bytes, duration_ms, notes, created_at
|
||
FROM backups.snapshot_runs WHERE id=$1`, runID).
|
||
Scan(&r.ID, &r.Name, &r.Principal, &r.SchemaName, &r.FormatVersion, &r.TableCount, &r.RowCount,
|
||
&r.SizeBytes, &r.DurationMS, &r.Notes, &r.CreatedAt)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return r, nil
|
||
}
|
||
|
||
// listBackups 列出最近的快照。
|
||
func (a *app) listBackups(ctx context.Context, limit, offset int) ([]*backupRun, error) {
|
||
if limit <= 0 || limit > 200 {
|
||
limit = 50
|
||
}
|
||
if offset < 0 {
|
||
offset = 0
|
||
}
|
||
rows, err := a.pool.Query(ctx,
|
||
`SELECT id, name, principal, COALESCE(schema_name,''), format_version, table_count, row_count,
|
||
size_bytes, duration_ms, notes, created_at
|
||
FROM backups.snapshot_runs ORDER BY id DESC LIMIT $1 OFFSET $2`, limit, offset)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []*backupRun
|
||
for rows.Next() {
|
||
r := &backupRun{}
|
||
if err := rows.Scan(&r.ID, &r.Name, &r.Principal, &r.SchemaName, &r.FormatVersion, &r.TableCount,
|
||
&r.RowCount, &r.SizeBytes, &r.DurationMS, &r.Notes, &r.CreatedAt); err != nil {
|
||
return nil, err
|
||
}
|
||
out = append(out, r)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// countBackups 快照总数(分页用)。
|
||
func (a *app) countBackups(ctx context.Context) (int64, error) {
|
||
var n int64
|
||
err := a.pool.QueryRow(ctx, `SELECT count(*) FROM backups.snapshot_runs`).Scan(&n)
|
||
return n, err
|
||
}
|
||
|
||
// deleteBackup 删除一份快照(级联 DROP SCHEMA)。
|
||
func (a *app) deleteBackup(ctx context.Context, runID int64) error {
|
||
var schemaName string
|
||
if err := a.pool.QueryRow(ctx,
|
||
`SELECT COALESCE(schema_name,'') FROM backups.snapshot_runs WHERE id=$1`, runID).Scan(&schemaName); err != nil {
|
||
return fmt.Errorf("快照 #%d 不存在", runID)
|
||
}
|
||
if schemaName == "" {
|
||
return fmt.Errorf("快照 #%d schema 字段为空(异常状态,请联系管理员)", runID)
|
||
}
|
||
// DROP SCHEMA 后元信息行还在 → 还要清理
|
||
if _, err := a.pool.Exec(ctx, fmt.Sprintf(`DROP SCHEMA %s CASCADE`, quoteIdent(schemaName))); err != nil {
|
||
return fmt.Errorf("DROP SCHEMA %s 失败: %w", schemaName, err)
|
||
}
|
||
if _, err := a.pool.Exec(ctx, `DELETE FROM backups.snapshot_runs WHERE id=$1`, runID); err != nil {
|
||
return err
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// resolveSnapshotSchema 把请求里的「快照标识」解析为具体 schema_name。
|
||
//
|
||
// 支持:
|
||
// - X-Snapshot-Id: 42(直接 run id)
|
||
// - X-Snapshot-Ts: 1700000000(秒级时间戳 → created_at ≤ ts 的最近 run)
|
||
// - 都不传 → 返回 "" + nil(按当前活库走)
|
||
func (a *app) resolveSnapshotSchema(ctx context.Context, idStr, tsStr string) (string, int64, error) {
|
||
if idStr != "" {
|
||
var id int64
|
||
var schemaName string
|
||
if err := a.pool.QueryRow(ctx,
|
||
`SELECT id, COALESCE(schema_name,'') FROM backups.snapshot_runs WHERE id=$1`, idStr).Scan(&id, &schemaName); err != nil {
|
||
return "", 0, fmt.Errorf("快照 #%s 不存在", idStr)
|
||
}
|
||
if schemaName == "" {
|
||
return "", 0, fmt.Errorf("快照 #%s schema 字段为空", idStr)
|
||
}
|
||
return schemaName, id, nil
|
||
}
|
||
if tsStr != "" {
|
||
var id int64
|
||
var schemaName string
|
||
// ts 单位是秒(HTTP 头常用),兼容「毫秒时间戳」(>1e12 视为毫秒)
|
||
ts := tsStr
|
||
if len(ts) >= 13 {
|
||
ts = ts[:10]
|
||
}
|
||
if err := a.pool.QueryRow(ctx,
|
||
`SELECT id, COALESCE(schema_name,'') FROM backups.snapshot_runs
|
||
WHERE EXTRACT(EPOCH FROM created_at) <= $1::bigint
|
||
ORDER BY id DESC LIMIT 1`, ts).Scan(&id, &schemaName); err != nil {
|
||
return "", 0, fmt.Errorf("时间戳 %s 之前没有快照", tsStr)
|
||
}
|
||
if schemaName == "" {
|
||
return "", 0, fmt.Errorf("时间戳 %s 最近快照 schema 字段为空", tsStr)
|
||
}
|
||
return schemaName, id, nil
|
||
}
|
||
return "", 0, nil
|
||
}
|
||
|
||
// requireBackupManager 统一鉴权:仅 manager 主体可调备份接口。
|
||
func (a *app) requireBackupManager(c *gin.Context) bool {
|
||
authz, _ := a.authzOf(c)
|
||
if authz != nil && authz.Principal == "manager" {
|
||
return true
|
||
}
|
||
a.abortForbidden(c, "备份接口仅 manager 主体可访问")
|
||
return false
|
||
}
|
||
|
||
// 编译期引用
|
||
var _ = listBackupableTables |