「让客户端敢操作」的关键能力:写前/写后打一份快照 → 任意 GET 带 X-Snapshot-Id
头即可命中 backups_<id> schema 的同名表(业务 SQL 不带 schema 前缀照样切)。
设计要点:
- 备份 = 独立 schema backups_<id>(真表,不是单存 JSONB)—— 查询性能与活库一致
- snapshotRouter 中间件:BeginTx + SET LOCAL search_path → 把 a.pool 临时换
为事务视图 txPool → handler 结束 commit + 还原 a.pool。业务代码一行不改。
- a.pool 字段类型从 *pgxpool.Pool 升级为 dbRunner interface(中间件可换 txPool)。
- 11 个 helper 函数签名同步改为 dbExec(与现有 dbExec 共存,无破坏性)。
- 写方法(POST/PUT/PATCH/DELETE)带快照头 → 403 拒绝(mistake-safety)。
接口:
- POST /api/v1/backups 创建
- GET /api/v1/backups 列表
- GET /api/v1/backups/:id 元信息(含 schema_name)
- GET /api/v1/backups/:id/switch 切库提示
- DELETE /api/v1/backups/:id 级联删 schema
- 支持 X-Snapshot-Id (直接 run id) + X-Snapshot-Ts (秒级时间戳) 两种切换入口
MCP 工具 (5 个):backup_create / backup_list / backup_inspect / backup_switch / backup_delete
测试 (22 个全绿,~5.5s 集成):
- SnapshotRouter:改前建快照 → 改后读快照只见改前
- SnapshotSearchPathPersist:不污染下一个请求
- SnapshotByTimestamp:ts 头也能切库
- SnapshotDeleteCleansSchema:delete 真 DROP schema
- BackupLifecycle / BackupDiffAfterMutation:完整 create→list→inspect→switch→delete
文档:docs/BACKUP.md 写清用法、机制、客户端工作流;CHANGELOG.md [1.3.0]。
344 lines
12 KiB
Go
344 lines
12 KiB
Go
package main
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/gin-gonic/gin"
|
||
)
|
||
|
||
// ============================================================================
|
||
// 数据备份(v1.3.0,按 schema 镜像)
|
||
//
|
||
// 设计要点(取代旧的 JSONB 方案):
|
||
// 1) 每次备份 = 一个独立 schema `backups_<run_id>`,包含全部 public 业务表
|
||
// 的同结构副本 + 一致的 INSERT…SELECT 数据。
|
||
// 2) 业务 SQL 完全不动:snapshotRouter 给请求开事务 + SET LOCAL search_path,
|
||
// 业务代码里 `SELECT * FROM objects` 会自然命中 `backups_42.objects`。
|
||
// 3) 写路径检测 X-Snapshot-Id + method 非 GET/HEAD → 403 拒绝,保证快照库只读。
|
||
// 4) 不提供 restore 到线上:MCP 客户端拿到 schema 名后,可在另一个 DB 容器恢复
|
||
// 验证;切换 + 比较是更稳的工作流。
|
||
// 5) 不自动备份:每次 create/update/delete 不触发,让客户端掌握节奏。
|
||
// ============================================================================
|
||
|
||
const backupFormatVersion = 1
|
||
|
||
// backupRun 备份元信息(API 响应 / MCP 工具返回结构)。
|
||
type backupRun struct {
|
||
ID int64 `json:"id"`
|
||
Name string `json:"name"`
|
||
Principal string `json:"principal"`
|
||
SchemaName string `json:"schema_name"` // 数据所在的 schema(backups_<id>)
|
||
FormatVersion int `json:"format_version"`
|
||
TableCount int `json:"table_count"`
|
||
RowCount int64 `json:"row_count"`
|
||
SizeBytes int64 `json:"size_bytes"`
|
||
DurationMS int `json:"duration_ms"`
|
||
Notes string `json:"notes"`
|
||
CreatedAt time.Time `json:"created_at"`
|
||
}
|
||
|
||
// snapshotColumnDef 描述单列(CREATE TABLE 用)。
|
||
type snapshotColumnDef struct {
|
||
Name string
|
||
Type string
|
||
Nullable bool
|
||
Default string // 含 DEFAULT 子句("now()" / "'L0'" 等)
|
||
}
|
||
|
||
// collectPublicSchema 读取 public schema 中所有表 + 列结构,用于动态生成备份 DDL。
|
||
//
|
||
// 实现要点:
|
||
// - 从 information_schema.columns 取列定义(类型、可空性、默认值);
|
||
// - data_type 含数组 / 自定义类型时直接原样照搬,PG 接受;
|
||
// - 列顺序 ordinal_position,保证 CREATE TABLE 顺序与原表一致。
|
||
func collectPublicSchema(ctx context.Context, p dbExec) (map[string][]snapshotColumnDef, error) {
|
||
rows, err := p.Query(ctx,
|
||
`SELECT table_name, column_name, data_type, is_nullable, COALESCE(column_default,'')
|
||
FROM information_schema.columns
|
||
WHERE table_schema='public'
|
||
ORDER BY table_name, ordinal_position`)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
out := map[string][]snapshotColumnDef{}
|
||
for rows.Next() {
|
||
var tbl, col, typ, nullable, defv string
|
||
if err := rows.Scan(&tbl, &col, &typ, &nullable, &defv); err != nil {
|
||
return nil, err
|
||
}
|
||
out[tbl] = append(out[tbl], snapshotColumnDef{
|
||
Name: col, Type: typ, Nullable: nullable == "YES", Default: defv,
|
||
})
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// listBackupableTables 返回 public schema 中所有表名(按字母序)。
|
||
func listBackupableTables(ctx context.Context, p dbExec) ([]string, error) {
|
||
rows, err := p.Query(ctx,
|
||
`SELECT table_name FROM information_schema.tables
|
||
WHERE table_schema='public' AND table_type='BASE TABLE'
|
||
ORDER BY table_name`)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []string
|
||
for rows.Next() {
|
||
var name string
|
||
if err := rows.Scan(&name); err == nil {
|
||
out = append(out, name)
|
||
}
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// snapshotDialect data_type → CREATE TABLE 子句的转换。
|
||
//
|
||
// PG 的 information_schema.columns.data_type 是规范化名(integer/text/jsonb 等),
|
||
// 直接拿来当 CREATE TABLE 类型在 90% 场景可用。复杂类型(数组、域)会有惊喜,
|
||
// 备份方案本身不追求 100% 类型完美——快照是只读恢复用,少数边界退回发现再补。
|
||
func snapshotColumnClause(c snapshotColumnDef) string {
|
||
var sb strings.Builder
|
||
sb.WriteString(quoteIdent(c.Name))
|
||
sb.WriteByte(' ')
|
||
sb.WriteString(c.Type)
|
||
if ! c.Nullable {
|
||
sb.WriteString(" NOT NULL")
|
||
}
|
||
if c.Default != "" {
|
||
sb.WriteString(" DEFAULT ")
|
||
sb.WriteString(c.Default)
|
||
}
|
||
return sb.String()
|
||
}
|
||
|
||
// quoteIdent 双引号包裹 identifier,保留大小写与 PG 关键字安全。
|
||
// information_schema 返回的列名都是规范形式(已转小写),引号包裹无副作用。
|
||
func quoteIdent(s string) string {
|
||
return `"` + strings.ReplaceAll(s, `"`, `""`) + `"`
|
||
}
|
||
|
||
// createBackup 创建一份全库 schema 镜像快照。
|
||
//
|
||
// 流程:
|
||
// 1) BEGIN 事务
|
||
// 2) 预分配 run_id(INSERT snapshot_runs RETURNING id),schema 名 = backups_<id>
|
||
// 3) CREATE SCHEMA backups_<id>
|
||
// 4) 对每张业务表:CREATE TABLE backups_<id>.<tbl> (… 同结构 …) +
|
||
// INSERT INTO backups_<id>.<tbl> SELECT * FROM public.<tbl>
|
||
// 5) UPDATE snapshot_runs SET table_count / size_bytes / duration_ms
|
||
// 6) COMMIT
|
||
//
|
||
// 失败时事务回滚 → schema 与元信息都不会留下垃圾。
|
||
func (a *app) createBackup(ctx context.Context, principal, name, notes string) (*backupRun, error) {
|
||
start := time.Now()
|
||
tx, err := a.pool.Begin(ctx)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("开启事务失败: %w", err)
|
||
}
|
||
defer tx.Rollback(ctx)
|
||
|
||
tables, err := listBackupableTables(ctx, a.pool)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("列备份表失败: %w", err)
|
||
}
|
||
schemaCols, err := collectPublicSchema(ctx, a.pool)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("读取列结构失败: %w", err)
|
||
}
|
||
if name == "" {
|
||
name = fmt.Sprintf("backup-%s", start.Format("20060102-150405"))
|
||
}
|
||
|
||
// 预分配 run id → schema 名稳定且可预测
|
||
var runID int64
|
||
if err := tx.QueryRow(ctx,
|
||
`INSERT INTO backups.snapshot_runs(name, principal, schema_name, format_version, table_count, duration_ms, notes)
|
||
VALUES($1, $2, '', $3, $4, 0, $5) RETURNING id`,
|
||
name, principal, backupFormatVersion, len(tables), notes).Scan(&runID); err != nil {
|
||
return nil, fmt.Errorf("创建快照元信息失败: %w", err)
|
||
}
|
||
schemaName := fmt.Sprintf("backups_%d", runID)
|
||
|
||
if _, err := tx.Exec(ctx,
|
||
fmt.Sprintf(`CREATE SCHEMA %s`, quoteIdent(schemaName))); err != nil {
|
||
return nil, fmt.Errorf("建 schema %s 失败: %w", schemaName, err)
|
||
}
|
||
|
||
var totalBytes int64
|
||
var rowCount int64
|
||
for _, tbl := range tables {
|
||
cols := schemaCols[tbl]
|
||
if len(cols) == 0 {
|
||
continue // 视图或非真实表,跳过
|
||
}
|
||
clauses := make([]string, 0, len(cols))
|
||
for _, c := range cols {
|
||
clauses = append(clauses, snapshotColumnClause(c))
|
||
}
|
||
// CREATE TABLE backups_<id>.<tbl> (col1 type, col2 type ...)
|
||
createSQL := fmt.Sprintf(`CREATE TABLE %s.%s (%s)`,
|
||
quoteIdent(schemaName), quoteIdent(tbl), strings.Join(clauses, ", "))
|
||
if _, err := tx.Exec(ctx, createSQL); err != nil {
|
||
return nil, fmt.Errorf("CREATE %s.%s 失败: %w", schemaName, tbl, err)
|
||
}
|
||
// INSERT…SELECT(同一 schema 内),走大表的 COPY 也行;这里 INSERT 兼容更好
|
||
insertSQL := fmt.Sprintf(`INSERT INTO %s.%s SELECT * FROM public.%s`,
|
||
quoteIdent(schemaName), quoteIdent(tbl), quoteIdent(tbl))
|
||
tag, err := tx.Exec(ctx, insertSQL)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("复制 %s → %s.%s 失败: %w", tbl, schemaName, tbl, err)
|
||
}
|
||
rowsCopied := tag.RowsAffected()
|
||
rowCount += rowsCopied
|
||
|
||
// 表字节数(pg_total_relation_size 包含 TOAST,但用于估算足够)
|
||
var sizeBytes int64
|
||
_ = tx.QueryRow(ctx,
|
||
`SELECT pg_total_relation_size($1::regclass)`,
|
||
fmt.Sprintf(`%s.%s`, schemaName, tbl)).Scan(&sizeBytes)
|
||
totalBytes += sizeBytes
|
||
}
|
||
|
||
durationMS := int(time.Since(start) / time.Millisecond)
|
||
if _, err := tx.Exec(ctx,
|
||
`UPDATE backups.snapshot_runs
|
||
SET schema_name=$1, table_count=$2, row_count=$3, size_bytes=$4, duration_ms=$5
|
||
WHERE id=$6`,
|
||
schemaName, len(tables), rowCount, totalBytes, durationMS, runID); err != nil {
|
||
return nil, fmt.Errorf("更新快照统计失败: %w", err)
|
||
}
|
||
if err := tx.Commit(ctx); err != nil {
|
||
return nil, fmt.Errorf("提交事务失败: %w", err)
|
||
}
|
||
return a.loadBackupRun(ctx, runID)
|
||
}
|
||
|
||
// loadBackupRun 读取一次快照的元信息。
|
||
func (a *app) loadBackupRun(ctx context.Context, runID int64) (*backupRun, error) {
|
||
r := &backupRun{}
|
||
err := a.pool.QueryRow(ctx,
|
||
`SELECT id, name, principal, COALESCE(schema_name,''), format_version, table_count, row_count,
|
||
size_bytes, duration_ms, notes, created_at
|
||
FROM backups.snapshot_runs WHERE id=$1`, runID).
|
||
Scan(&r.ID, &r.Name, &r.Principal, &r.SchemaName, &r.FormatVersion, &r.TableCount, &r.RowCount,
|
||
&r.SizeBytes, &r.DurationMS, &r.Notes, &r.CreatedAt)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return r, nil
|
||
}
|
||
|
||
// listBackups 列出最近的快照。
|
||
func (a *app) listBackups(ctx context.Context, limit, offset int) ([]*backupRun, error) {
|
||
if limit <= 0 || limit > 200 {
|
||
limit = 50
|
||
}
|
||
if offset < 0 {
|
||
offset = 0
|
||
}
|
||
rows, err := a.pool.Query(ctx,
|
||
`SELECT id, name, principal, COALESCE(schema_name,''), format_version, table_count, row_count,
|
||
size_bytes, duration_ms, notes, created_at
|
||
FROM backups.snapshot_runs ORDER BY id DESC LIMIT $1 OFFSET $2`, limit, offset)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer rows.Close()
|
||
var out []*backupRun
|
||
for rows.Next() {
|
||
r := &backupRun{}
|
||
if err := rows.Scan(&r.ID, &r.Name, &r.Principal, &r.SchemaName, &r.FormatVersion, &r.TableCount,
|
||
&r.RowCount, &r.SizeBytes, &r.DurationMS, &r.Notes, &r.CreatedAt); err != nil {
|
||
return nil, err
|
||
}
|
||
out = append(out, r)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// countBackups 快照总数(分页用)。
|
||
func (a *app) countBackups(ctx context.Context) (int64, error) {
|
||
var n int64
|
||
err := a.pool.QueryRow(ctx, `SELECT count(*) FROM backups.snapshot_runs`).Scan(&n)
|
||
return n, err
|
||
}
|
||
|
||
// deleteBackup 删除一份快照(级联 DROP SCHEMA)。
|
||
func (a *app) deleteBackup(ctx context.Context, runID int64) error {
|
||
var schemaName string
|
||
if err := a.pool.QueryRow(ctx,
|
||
`SELECT COALESCE(schema_name,'') FROM backups.snapshot_runs WHERE id=$1`, runID).Scan(&schemaName); err != nil {
|
||
return fmt.Errorf("快照 #%d 不存在", runID)
|
||
}
|
||
if schemaName == "" {
|
||
return fmt.Errorf("快照 #%d schema 字段为空(异常状态,请联系管理员)", runID)
|
||
}
|
||
// DROP SCHEMA 后元信息行还在 → 还要清理
|
||
if _, err := a.pool.Exec(ctx, fmt.Sprintf(`DROP SCHEMA %s CASCADE`, quoteIdent(schemaName))); err != nil {
|
||
return fmt.Errorf("DROP SCHEMA %s 失败: %w", schemaName, err)
|
||
}
|
||
if _, err := a.pool.Exec(ctx, `DELETE FROM backups.snapshot_runs WHERE id=$1`, runID); err != nil {
|
||
return err
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// resolveSnapshotSchema 把请求里的「快照标识」解析为具体 schema_name。
|
||
//
|
||
// 支持:
|
||
// - X-Snapshot-Id: 42(直接 run id)
|
||
// - X-Snapshot-Ts: 1700000000(秒级时间戳 → created_at ≤ ts 的最近 run)
|
||
// - 都不传 → 返回 "" + nil(按当前活库走)
|
||
func (a *app) resolveSnapshotSchema(ctx context.Context, idStr, tsStr string) (string, int64, error) {
|
||
if idStr != "" {
|
||
var id int64
|
||
var schemaName string
|
||
if err := a.pool.QueryRow(ctx,
|
||
`SELECT id, COALESCE(schema_name,'') FROM backups.snapshot_runs WHERE id=$1`, idStr).Scan(&id, &schemaName); err != nil {
|
||
return "", 0, fmt.Errorf("快照 #%s 不存在", idStr)
|
||
}
|
||
if schemaName == "" {
|
||
return "", 0, fmt.Errorf("快照 #%s schema 字段为空", idStr)
|
||
}
|
||
return schemaName, id, nil
|
||
}
|
||
if tsStr != "" {
|
||
var id int64
|
||
var schemaName string
|
||
// ts 单位是秒(HTTP 头常用),兼容「毫秒时间戳」(>1e12 视为毫秒)
|
||
ts := tsStr
|
||
if len(ts) >= 13 {
|
||
ts = ts[:10]
|
||
}
|
||
if err := a.pool.QueryRow(ctx,
|
||
`SELECT id, COALESCE(schema_name,'') FROM backups.snapshot_runs
|
||
WHERE EXTRACT(EPOCH FROM created_at) <= $1::bigint
|
||
ORDER BY id DESC LIMIT 1`, ts).Scan(&id, &schemaName); err != nil {
|
||
return "", 0, fmt.Errorf("时间戳 %s 之前没有快照", tsStr)
|
||
}
|
||
if schemaName == "" {
|
||
return "", 0, fmt.Errorf("时间戳 %s 最近快照 schema 字段为空", tsStr)
|
||
}
|
||
return schemaName, id, nil
|
||
}
|
||
return "", 0, nil
|
||
}
|
||
|
||
// requireBackupManager 统一鉴权:仅 manager 主体可调备份接口。
|
||
func (a *app) requireBackupManager(c *gin.Context) bool {
|
||
authz, _ := a.authzOf(c)
|
||
if authz != nil && authz.Principal == "manager" {
|
||
return true
|
||
}
|
||
a.abortForbidden(c, "备份接口仅 manager 主体可访问")
|
||
return false
|
||
}
|
||
|
||
// 编译期引用
|
||
var _ = listBackupableTables |