Files
ssw-mvp/scripts/export_j7_platform.py
T
admin 6e7b84e424 chore(db): j7 平台库按应用导出 + 应用库 schema-only + 0810 tag
- docs/db/README.md: 总览文档,说明应用库 schema-only / j7 按应用配置导出的设计原则
- docs/db/0809/j7/  + scripts/export_j7_platform.py:
  jnpf 平台库 j7 共 123 张 base_* 表只导出 schema,
  按 JNPF SystemExportVo 语义导出 SSW 三个应用配置数据
  (base_system 3 + base_module 131 + base_visual_dev/release 各 11)。
- docs/db/0809/{sswflow,tdc,tds}/: 删除 data/,只保留 schema.sql + overview.json
- summary.json: 反映 schema-only / app-config-only 新结构
- dag-panel/human 微调:Demand 表单补需求编码、响应按钮 key 修正、Ctrl+A 模拟人类按键
2026-08-10 09:19:58 +08:00

518 lines
19 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
导出 JNPF 平台库 j7 的结构和"按应用配置"数据备份。
设计原则(参考 jnpf-java-boot/jnpf-system/jnpf-system-controller
SystemController#exportFile 涉及的 SystemExportVo):
- **平台基础表**(base_user/base_role/base_organize/base_*/...共 123 张):
只导出 schema(CREATE TABLE / INDEX / COMMENT),**不**导出数据。
这些是 JNPF 平台自带的组织/权限/账号等数据,与 SSW 应用配置无关,
不应被打包进代码仓库。
- **SSW 三个应用相关表**:按 `f_system_id IN (...)` 或经 base_module 关联导出数据。
这些是平台"应用导入导出"功能会连带备份的内容,是 SSW 应用在平台上的配置。
涉及的应用 systemId:
- sswflow 隐私计算平台:853028206192623813
- 可信数据空间 :855102009768939909
- 数据接入连接器 :855102050973782405
涉及的表(按应用维度):
base_system - 应用主表 (按 systemId)
base_module / base_module_button /
base_module_column / base_module_form /
base_module_data / base_module_authorize /
base_module_scheme / base_module_link - 模块菜单 + 各种权限配置
(按 f_module_id IN 子查询)
base_visual_dev / base_visual_alias /
base_visual_release - 功能设计 + 别名 + 历史版本
base_portal - 门户配置
base_print_template / base_print_version - 打印模板
base_flow_template - 流程模板
输出:
docs/db/0809/j7/
schema.sql - 123 张表结构
overview.json - 列/类型/注释 + 行数(SSW 应用数据行数;其他 0)
data/
base_system.sql - sswflow/tds/tdc 三条应用记录
base_module*.sql - 关联菜单/权限数据
base_visual_*.sql - 功能设计数据
"""
from __future__ import annotations
import json
import os
import sys
from datetime import datetime
from typing import Any
import psycopg2
import psycopg2.extras
# ============================================================================
# 配置
# ============================================================================
DB_CONFIG = {
"host": "127.0.0.1",
"port": 15432,
"user": "dacli",
"password": "dacli123",
"dbname": "j7",
}
SCHEMA = "public"
OUTPUT_DIR = "/Users/xs/Desktop/ssw/docs/db/0809/j7"
# SSW 三个应用的 systemId
SSW_SYSTEM_IDS = (
"853028206192623813", # sswflow
"855102009768939909", # tds
"855102050973782405", # tdc
)
# 应用配置相关表,按 systemId 直接导出
APP_TABLES_BY_SYSTEM_ID = [
"base_system",
"base_module_data", # f_system_id 列
"base_visual_dev",
"base_visual_release",
"base_portal",
"base_print_template",
"base_print_version",
"base_flow_template",
"base_visual_screen",
]
# 按 module_id 间接关联的表(通过 base_module WHERE f_system_id IN (...) 关联)
APP_TABLES_BY_MODULE_ID = [
"base_module",
"base_module_button",
"base_module_column",
"base_module_form",
"base_module_authorize",
"base_module_scheme",
"base_module_link",
"base_visual_alias", # 通过 visual_dev.id 关联
]
# ============================================================================
# 工具函数
# ============================================================================
def connect():
conn = psycopg2.connect(
host=DB_CONFIG["host"],
port=DB_CONFIG["port"],
user=DB_CONFIG["user"],
password=DB_CONFIG["password"],
dbname=DB_CONFIG["dbname"],
)
conn.set_session(readonly=True, autocommit=True)
return conn
def quote_ident(name: str) -> str:
return '"' + name.replace('"', '""') + '"'
def list_tables(conn) -> list[str]:
"""列出 public schema 下所有用户表(base_ 前缀的 JNPF 平台表)。"""
with conn.cursor() as cur:
cur.execute(
"""
SELECT table_name FROM information_schema.tables
WHERE table_schema = %s AND table_type = 'BASE TABLE'
AND table_name LIKE 'base\\_%%'
ORDER BY table_name
""",
(SCHEMA,),
)
return [r[0] for r in cur.fetchall()]
def table_has_column(conn, table: str, column: str) -> bool:
with conn.cursor() as cur:
cur.execute(
"""
SELECT 1 FROM information_schema.columns
WHERE table_schema = %s AND table_name = %s AND column_name = %s
LIMIT 1
""",
(SCHEMA, table, column),
)
return cur.fetchone() is not None
def get_table_columns(conn, table: str) -> list[dict[str, Any]]:
"""获取表的列信息(列名、类型、是否可空、默认值)。"""
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
cur.execute(
"""
SELECT column_name, data_type, udt_name, is_nullable, column_default,
character_maximum_length, numeric_precision, numeric_scale
FROM information_schema.columns
WHERE table_schema = %s AND table_name = %s
ORDER BY ordinal_position
""",
(SCHEMA, table),
)
return list(cur.fetchall())
def get_table_comments(conn, table: str) -> tuple[str | None, dict[str, str]]:
"""获取表注释 + 列注释。"""
table_comment = None
col_comments: dict[str, str] = {}
with conn.cursor() as cur:
cur.execute(
"""
SELECT obj_description(c.oid) AS tbl_comment
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = %s AND c.relname = %s
""",
(SCHEMA, table),
)
row = cur.fetchone()
if row:
table_comment = row[0]
cur.execute(
"""
SELECT a.attname, pg_catalog.col_description(a.attrelid, a.attnum)
FROM pg_attribute a
JOIN pg_class c ON c.oid = a.attrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = %s AND c.relname = %s AND a.attnum > 0 AND NOT a.attisdropped
ORDER BY a.attnum
""",
(SCHEMA, table),
)
for r in cur.fetchall():
if r[1]:
col_comments[r[0]] = r[1]
return table_comment, col_comments
def get_indexes(conn, table: str) -> list[str]:
"""获取表的索引 DDL(不含主键约束,主键会在 CREATE TABLE 中)。"""
with conn.cursor() as cur:
cur.execute(
"""
SELECT indexdef FROM pg_indexes
WHERE schemaname = %s AND tablename = %s
AND indexname NOT IN (
SELECT constraint_name FROM information_schema.table_constraints
WHERE table_schema = %s AND table_name = %s AND constraint_type = 'PRIMARY KEY'
)
ORDER BY indexname
""",
(SCHEMA, table, SCHEMA, table),
)
return [r[0] for r in cur.fetchall()]
def get_pk_constraint(conn, table: str) -> str | None:
"""获取主键约束 DDL(可在 CREATE TABLE 中追加)。"""
with conn.cursor() as cur:
cur.execute(
"""
SELECT pg_get_constraintdef(c.oid), c.conname
FROM pg_constraint c
JOIN pg_class t ON t.oid = c.conrelid
JOIN pg_namespace n ON n.oid = t.relnamespace
WHERE n.nspname = %s AND t.relname = %s AND c.contype = 'p'
""",
(SCHEMA, table),
)
row = cur.fetchone()
if not row:
return None
return f'CONSTRAINT "{row[1]}" {row[0]}'
def get_extra_constraints(conn, table: str) -> list[str]:
"""获取 unique / check 约束 DDL(不含主键、外键)。"""
out: list[str] = []
with conn.cursor() as cur:
cur.execute(
"""
SELECT pg_get_constraintdef(c.oid), c.conname, c.contype
FROM pg_constraint c
JOIN pg_class t ON t.oid = c.conrelid
JOIN pg_namespace n ON n.oid = t.relnamespace
WHERE n.nspname = %s AND t.relname = %s AND c.contype IN ('u','c')
ORDER BY c.contype, c.conname
""",
(SCHEMA, table),
)
for cdef, cname, _ctype in cur.fetchall():
out.append(f'ALTER TABLE {quote_ident(table)} ADD CONSTRAINT "{cname}" {cdef};')
return out
def get_pg_size_pretty(conn, table: str) -> str:
with conn.cursor() as cur:
cur.execute(
"SELECT pg_size_pretty(pg_total_relation_size(%s))",
(f"{SCHEMA}.{table}",),
)
return cur.fetchone()[0] or "0 bytes"
def count_rows(conn, table: str, where: str = "", params: tuple = ()) -> int:
"""统计行数。"""
sql = f"SELECT count(*) FROM {quote_ident(table)}"
if where:
sql += " WHERE " + where
with conn.cursor() as cur:
cur.execute(sql, params)
return cur.fetchone()[0]
def escape_sql_value(v: Any) -> str:
"""把 Python 值转成 SQL 字面量。"""
if v is None:
return "NULL"
if isinstance(v, bool):
return "TRUE" if v else "FALSE"
if isinstance(v, (int, float)):
return str(v)
if isinstance(v, datetime):
return f"'{v.strftime('%Y-%m-%d %H:%M:%S')}'"
s = str(v).replace("'", "''")
return f"'{s}'"
# ============================================================================
# Schema 输出
# ============================================================================
def write_table_schema(f, table: str, columns: list[dict], table_comment: str | None, pk_constraint: str | None = None):
"""把一张表的 CREATE TABLE 写到文件。"""
f.write(f"\n-- Table: {table}\n")
f.write(f"CREATE TABLE {quote_ident(table)} (\n")
col_lines = []
for col in columns:
parts = [f" {quote_ident(col['column_name'])}"]
# 类型
if col["character_maximum_length"]:
parts.append(f"varchar({col['character_maximum_length']})")
elif col["data_type"] == "integer":
parts.append("integer")
elif col["data_type"] == "bigint":
parts.append("bigint")
elif col["data_type"] == "smallint":
parts.append("smallint")
elif col["data_type"] == "numeric":
parts.append(
f"numeric({col['numeric_precision']},{col['numeric_scale']})"
)
elif col["data_type"] == "timestamp without time zone":
parts.append("timestamp without time zone")
elif col["data_type"] == "text":
parts.append("text")
elif col["data_type"] == "boolean":
parts.append("boolean")
elif col["data_type"] == "double precision":
parts.append("double precision")
elif col["data_type"] == "real":
parts.append("real")
elif col["data_type"] == "json":
parts.append("json")
elif col["data_type"] == "jsonb":
parts.append("jsonb")
else:
parts.append(col["udt_name"])
if col["is_nullable"] == "NO":
parts.append("NOT NULL")
if col["column_default"] is not None:
parts.append(f"DEFAULT {col['column_default']}")
col_lines.append(" ".join(parts))
if pk_constraint:
col_lines.append(f" {pk_constraint}")
f.write(",\n".join(col_lines))
f.write("\n);\n\n")
if table_comment:
f.write(f"COMMENT ON TABLE {quote_ident(table)} IS '{table_comment.replace(chr(39), chr(39)*2)}';\n")
def write_indexes(f, table: str, indexes: list[str]):
for idx in indexes:
f.write(idx + ";\n")
def write_column_comments(f, table: str, col_comments: dict[str, str]):
for col, comment in col_comments.items():
safe = comment.replace("'", "''")
f.write(f"COMMENT ON COLUMN {quote_ident(table)}.{quote_ident(col)} IS '{safe}';\n")
# ============================================================================
# 数据导出(按应用维度)
# ============================================================================
def export_app_data(conn, table: str, where: str, params: tuple, out_path: str):
"""把符合 where 的行导出成 INSERT 语句。"""
with conn.cursor() as cur:
cur.execute(
f"SELECT * FROM {quote_ident(table)} WHERE {where}",
params,
)
cols = [d[0] for d in cur.description]
rows = cur.fetchall()
with open(out_path, "w", encoding="utf-8") as f:
f.write(f"-- j7.{table} rows={len(rows)} exported={datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
if not rows:
return 0
f.write("BEGIN;\n")
col_list = ",".join(quote_ident(c) for c in cols)
for row in rows:
values = ",".join(escape_sql_value(v) for v in row)
f.write(f"INSERT INTO {quote_ident(table)} ({col_list}) VALUES ({values});\n")
f.write("COMMIT;\n")
return len(rows)
# ============================================================================
# 主流程
# ============================================================================
def main():
os.makedirs(OUTPUT_DIR, exist_ok=True)
os.makedirs(os.path.join(OUTPUT_DIR, "data"), exist_ok=True)
conn = connect()
tables = list_tables(conn)
print(f"Found {len(tables)} JNPF platform tables in j7.{SCHEMA}")
overview: dict[str, Any] = {
"database": "j7",
"schema": SCHEMA,
"note": "JNPF 平台库按 SSW 应用配置导出:所有 123 张表只导出 schema;仅 SSW 三个应用(sswflow/tds/tdc)相关的 base_module / base_visual_dev / base_system 等表导出数据。",
"ssw_system_ids": list(SSW_SYSTEM_IDS),
"exported_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"tables": [],
"data_tables": [],
}
schema_path = os.path.join(OUTPUT_DIR, "schema.sql")
with open(schema_path, "w", encoding="utf-8") as f:
f.write(f"""-- ============================================================
-- Database: j7 (JNPF 7.0 平台主库)
-- Schema snapshot at {overview['exported_at']}
-- 仅导出表结构(CREATE TABLE / INDEX / COMMENT),不导出业务数据。
-- 应用相关数据见 data/ 子目录(按 SSW 三个应用 systemId 过滤)。
-- ============================================================
SET client_min_messages = warning;
""")
# 1. 所有表的 schema
for table in tables:
cols = get_table_columns(conn, table)
tbl_comment, col_comments = get_table_comments(conn, table)
indexes = get_indexes(conn, table)
pk = get_pk_constraint(conn, table)
extras = get_extra_constraints(conn, table)
size = get_pg_size_pretty(conn, table)
with open(schema_path, "a", encoding="utf-8") as f:
f.write(f"\n-- ----------------------------------------------------------\n")
f.write(f"-- Table: {table} ({size})\n")
f.write(f"-- ----------------------------------------------------------\n")
write_table_schema(f, table, cols, tbl_comment, pk)
if indexes:
write_indexes(f, table, indexes)
if extras:
f.write("\n")
f.write("\n".join(extras) + "\n")
if col_comments:
write_column_comments(f, table, col_comments)
overview["tables"].append({
"name": table,
"size": size,
"columns": [c["column_name"] for c in cols],
"table_comment": tbl_comment,
})
# 2. 应用相关数据导出
print("\nExporting SSW application data...")
sys_ids_str = ",".join(f"'{s}'" for s in SSW_SYSTEM_IDS)
module_ids_subquery = (
f"(SELECT f_id FROM {quote_ident('base_module')} "
f"WHERE f_system_id IN ({sys_ids_str}))"
)
visual_ids_subquery = (
f"(SELECT f_id FROM {quote_ident('base_visual_dev')} "
f"WHERE f_system_id IN ({sys_ids_str}))"
)
def emit_data(table: str, where: str, params: tuple):
out_path = os.path.join(OUTPUT_DIR, "data", f"{table}.sql")
rows = export_app_data(conn, table, where, params, out_path)
overview["data_tables"].append({
"table": table,
"rows": rows,
"path": f"data/{table}.sql",
"filter": where,
})
print(f" {table:30s} rows={rows}")
return rows
# base_system(应用主表,只导出 SSW 三条)
emit_data("base_system", f"f_id IN ({sys_ids_str})", ())
# base_module(菜单),按 systemId 过滤
emit_data("base_module", f"f_system_id IN ({sys_ids_str})", ())
# base_module_data / base_visual_dev / base_portal / ... 按 systemId
# 只处理 j7 库实际存在的表,避免 flowable 子系统的表不存在导致崩溃
CANDIDATE_BY_SYSTEM_ID = [
"base_module_data", "base_visual_dev", "base_visual_release",
"base_portal", "base_print_template",
]
for t in CANDIDATE_BY_SYSTEM_ID:
if t not in tables:
print(f" {t:30s} skip (table does not exist in j7)")
continue
if not table_has_column(conn, t, "f_system_id"):
print(f" {t:30s} skip (no f_system_id column)")
continue
emit_data(t, f"f_system_id IN ({sys_ids_str})", ())
# base_print_version 经 base_print_template 关联
if "base_print_version" in tables:
print_tpl_sub = (
f"(SELECT f_id FROM {quote_ident('base_print_template')} "
f"WHERE f_system_id IN ({sys_ids_str}))"
)
emit_data("base_print_version", f"f_template_id IN {print_tpl_sub}", ())
# base_module_button/column/form/authorize/scheme/link 按 module_id 关联
for t in ["base_module_button", "base_module_column", "base_module_form",
"base_module_authorize", "base_module_scheme", "base_module_link"]:
try:
emit_data(t, f"f_module_id IN {module_ids_subquery}", ())
except psycopg2.errors.UndefinedColumn:
print(f" {t:30s} skip (no f_module_id column)")
# base_visual_alias 通过 visual_dev.id 关联
if "base_visual_alias" in tables and table_has_column(conn, "base_visual_alias", "f_visual_id"):
emit_data("base_visual_alias", f"f_visual_id IN {visual_ids_subquery}", ())
# overview.json
with open(os.path.join(OUTPUT_DIR, "overview.json"), "w", encoding="utf-8") as f:
json.dump(overview, f, ensure_ascii=False, indent=2)
conn.close()
print(f"\nDone. {len(tables)} tables schema + {len(overview['data_tables'])} app-data tables.")
print(f" schema.sql {schema_path}")
print(f" overview.json {os.path.join(OUTPUT_DIR, 'overview.json')}")
print(f" data/ {os.path.join(OUTPUT_DIR, 'data')}")
if __name__ == "__main__":
main()