fix(db): 切分器支持块注释与 dollar-quoting,生成器先校验后落盘,合并 valid_table 单一来源
- db: split_sql_statements 支持 dollar-quoting($$ / $tag$)与 E'' 反斜杠转义 - db: has_code 正确剔除 /* */ 块注释,纯注释块不再当成语句发给 PG - db: 新增 5 个测试(块注释、dollar-quote、tagged dollar-quote、E 串转义) - gen-migrate.sh: 生成物先落一次性临时目录,表数校验通过后才覆盖 static/migrate.sql - gen-migrate.sh: 目标库 public schema 非空即快速失败(不做未授权的 DROP/重建) - reset-data.sh: 回收站目标名加 PID + 去重后缀,避免 mv 嵌套 - main.rs: 删除本地 valid_table 副本,统一走 tdc_connector::db::valid_table Co-Authored-By: Claude Code <[email protected]>
This commit is contained in:
1 parent
1d26a106f5
commit
4ae8054e68
4 files changed
+237
-35
No files matched your search
+37
-11
@@ -6,6 +6,11 @@
|
||||
# tdc_notify 等表并 ALTER tdc_delivery_task / tdc_resource / tdc_product。
|
||||
# ⚠️ V2.10.0 版本号重复(两个文件)⇒ 不能靠 Flyway 排序,必须显式按文件名兜底。
|
||||
#
|
||||
# 🔴 本脚本**不会** DROP / 重建目标库:目标库 public schema 非空时直接快速失败,
|
||||
# 由调用方自己准备空库(未授权的破坏性数据库操作不做)。
|
||||
# 🔴 自检在**落盘之前**做:生成物先写进一次性临时目录,表数校验通过后才覆盖 static/migrate.sql,
|
||||
# 避免一次顺序错误的运行把已入库的 schema 写坏。
|
||||
#
|
||||
# 用法:
|
||||
# UPSTREAM_REPO=/path/to/ssw-tdc \
|
||||
# PGHOST=127.0.0.1 PGPORT=15432 PGUSER=postgres PGDATABASE=tdc_gen \
|
||||
@@ -15,8 +20,9 @@ set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
UPSTREAM_REPO="${UPSTREAM_REPO:-/tmp/sswtdc-probe}"
|
||||
OUT="${ROOT}/static/migrate.sql"
|
||||
WORK="${ROOT}/target/gen-migrate"
|
||||
mkdir -p "${WORK}"
|
||||
mkdir -p "${ROOT}/target"
|
||||
# 一次性工作目录:失败时保留,便于排查(不做 rm)
|
||||
WORK="$(mktemp -d "${ROOT}/target/gen-migrate.XXXXXX")"
|
||||
|
||||
if [ ! -d "${UPSTREAM_REPO}/boot/db" ]; then
|
||||
echo "[gen-migrate] 上游仓库不存在:${UPSTREAM_REPO}" >&2
|
||||
@@ -26,7 +32,21 @@ fi
|
||||
command -v psql >/dev/null 2>&1 || { echo "[gen-migrate] 需要 psql(PostgreSQL 客户端)" >&2; exit 1; }
|
||||
command -v pg_dump >/dev/null 2>&1 || { echo "[gen-migrate] 需要 pg_dump" >&2; exit 1; }
|
||||
|
||||
# ① 收集迁移并排序(版本三元组 + 文件名兜底)
|
||||
# ① 预检:目标库 public schema 必须为空(本脚本不替调用方清理数据库)
|
||||
EXISTING="$(psql -v ON_ERROR_STOP=1 -tAc \
|
||||
"SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'public';" \
|
||||
| tr -d '[:space:]')"
|
||||
if [ -z "${EXISTING}" ]; then
|
||||
echo "[gen-migrate] 无法确认目标库状态(psql 无输出),已中止。" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "${EXISTING}" != "0" ]; then
|
||||
echo "[gen-migrate] 目标库 public schema 已有 ${EXISTING} 个对象,已中止。" >&2
|
||||
echo "[gen-migrate] 请自行准备一个空库(例如新建 tdc_gen)后再跑;本脚本不会 DROP/重建数据库。" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ② 收集迁移并排序(版本三元组 + 文件名兜底)
|
||||
{
|
||||
ls -1 "${UPSTREAM_REPO}"/boot/db/tdc/V*.sql 2>/dev/null || true
|
||||
ls -1 "${UPSTREAM_REPO}"/boot/db/V*.sql 2>/dev/null || true
|
||||
@@ -42,26 +62,32 @@ command -v pg_dump >/dev/null 2>&1 || { echo "[gen-migrate] 需要 pg_dump" >&2;
|
||||
COUNT="$(wc -l < "${WORK}/migrations.txt" | tr -d ' ')"
|
||||
echo "[gen-migrate] 迁移文件数:${COUNT}(上游应为 82)"
|
||||
|
||||
# ② 逐个打进目标库
|
||||
# ③ 逐个打进目标库
|
||||
: > "${WORK}/replay.log"
|
||||
while IFS= read -r f; do
|
||||
if ! psql -v ON_ERROR_STOP=1 -q -f "${f}" >>"${WORK}/replay.log" 2>&1; then
|
||||
echo "[gen-migrate] 迁移失败:${f}" >&2
|
||||
tail -40 "${WORK}/replay.log" >&2
|
||||
echo "[gen-migrate] 工作目录保留在:${WORK}" >&2
|
||||
exit 1
|
||||
fi
|
||||
done < "${WORK}/migrations.txt"
|
||||
|
||||
# ③ 导出 schema
|
||||
# ④ 导出 schema 到临时文件(先不碰 static/migrate.sql)
|
||||
pg_dump --schema-only --no-owner --no-privileges > "${WORK}/raw.sql"
|
||||
|
||||
# ④ 剥掉 pg_dump 前言(restrict/unrestrict/SET/set_config)
|
||||
grep -vE '^\\restrict|^\\unrestrict|^SET |^SELECT pg_catalog\.set_config' "${WORK}/raw.sql" > "${OUT}"
|
||||
# ⑤ 剥掉 pg_dump 前言(restrict/unrestrict/SET/set_config)
|
||||
grep -vE '^\\restrict|^\\unrestrict|^SET |^SELECT pg_catalog\.set_config' "${WORK}/raw.sql" > "${WORK}/migrate.sql"
|
||||
|
||||
# ⑤ 自检
|
||||
TABLES="$(grep -cE '^CREATE TABLE public\.' "${OUT}" || true)"
|
||||
echo "[gen-migrate] 已生成 ${OUT}(${TABLES} 张表,字节数 $(wc -c < "${OUT}" | tr -d ' '))"
|
||||
# ⑥ 自检:表数不对就中止,绝不覆盖已入库的 static/migrate.sql
|
||||
TABLES="$(grep -cE '^CREATE TABLE public\.' "${WORK}/migrate.sql" || true)"
|
||||
if [ "${TABLES}" != "30" ]; then
|
||||
echo "[gen-migrate] ⚠️ 表数不是 30,请复核迁移顺序" >&2
|
||||
echo "[gen-migrate] ⚠️ 表数是 ${TABLES}(应为 30),请复核迁移顺序;已中止,未改动 ${OUT}。" >&2
|
||||
echo "[gen-migrate] 候选产物保留在:${WORK}/migrate.sql" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ⑦ 校验通过后才落盘
|
||||
cp "${WORK}/migrate.sql" "${OUT}"
|
||||
echo "[gen-migrate] 已生成 ${OUT}(${TABLES} 张表,字节数 $(wc -c < "${OUT}" | tr -d ' '))"
|
||||
echo "[gen-migrate] 工作目录:${WORK}"
|
||||
@@ -22,7 +22,13 @@ moved=0
|
||||
for item in "tdc.pg" "node-identity.json"; do
|
||||
src="${DATA_DIR}/${item}"
|
||||
if [ -e "${src}" ]; then
|
||||
dest="${TRASH}/tdc-rs-${item}.${STAMP}"
|
||||
# 目标名带秒级时间戳 + PID,且不撞已有目标(撞了会让 mv 把源移进已有目录,形成一层嵌套)
|
||||
dest="${TRASH}/tdc-rs-${item}.${STAMP}.$$"
|
||||
n=1
|
||||
while [ -e "${dest}" ]; do
|
||||
dest="${TRASH}/tdc-rs-${item}.${STAMP}.$$.${n}"
|
||||
n=$((n + 1))
|
||||
done
|
||||
mv "${src}" "${dest}"
|
||||
echo "[reset-data] 已移入访达回收站:${dest}"
|
||||
moved=1
|
||||
|
||||
@@ -19,11 +19,15 @@ pub const ALIVE: &str = "(f_delete_mark IS NULL OR f_delete_mark = 0)";
|
||||
|
||||
/// 切分 SQL 脚本为可执行语句。
|
||||
///
|
||||
/// **字面量感知**:单引号串内的 `;` 不切分(`''` 是转义的单引号);
|
||||
/// `--` 行注释内的 `;` 不切分;`/* */` 块注释整体忽略。
|
||||
/// **字面量感知**:
|
||||
/// - 单引号串内的 `;` 不切分(`''` 是转义的单引号;`E'...'` 里 `\` 也是转义);
|
||||
/// - dollar-quoted 字面量(`$$ … $$` / `$tag$ … $tag$`)内的一切都是字面量,不切分;
|
||||
/// - `--` 行注释内的 `;` 不切分;`/* */` 块注释整体忽略(注释文本保留在语句里,
|
||||
/// 但纯注释块不会被当成一条语句发出去)。
|
||||
///
|
||||
/// ⚠️ 必须字面量感知:`static/migrate.sql` 里 `tdc_policy.f_priority` 的 COMMENT 文本含 `;`,
|
||||
/// 朴素 `split(';')` 会把那条语句打断,导致建表失败。
|
||||
/// ⚠️ 将来上游若带函数(pg_dump 会输出 `$$ … $$` 函数体),不认识 dollar-quoting 会把函数体切开。
|
||||
pub fn split_sql_statements(sql: &str) -> Vec<String> {
|
||||
let chars: Vec<char> = sql.chars().collect();
|
||||
let n = chars.len();
|
||||
@@ -31,11 +35,28 @@ pub fn split_sql_statements(sql: &str) -> Vec<String> {
|
||||
let mut buf = String::new();
|
||||
let mut i = 0usize;
|
||||
let mut in_string = false;
|
||||
let mut e_string = false;
|
||||
let mut in_line_comment = false;
|
||||
let mut in_block_comment = false;
|
||||
let mut dollar_tag: Option<String> = None;
|
||||
|
||||
while i < n {
|
||||
let c = chars[i];
|
||||
|
||||
// dollar-quoted 字面量内部:直到遇到同样的 `$tag$` 才出字面量态
|
||||
if let Some(tag) = dollar_tag.as_ref() {
|
||||
if c == '$' && is_dollar_close(&chars, i, tag) {
|
||||
let close = format!("${}$", tag);
|
||||
buf.push_str(&close);
|
||||
i += close.chars().count();
|
||||
dollar_tag = None;
|
||||
continue;
|
||||
}
|
||||
buf.push(c);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if in_line_comment {
|
||||
buf.push(c);
|
||||
if c == '\n' {
|
||||
@@ -57,6 +78,12 @@ pub fn split_sql_statements(sql: &str) -> Vec<String> {
|
||||
}
|
||||
if in_string {
|
||||
buf.push(c);
|
||||
// E'...' 里反斜杠是转义符(`\'` 不结束字符串)
|
||||
if e_string && c == '\\' && i + 1 < n {
|
||||
buf.push(chars[i + 1]);
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if c == '\'' {
|
||||
if i + 1 < n && chars[i + 1] == '\'' {
|
||||
buf.push('\'');
|
||||
@@ -68,7 +95,7 @@ pub fn split_sql_statements(sql: &str) -> Vec<String> {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
// 不在字符串 / 注释里
|
||||
// 不在字符串 / 注释 / dollar-quote 里
|
||||
if c == '-' && i + 1 < n && chars[i + 1] == '-' {
|
||||
in_line_comment = true;
|
||||
buf.push(c);
|
||||
@@ -81,8 +108,18 @@ pub fn split_sql_statements(sql: &str) -> Vec<String> {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if c == '$' {
|
||||
if let Some(tag) = dollar_open(&chars, i) {
|
||||
let open = format!("${}$", tag);
|
||||
buf.push_str(&open);
|
||||
i += open.chars().count();
|
||||
dollar_tag = Some(tag);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if c == '\'' {
|
||||
in_string = true;
|
||||
e_string = has_e_prefix(&buf);
|
||||
buf.push(c);
|
||||
i += 1;
|
||||
continue;
|
||||
@@ -100,22 +137,121 @@ pub fn split_sql_statements(sql: &str) -> Vec<String> {
|
||||
out
|
||||
}
|
||||
|
||||
/// 匹配 dollar-quote 起始定界符(`$$` 或 `$tag$`),返回 tag(空串表示 `$$`)。
|
||||
fn dollar_open(chars: &[char], i: usize) -> Option<String> {
|
||||
let n = chars.len();
|
||||
if chars.get(i) != Some(&'$') {
|
||||
return None;
|
||||
}
|
||||
let mut j = i + 1;
|
||||
if j < n && chars[j] == '$' {
|
||||
return Some(String::new());
|
||||
}
|
||||
// tag 形如标识符:首字符字母或下划线,其余字母/数字/下划线
|
||||
if j < n && (chars[j].is_ascii_alphabetic() || chars[j] == '_') {
|
||||
let mut tag = String::new();
|
||||
while j < n && (chars[j].is_ascii_alphanumeric() || chars[j] == '_') {
|
||||
tag.push(chars[j]);
|
||||
j += 1;
|
||||
}
|
||||
if j < n && chars[j] == '$' {
|
||||
return Some(tag);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// `i` 处是否是 dollar-quote 的结束定界符(`$tag$`)。
|
||||
fn is_dollar_close(chars: &[char], i: usize, tag: &str) -> bool {
|
||||
let n = chars.len();
|
||||
if chars.get(i) != Some(&'$') {
|
||||
return false;
|
||||
}
|
||||
let t: Vec<char> = tag.chars().collect();
|
||||
if i + 1 + t.len() >= n {
|
||||
return false;
|
||||
}
|
||||
if chars[i + 1..i + 1 + t.len()] != t[..] {
|
||||
return false;
|
||||
}
|
||||
chars[i + 1 + t.len()] == '$'
|
||||
}
|
||||
|
||||
/// 引号前是否是 `E`/`e` 前缀(决定串内 `\` 是否为转义符)。
|
||||
fn has_e_prefix(buf: &str) -> bool {
|
||||
let mut rev = buf.chars().rev();
|
||||
match rev.next() {
|
||||
Some('E') | Some('e') => {}
|
||||
_ => return false,
|
||||
}
|
||||
// 前一个字符不能是标识符字符(否则只是普通标识符以 e 结尾)
|
||||
match rev.next() {
|
||||
Some(c) => !(c.is_ascii_alphanumeric() || c == '_' || c == '$'),
|
||||
None => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// 只有「去掉注释后仍有代码」的块才算一条语句。
|
||||
fn push_statement(out: &mut Vec<String>, buf: &str) {
|
||||
let trimmed = buf.trim();
|
||||
if trimmed.is_empty() {
|
||||
return;
|
||||
}
|
||||
let has_code = trimmed.lines().any(|l| {
|
||||
let t = l.trim();
|
||||
!t.is_empty() && !t.starts_with("--")
|
||||
});
|
||||
if !has_code {
|
||||
if !has_code(trimmed) {
|
||||
return;
|
||||
}
|
||||
out.push(trimmed.to_string());
|
||||
}
|
||||
|
||||
/// 块内是否存在可执行内容(行注释 `--`、块注释 `/* */` 都不算;字符串字面量算内容)。
|
||||
fn has_code(text: &str) -> bool {
|
||||
let chars: Vec<char> = text.chars().collect();
|
||||
let n = chars.len();
|
||||
let mut i = 0usize;
|
||||
while i < n {
|
||||
let c = chars[i];
|
||||
if c == '-' && i + 1 < n && chars[i + 1] == '-' {
|
||||
while i < n && chars[i] != '\n' {
|
||||
i += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if c == '/' && i + 1 < n && chars[i + 1] == '*' {
|
||||
i += 2;
|
||||
while i + 1 < n && !(chars[i] == '*' && chars[i + 1] == '/') {
|
||||
i += 1;
|
||||
}
|
||||
i = (i + 2).min(n);
|
||||
continue;
|
||||
}
|
||||
if c == '\'' {
|
||||
// 字符串字面量整体算内容
|
||||
i += 1;
|
||||
while i < n {
|
||||
if chars[i] == '\\' && i + 1 < n {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if chars[i] == '\'' {
|
||||
if i + 1 < n && chars[i + 1] == '\'' {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
i += 1;
|
||||
break;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if !c.is_whitespace() {
|
||||
return true;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// 校验表名为合法标识符(防 SQL 注入):`[A-Za-z][A-Za-z0-9_]*`。
|
||||
pub fn valid_table(name: &str) -> Option<String> {
|
||||
let ok = !name.is_empty()
|
||||
@@ -336,4 +472,52 @@ mod tests {
|
||||
assert_eq!(json_key_to_column(&key), col);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drops_block_comment_only_chunks() {
|
||||
// 纯块注释块(连同其后的分号)不是一条语句,否则会被当成空查询发给 PG
|
||||
let sql = "/* a block comment; with semicolon */; CREATE TABLE a (id int);";
|
||||
let stmts = split_sql_statements(sql);
|
||||
assert_eq!(stmts.len(), 1);
|
||||
assert!(stmts[0].starts_with("CREATE TABLE a"));
|
||||
// 整段只有块注释 ⇒ 没有可执行语句
|
||||
assert!(split_sql_statements("/* only a comment */").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_split_inside_block_comment() {
|
||||
let sql = "CREATE TABLE a (id int /* inner; comment */); SELECT 1;";
|
||||
let stmts = split_sql_statements(sql);
|
||||
assert_eq!(stmts.len(), 2);
|
||||
assert!(stmts[0].contains("/* inner; comment */"));
|
||||
assert!(stmts[1].starts_with("SELECT 1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_split_inside_dollar_quoted_body() {
|
||||
// pg_dump 对带函数的 schema 会输出 $$ … $$ 函数体,体内分号不能切
|
||||
let sql = "CREATE FUNCTION f() RETURNS int AS $$ BEGIN RETURN 1; END; $$ LANGUAGE plpgsql; SELECT 2;";
|
||||
let stmts = split_sql_statements(sql);
|
||||
assert_eq!(stmts.len(), 2);
|
||||
assert!(stmts[0].contains("RETURN 1; END;"));
|
||||
assert!(stmts[0].starts_with("CREATE FUNCTION f()"));
|
||||
assert!(stmts[1].starts_with("SELECT 2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_split_inside_tagged_dollar_quote() {
|
||||
let sql = "CREATE FUNCTION g() RETURNS int AS $body$ BEGIN RETURN 1; END; $body$ LANGUAGE plpgsql;";
|
||||
let stmts = split_sql_statements(sql);
|
||||
assert_eq!(stmts.len(), 1);
|
||||
assert!(stmts[0].contains("$body$ BEGIN RETURN 1; END; $body$"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handles_backslash_escape_in_e_string() {
|
||||
// E'...' 里反斜杠是转义符,\' 不结束字符串
|
||||
let sql = "INSERT INTO t VALUES (E'a\\';b'); SELECT 1;";
|
||||
let stmts = split_sql_statements(sql);
|
||||
assert_eq!(stmts.len(), 2);
|
||||
assert!(stmts[0].contains("E'a\\';b'"));
|
||||
}
|
||||
}
|
||||
+1
-15
@@ -29,7 +29,7 @@ use rand::rngs::OsRng;
|
||||
use rand::RngCore;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use tdc_connector::db::{ensure_schema, query_json};
|
||||
use tdc_connector::db::{ensure_schema, query_json, valid_table};
|
||||
use tokio::sync::Mutex;
|
||||
use tower_http::cors::CorsLayer;
|
||||
use tower_http::services::ServeDir;
|
||||
@@ -117,20 +117,6 @@ fn write_identity(data_dir: &PathBuf, id: &NodeIdentity) -> std::io::Result<()>
|
||||
|
||||
// ---------------- 内嵌库:查询/执行辅助 ----------------
|
||||
|
||||
/// 校验表名为合法标识符(防止 SQL 注入);仅允许字母/数字/下划线,且以字母开头。
|
||||
fn valid_table(name: &str) -> Option<String> {
|
||||
let ok = !name.is_empty()
|
||||
&& name
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == '_')
|
||||
&& name.chars().next().map(|c| c.is_ascii_alphabetic()).unwrap_or(false);
|
||||
if ok {
|
||||
Some(name.to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn pg_err(status: StatusCode, e: PgError) -> Response {
|
||||
(
|
||||
status,
|
||||
|
||||
Reference in new issue
Block a user