Files
tdc-rs/src/tdc/engine/mod.rs
T

1073 lines
45 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! 执行引擎:把一次平台 `Exec/Start` 串成「控制面对账 → ET 组网 → Ray 计算层 → MPC 点火」。
//!
//! ## 从占位到实现(2026-11-13)
//!
//! 原 `_PLACEHOLDER` 声称「上游 Nax 执行引擎本地不可实现」。用户口径裁决后不再成立:
//! **不起 Pod**,tdc-rs 在宿主机进程内承担执行,是统一接入点。本模块替代那个占位常量。
//!
//! ## 架构口径(用户裁决 + `decision` §6 三处分歧的处理)
//!
//! | 层 | 谁负责 | 本模块的落点 |
//! |---|---|---|
//! | 跨机网络 | EasyTier | `ensure_et`:`vnetName/vnetSecret` 幂等拉起,密钥落 0600 env 文件 |
//! | 分布式计算 | Ray(**平台 nax 语义**:跨机 actor 放置与调度) | `ensure_ray`:head 起在使用方 vIP 侧 x86_64 宿主机,`rayHead` 端口按 GCS 解释;未启用时如实记 skipped,ZYNQ 侧走 `mpc cluster/agent` 等价路径 |
//! | 算子执行 | `mpc` 二进制(计算数据不出 mpc) | `ensure_mpc`:`party→site` 映射 + `--peers` 点火 |
//! | 控制面调度 | `:3020` 选址 / 起实例 / 拿端点 / 回报状态 | `control_plane_phase`:`GET /nodes` → `POST /desired` → `GET /sessions` → `GET /report` |
//!
//! 🔴 **绝不在 HTTP 请求里同步等 MPC 跑完**:真算分钟级,平台 `execStartTimeoutSeconds`
//! 有上限。`Exec/Start` 立即回 `_code=200, ok=true`(只表示**已受理**),后台任务逐步
//! 落状态;真实成败查 `Exec/Status`。「受理成功」与「执行成功」是两个不同事实,不合并汇报。
//!
//! 🔴 **fail-closed,不伪装**:缺 `TDC_EXEC_SITE_ENDPOINTS` / 缺二进制 / 缺物料 / 摘要不符
//! ⇒ 状态 `FAILED` + 具体真因,绝不回落 K8s Service DNS、绝不假装跑过。
//! `dry_run` 只落计划,状态是 `PLANNED`,**不是** `SUCCEEDED`。
pub mod config;
pub mod digest;
pub mod process;
pub mod report;
pub mod state;
use std::collections::BTreeMap;
use std::path::Path;
use std::time::Duration;
use serde_json::{json, Value};
use crate::tdc::client::mpc_control_plane::{MpcControlPlaneClient, NOOP_MESSAGE};
use crate::tdc::client::tds::TdsClient;
pub use config::ExecConfig;
pub use state::{ExecPhase, ExecRecord, ExecStore};
use digest::{base64_encode, dir_digest, short_key};
use process::{
ensure_single_instance, spawn_detached, write_secret_file, InstanceGuard, SpawnRequest,
};
use report::{auto_report, ReportInput};
/// 允许的算子白名单。**不在表内一律拒**(不回落 `psi`:静默换算子会让「审的 ≠ 跑的」)。
const MPC_OPERATORS: [&str; 3] = ["psi", "lr", "lr-vertical"];
/// 一次执行实际生效的算子参数。
///
/// 真相源优先级(高 → 低):
/// 1. payload `params.mpc` —— 平台随下发给出(扩展位;有则以其为准)
/// 2. `TDC_EXEC_MPC_PARAMS` —— 运维显式覆盖(整段 JSON)
/// 3. `<artifact_dir>/manifest.json` —— **常规来源**:manifest 与算法一起进 `dirDigest`,
/// 改算子等于改产物 ⇒ 必须重新提交、重新审核(正合「审的那版 = 跑的那版」)
/// 4. `TDC_EXEC_MPC_OPERATOR` —— 兜底(无产物场景)
#[derive(Debug, Clone, PartialEq)]
pub struct MpcParams {
pub operator: String,
pub col_blocks: Option<String>,
pub label_owner: Option<String>,
pub epochs: Option<u32>,
pub lr: Option<f64>,
/// 参数来源,落进执行状态与 plan,便于追溯「凭什么跑这个算子」
pub source: &'static str,
}
impl MpcParams {
fn parse(v: &Value, source: &'static str) -> Result<Self, String> {
let operator = v
.get("operator")
.and_then(Value::as_str)
.unwrap_or("psi")
.trim()
.to_string();
if !MPC_OPERATORS.contains(&operator.as_str()) {
return Err(format!(
"算子 {:?} 不在白名单 {:?} ⇒ 拒绝点火(不静默回落 psi;静默换算子会让「审的 ≠ 跑的」)",
operator, MPC_OPERATORS
));
}
let text = |k: &str| {
v.get(k)
.and_then(Value::as_str)
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
};
Ok(Self {
operator,
col_blocks: text("colBlocks"),
label_owner: text("labelOwner"),
epochs: v.get("epochs").and_then(Value::as_u64).map(|n| n as u32),
lr: v.get("lr").and_then(Value::as_f64),
source,
})
}
pub fn is_lr(&self) -> bool {
self.operator == "lr" || self.operator == "lr-vertical"
}
}
/// 算子命令行计划(dry-run 展示与真实点火共用同一份推导,避免两边不一致)。
#[derive(Debug, Clone)]
pub struct MpcPlan {
pub args: Vec<String>,
pub peers: String,
/// 必需物料(真实路径)
pub needed: Vec<std::path::PathBuf>,
/// 缺失的必需物料(真实路径字符串)
pub missing: Vec<String>,
pub site: String,
pub operator: String,
pub params_source: &'static str,
/// 非致命说明(如「非标签方不给 --data-y」)
pub notes: Vec<String>,
}
impl MpcPlan {
pub fn is_lr(&self) -> bool {
self.operator == "lr" || self.operator == "lr-vertical"
}
}
/// 读物料正文;缺失时给占位串(只用于 dry-run 计划展示,真跑前有 `missing` 硬门)。
fn read_or_placeholder(p: &Path) -> String {
std::fs::read_to_string(p)
.map(|s| s.trim().to_string())
.unwrap_or_else(|_| format!("<缺失:{}>", p.display()))
}
/// 执行层运行态(挂在 `TdcState` 上,后台任务持 `Arc` clone)。
pub struct ExecRuntime {
pub config: ExecConfig,
pub store: ExecStore,
pub control_plane: MpcControlPlaneClient,
/// 出向 TDS internal 通道(自报执行事实 / 脱敏日志用,与入向闸门令牌方向相反)。
pub tds: TdsClient,
}
impl ExecRuntime {
pub fn new(data_dir: &Path) -> Self {
let config = ExecConfig::from_env(data_dir);
let _ = std::fs::create_dir_all(&config.state_dir);
let store = ExecStore::new(&config.state_dir);
let control_plane = MpcControlPlaneClient::new(
config.control_plane_url.clone(),
config.control_plane_token.clone(),
);
// 与 TdcState 里那份同源配置;这里读超时放宽到 30s(日志正文可能上百 KiB)。
let tds = TdsClient::with_timeouts(
std::env::var("SSW_TDS_BASE_URL").ok(),
std::env::var("SSW_TDS_INTERNAL_TOKEN").unwrap_or_default(),
Duration::from_secs(5),
Duration::from_secs(30),
);
Self {
config,
store,
control_plane,
tds,
}
}
// ================================================================
// 自检 / 产物(入向端点直接消费)
// ================================================================
/// 能力自检:**如实列缺口**。`readyStatus=READY` 只表示「能接单」,不表示各二进制可用。
pub fn capabilities(&self) -> Value {
let c = &self.config;
let files = self.artifact_files().ok();
json!({
"engineReady": true,
"endpointReady": true,
"stateMachine": "ACCEPTED→CHECKING→CONTROL_PLANE→NETWORK→RAY→MPC",
"dryRun": c.dry_run,
"artifact": {
"dir": c.artifact_dir.display().to_string(),
"files": files.as_ref().map(|f| f.len()).unwrap_or(0),
"algorithmHash": files.as_ref().map(dir_digest),
},
"controlPlane": {
"configured": !self.control_plane.is_noop(),
"endpoint": self.control_plane.endpoint(),
"desiredConfigured": c.desired_sessions.is_some(),
},
"network": {
"easytierBin": c.easytier_bin.display().to_string(),
"easytierPresent": c.easytier_bin.exists(),
"noTun": c.et_no_tun,
"localVipConfigured": c.local_vip.is_some(),
},
"ray": {
"enabled": c.ray_enabled,
"isHead": c.ray_is_head,
"clientPort": c.ray_client_port,
"note": "未启用 ⇒ ZYNQ(armv7/无 python3) 由 mpc cluster/agent 单向可达等价接入",
},
"mpc": {
"bin": c.mpc_bin.display().to_string(),
"binPresent": c.mpc_bin.exists(),
"workdir": c.mpc_workdir.display().to_string(),
"siteEndpointsConfigured": !c.endpoint_of_site.is_empty(),
"siteCount": c.endpoint_of_site.len(),
},
})
}
/// 读算法产物目录(排序、跳过隐藏文件),返回 `文件名 → 原始字节`。
pub fn artifact_files(&self) -> Result<BTreeMap<String, Vec<u8>>, String> {
let rd = std::fs::read_dir(&self.config.artifact_dir)
.map_err(|_| format!("算法产物目录不可读: {}", self.config.artifact_dir.display()))?;
let mut files = BTreeMap::new();
for entry in rd.flatten() {
let path = entry.path();
let name = entry.file_name().to_string_lossy().to_string();
if name.starts_with('.') || !path.is_file() {
continue;
}
let bytes = std::fs::read(&path)
.map_err(|e| format!("读产物 {} 失败: {}", name, e))?;
files.insert(name, bytes);
}
if files.is_empty() {
return Err(format!(
"算法产物目录为空: {}(平台 ensureArtifact 会从使用方拉产物)",
self.config.artifact_dir.display()
));
}
Ok(files)
}
/// `Dev/Artifact` 的响应体。
///
/// 🔴 `algorithmHash` 与 `files[].contentBase64` **出自同一次读盘**:平台重算
/// `dirDigest(files)` 与基准不符就硬拒(「审的 ≠ 要跑的」)。
pub fn artifact(&self) -> Result<Value, String> {
let files = self.artifact_files()?;
let hash = dir_digest(&files);
let list: Vec<Value> = files
.iter()
.map(|(name, content)| {
json!({ "name": name, "contentBase64": base64_encode(content) })
})
.collect();
let entry = files
.keys()
.find(|n| n.ends_with(".py"))
.or_else(|| files.keys().next())
.cloned()
.unwrap_or_default();
Ok(json!({ "algorithmHash": hash, "files": list, "entry": entry }))
}
// ================================================================
// 接单
// ================================================================
/// 幂等接单:`namespace + vnetName + taskId` 相同且已有非终态记录 ⇒ 复用,不重复起实例。
///
/// 返回 `(记录, 是否重复派发)`。
pub fn accept(&self, task_id: &str, payload: &Value) -> (ExecRecord, bool) {
let nax = payload
.get("clusterConfig")
.and_then(|c| c.get("nax"))
.cloned()
.unwrap_or(Value::Null);
let namespace = text(&nax, "namespace");
let vnet_name = text(&nax, "vnetName");
let key = ExecStore::key_of(&namespace, &vnet_name, task_id);
if let Some(existing) = self.store.get(&key) {
if existing.phase != ExecPhase::Failed && existing.phase != ExecPhase::Planned {
return (existing, true);
}
}
let rec = ExecRecord::new(key, task_id, &namespace, &vnet_name);
(self.store.insert(rec), false)
}
/// 后台状态机(由 `Exec/Start` handler `tokio::spawn`)。
pub async fn run(self: std::sync::Arc<Self>, exec_id: String, payload: Value) {
let nax = payload
.get("clusterConfig")
.and_then(|c| c.get("nax"))
.cloned()
.unwrap_or(Value::Null);
let vnet_name = text(&nax, "vnetName");
let vnet_secret = text(&nax, "vnetSecret");
let ray_head = text(&nax, "rayHead");
let self_party = text(&payload, "selfParty");
let parties: Vec<String> = nax
.get("parties")
.and_then(Value::as_array)
.map(|a| {
a.iter()
.filter_map(|v| v.as_str().map(str::to_string))
.collect()
})
.unwrap_or_default();
// ---- ① 校验产物摘要 + 组装(脱敏)执行计划 ----
let check_note = match self.check_artifact(&payload) {
Ok(note) => note,
Err(reason) => {
self.fail(&exec_id, ExecPhase::Checking, "校验 algorithmHash", reason);
return;
}
};
self.ok_step(
&exec_id,
ExecPhase::Checking,
"校验 algorithmHash",
check_note,
);
let plan = self.build_plan(&payload, &nax, &parties, &self_party, &ray_head);
self.store.mutate(&exec_id, |r| r.plan = Some(plan.clone()));
// ---- ② 控制面:选址 / 起实例 / 拿端点 / 回报状态 ----
match self.control_plane_phase().await {
Ok(detail) => self.ok_step(&exec_id, ExecPhase::ControlPlane, "控制面对账", detail),
Err(reason) => {
self.fail(&exec_id, ExecPhase::ControlPlane, "控制面对账", reason);
return;
}
}
// ---- ③ EasyTier 组网 ----
let et_tag = format!(
"et-{}",
short_key(&format!("{}|{}", vnet_name, payload.to_string()))
);
match self.ensure_et(&et_tag, &vnet_name, &vnet_secret) {
Ok(detail) => self.ok_step(&exec_id, ExecPhase::Network, "EasyTier 组网", detail),
Err(reason) => {
self.fail(&exec_id, ExecPhase::Network, "EasyTier 组网", reason);
return;
}
}
// ---- ④ Ray 分布式计算层 ----
let self_site = self
.config
.site_of_party
.get(&self_party)
.cloned()
.unwrap_or_else(|| self_party.clone());
match self.ensure_ray(&et_tag, &ray_head, &self_site) {
Ok(detail) => self.ok_step(&exec_id, ExecPhase::Ray, "Ray 计算层", detail),
Err(reason) => {
self.fail(&exec_id, ExecPhase::Ray, "Ray 计算层", reason);
return;
}
}
// ---- ⑤ mpc 算子点火 ----
// 🔴 用 exec_id 作 mpc 日志 tag ⇒ 每次执行独立文件,不会把上一轮日志 append 进
// 本轮摘要(resultHash 必须与设备侧回收到的整文件逐字对应)。
match self.ensure_mpc(&exec_id, &self_site, &parties, &payload) {
Ok(detail) => self.ok_step(&exec_id, ExecPhase::Mpc, "MPC 点火", detail),
Err(reason) => {
self.fail(&exec_id, ExecPhase::Mpc, "MPC 点火", reason);
return;
}
}
// ---- ⑥ 收口 ----
if self.config.dry_run {
self.ok_step(
&exec_id,
ExecPhase::Planned,
"收口",
"dry-run:执行计划已落盘,未拉起任何进程 / 未 POST /desired ⇒ 不算执行成功",
);
} else {
self.ok_step(
&exec_id,
ExecPhase::Succeeded,
"收口",
"各阶段已受理并拉起;MPC 结果需查算子日志与载荷通路证据",
);
// ---- ⑦ 连接器自报(G15 GAP② + G6)----
// 等本端 mpc 子进程真正退出、日志落定,再由连接器自己回传执行事实 +
// 脱敏日志正文。不再是验收脚本代报。结果写进 ExecRecord.report,
// `Exec/Status` 可查 ⇒ 验收能断言「是设备自报的」。
let report = self.auto_report_phase(&exec_id, &payload, &self_site).await;
self.store.mutate(&exec_id, |r| r.report = Some(report));
}
}
/// 组装自报输入并调用 [`report::auto_report`](失败也只落状态,不影响已 SUCCEEDED 的执行)。
async fn auto_report_phase(
&self,
exec_id: &str,
payload: &Value,
self_site: &str,
) -> report::ExecReport {
let local_task_id = self
.store
.get(exec_id)
.map(|r| r.task_id)
.unwrap_or_default();
// 🔴 平台 Exec/Start 载荷不带 connectorId(只在平台回执里)⇒ 本端 id 从
// partyMap[selfParty] 解析;解析不到 ⇒ 空串 ⇒ auto_report fail-closed 拒报。
let connector_id =
report::self_connector_id(payload, self_site, &self.config.site_of_party);
let version_no = payload
.get("versionNo")
.and_then(Value::as_i64)
.unwrap_or(1);
// `podNode` 取本端 site 端点的 host,与设备侧日志回收脚本的机器口径一致。
let pod_node = self
.config
.endpoint_of_site
.get(self_site)
.and_then(|ep| ep.rsplit_once(':').map(|(h, _)| h.to_string()))
.unwrap_or_else(|| self_site.to_string());
let tag = format!("mpc-{}", exec_id);
let input = ReportInput {
exec_id: exec_id.to_string(),
local_task_id,
connector_id,
version_no,
pod_node,
mpc_log: self.config.state_dir.join(format!("{}.log", tag)),
mpc_pid: self.config.state_dir.join(format!("{}.pid", tag)),
};
auto_report(&self.config.report, &self.tds, &input).await
}
// ================================================================
// 各阶段实现
// ================================================================
/// 产物摘要校验:**只有本地真有产物才校验**(非使用方可能没有产物,不硬拒)。
fn check_artifact(&self, payload: &Value) -> Result<String, String> {
let expected = payload
.get("algorithmHash")
.and_then(Value::as_str)
.unwrap_or("");
if expected.is_empty() {
return Ok("payload 未带 algorithmHash ⇒ 跳过摘要校验".to_string());
}
let files = match self.artifact_files() {
Ok(f) => f,
Err(_) => {
return Ok("本地无算法产物(本方可能不是使用方)⇒ 跳过摘要校验".to_string())
}
};
let actual = dir_digest(&files);
if actual != expected {
return Err(format!(
"algorithmHash 不符:平台={} 本地={}(审的 ≠ 要跑的,拒绝执行)",
expected, actual
));
}
Ok(format!("algorithmHash 一致({}…)", &actual[..12]))
}
/// 组装**脱敏**执行计划:命令、参数来源、能力边界一目了然,不含任何凭据。
fn build_plan(
&self,
payload: &Value,
nax: &Value,
parties: &[String],
self_party: &str,
ray_head: &str,
) -> Value {
let c = &self.config;
let vnet_name = text(nax, "vnetName");
let vnet_secret = text(nax, "vnetSecret");
let vip = c.local_vip.clone().unwrap_or_default();
let et_args = self.et_args(&vnet_name, &vnet_secret, &vip);
let (ray_args, ray_note) = self.ray_args(ray_head, self_party);
// 算子段:解析失败也落进计划(不在这里失败 —— 阶段语义由 ensure_mpc 如实给出)
let (mpc_args, peers, mpc_meta) = match self.effective_mpc_params(payload) {
Ok(p) => {
let mp = self.mpc_plan(&p, parties, self_party);
(
mp.args.clone(),
mp.peers.clone(),
json!({
"operator": mp.operator,
"paramSource": mp.params_source,
"isLr": mp.is_lr(),
"missingMaterials": mp.missing,
"notes": mp.notes,
"colBlocks": p.col_blocks,
"labelOwner": p.label_owner,
"epochs": p.epochs,
"lr": p.lr,
}),
)
}
Err(e) => (
Vec::new(),
String::new(),
json!({ "error": e, "note": "算子参数不可用 ⇒ ensure_mpc 会 fail-closed,不回落 psi" }),
),
};
json!({
"contract": {
"namespace": text(nax, "namespace"),
"vnetName": vnet_name,
"vnetSecretConfigured": !vnet_secret.is_empty(),
"rayHead": ray_head,
"rayHeadName": text(nax, "rayHeadName"),
"parties": parties,
"selfParty": self_party,
"consumerParty": text(nax, "consumerParty"),
// 🔴 死地址只作为「为什么不用它」的证据,绝不作为执行目标
"k8sServiceDnsUnusable": true,
"clusterConfigParties": payload.get("clusterConfig").and_then(|c| c.get("parties")).cloned().unwrap_or(Value::Null),
},
"controlPlane": {
"endpoint": self.control_plane.endpoint(),
"calls": ["GET /nodes", "POST /desired", "GET /sessions", "GET /report"],
"desiredConfigured": c.desired_sessions.is_some(),
"boundary": "/desired 只起 TEE guest + 公布端点,不下发算子",
},
"easyTier": {
"bin": c.easytier_bin.display().to_string(),
"args": process::redact_args(&et_args),
"secretInArgvKnownLimit": true,
"noTun": c.et_no_tun,
},
"ray": {
"enabled": c.ray_enabled,
"bin": c.ray_bin.display().to_string(),
"args": process::redact_args(&ray_args),
"note": ray_note,
"portMapping": "rayHead 端口按 GCS 解释;Ray Client Server 端口由 TDC_EXEC_RAY_CLIENT_PORT 单配,两者需同时可达",
},
"mpc": {
"bin": c.mpc_bin.display().to_string(),
"operator": c.mpc_operator,
"operatorEffective": mpc_meta,
"peers": peers,
"args": process::redact_args(&mpc_args),
"orderConstraint": "编号大的先起(carol → alice),且需先确认对端 LISTEN ≥6s",
},
"dryRun": c.dry_run,
})
}
async fn control_plane_phase(&self) -> Result<String, String> {
if self.control_plane.is_noop() {
return Ok(format!(
"{} ⇒ 跳过控制面对账(如实标注,未伪装调用);算子点火仍由执行层承担",
NOOP_MESSAGE
));
}
let nodes = self.control_plane.nodes().await?;
let desired = match &self.config.desired_sessions {
None => "未配置期望态 TDC_EXEC_DESIRED_SESSIONS ⇒ 未起 TEE 实例".to_string(),
Some(raw) if self.config.dry_run => {
format!("dry-run ⇒ 未 POST /desired(已备 {} 字节期望态)", raw.len())
}
Some(raw) => {
let sessions: Value = serde_json::from_str(raw)
.map_err(|e| format!("期望态不是合法 JSON 数组: {}", e))?;
self.control_plane.desired(&sessions).await?;
"已 POST /desired(选址 → create/attest)".to_string()
}
};
let sessions = self.control_plane.sessions().await?;
let report = self.control_plane.report().await?;
Ok(format!(
"GET /nodes ✓({} 字节)、{}、GET /sessions ✓({} 字节)、GET /report ✓({} 字节);端点已回收",
serde_json::to_string(&nodes).map(|s| s.len()).unwrap_or(0),
desired,
serde_json::to_string(&sessions).map(|s| s.len()).unwrap_or(0),
serde_json::to_string(&report).map(|s| s.len()).unwrap_or(0),
))
}
fn et_args(&self, vnet_name: &str, vnet_secret: &str, vip: &str) -> Vec<String> {
let c = &self.config;
let mut a = vec![
"--network-name".into(),
vnet_name.into(),
"--network-secret".into(),
vnet_secret.into(),
];
if !vip.is_empty() {
a.push("--ipv4".into());
a.push(vip.into());
}
a.extend([
"--hostname".into(),
"tdc-rs".into(),
"--instance-name".into(),
format!("ssw-{}", vip),
"--machine-id".into(),
"ssw-tdc-rs".into(),
"--listeners".into(),
c.et_listeners.clone(),
"--rpc-portal".into(),
c.et_rpc_portal.clone(),
]);
if c.et_no_tun {
a.push("--no-tun".into());
}
if let Some(p) = &c.et_proxy_networks {
a.push("--proxy-networks".into());
a.push(p.clone());
}
if let Some(s) = &c.et_socks5 {
a.push("--socks5".into());
a.push(s.clone());
}
if let Some(r) = &c.et_relay {
a.push("--peers".into());
a.push(r.clone());
}
a
}
fn ensure_et(&self, tag: &str, vnet_name: &str, vnet_secret: &str) -> Result<String, String> {
let c = &self.config;
if vnet_name.is_empty() || vnet_secret.is_empty() {
return Err("下发载荷缺 clusterConfig.nax.vnetName / vnetSecret ⇒ 拒绝建网".to_string());
}
let vip = c
.local_vip
.clone()
.ok_or_else(|| "未配置 TDC_EXEC_LOCAL_VIP ⇒ 不猜 vIP,拒绝建网".to_string())?;
let args = self.et_args(vnet_name, vnet_secret, &vip);
let preview = SpawnRequest::new(&c.easytier_bin, args.clone(), &c.state_dir, tag).preview();
if c.dry_run {
return Ok(format!("dry-run ⇒ 未拉起 ET;计划命令:{}", preview));
}
if !c.easytier_bin.exists() {
return Err(format!("EasyTier 二进制不存在: {}", c.easytier_bin.display()));
}
// 指纹含 secret 的哈希(不落明文)⇒ 密钥变了不会再复用旧实例。
let fingerprint = short_key(&format!("{}|{}|{}", vnet_name, vnet_secret, vip));
let guard = ensure_single_instance(&c.state_dir, tag, &fingerprint)?;
if let InstanceGuard::Reused(pid) = guard {
return Ok(format!("复用已有 EasyTier 实例(pid={}),未重复拉起", pid));
}
// 凭证落 0600 env 文件(不进状态 JSON / 不进日志);argv 展开是 easytier CLI 的已知限制。
let env_path = c.state_dir.join(format!("{}.env", tag));
write_secret_file(
&env_path,
&format!("EASYTIER_NETWORK_SECRET={}\n", vnet_secret),
)?;
let req = SpawnRequest::new(&c.easytier_bin, args, &c.state_dir, tag);
let pid = spawn_detached(&req)?;
guard.commit(&fingerprint);
Ok(format!(
"已拉起 EasyTier(pid={},vIP={},noTun={});凭证在 0600 env 文件",
pid, vip, c.et_no_tun
))
}
fn ray_args(&self, ray_head: &str, self_party: &str) -> (Vec<String>, String) {
let c = &self.config;
let (host, gcs_port) = ray_head
.split_once(':')
.map(|(h, p)| (h.to_string(), p.to_string()))
.unwrap_or_else(|| (ray_head.to_string(), "6379".to_string()));
let resources = c
.ray_resources
.clone()
.unwrap_or_else(|| format!("{{\"{}_node\":1}}", self_party));
if c.ray_is_head {
(
vec![
"start".into(),
"--head".into(),
format!("--port={}", gcs_port),
format!("--ray-client-server-port={}", c.ray_client_port),
format!("--node-ip-address={}", host),
"--dashboard-host=0.0.0.0".into(),
// 🔴 不经 shell ⇒ 参数里不能带引号,否则引号会原样传给 ray
format!("--resources={}", resources),
],
format!(
"head 绑 {}(GCS 端口取 rayHead 的 {}),Client 端口 {} 需同时可达",
host, gcs_port, c.ray_client_port
),
)
} else {
(
vec![
"start".into(),
format!("--address={}:{}", host, gcs_port),
format!("--resources={}", resources),
],
format!("worker 加入 {}:{}", host, gcs_port),
)
}
}
fn ensure_ray(&self, tag: &str, ray_head: &str, self_site: &str) -> Result<String, String> {
let c = &self.config;
if !c.ray_enabled {
return Ok(format!(
"未启用 Ray(site={});按裁决由 mpc cluster/agent 单向可达形态等价接入,不硬移植 Ray 到 armv7",
self_site
));
}
if ray_head.is_empty() {
return Err("启用了 Ray 但下发载荷缺 clusterConfig.nax.rayHead ⇒ 无法确定 head".to_string());
}
let (args, note) = self.ray_args(ray_head, self_site);
let ray_tag = format!("ray-{}", tag.trim_start_matches("et-"));
let preview = format!("{} {}", c.ray_bin.display(), args.join(" "));
if c.dry_run {
return Ok(format!("dry-run ⇒ 未拉起 Ray;{};计划命令:{}", note, preview));
}
// `ray start` 会自行 daemonize ⇒ 本层 pid 只记录启动器,重复派发由 ray 自身的
// "already running" 检测兜底;这里仍写 pid/fp 文件留痕。
let fingerprint = short_key(&format!("ray|{}|{}", ray_head, self_site));
let guard = ensure_single_instance(&c.state_dir, &ray_tag, &fingerprint)?;
if let InstanceGuard::Reused(pid) = guard {
return Ok(format!("复用 Ray 启动记录(pid={})", pid));
}
let req = SpawnRequest::new(&c.ray_bin, args, &c.state_dir, &ray_tag);
let pid = spawn_detached(&req)?;
guard.commit(&fingerprint);
Ok(format!("已执行 ray start(启动器 pid={});{}", pid, note))
}
/// 一次执行**实际生效**的算子参数(解析优先级见 [`MpcParams`])。
fn effective_mpc_params(&self, payload: &Value) -> Result<MpcParams, String> {
let c = &self.config;
// ① 平台随下发给出的算子段(未来扩展位;有就以它为准)
if let Some(v) = payload
.get("params")
.and_then(|p| p.get("mpc"))
.filter(|v| v.is_object())
{
return MpcParams::parse(v, "payload:params.mpc");
}
// ② 运维显式覆盖(整段 JSON)
if let Some(raw) = c.mpc_params.as_deref() {
let v: Value = serde_json::from_str(raw)
.map_err(|e| format!("TDC_EXEC_MPC_PARAMS 不是合法 JSON: {}", e))?;
if !v.is_object() {
return Err("TDC_EXEC_MPC_PARAMS 不是 JSON 对象 ⇒ 拒绝(不静默回落默认算子)".into());
}
return MpcParams::parse(&v, "env:TDC_EXEC_MPC_PARAMS");
}
// ③ 算法产物自描述(**常规来源**:manifest 进 dirDigest ⇒ 改算子必须重新审核)
let manifest = c.artifact_dir.join("manifest.json");
if manifest.is_file() {
let text = std::fs::read_to_string(&manifest)
.map_err(|e| format!("读产物 manifest 失败 {}: {}", manifest.display(), e))?;
let v: Value = serde_json::from_str(&text)
.map_err(|e| format!("产物 manifest 不是合法 JSON: {}", e))?;
return MpcParams::parse(&v, "artifact:manifest.json");
}
// ④ 兜底:部署侧配置的算子名
MpcParams::parse(&json!({ "operator": c.mpc_operator }), "env:TDC_EXEC_MPC_OPERATOR")
}
/// 组装算子命令(**PSI 与纵向 LR 走同一条判据**)。
///
/// 物料缺失**不在这里报错**(dry-run 也要能展示计划);缺失项由 [`MpcPlan::missing`]
/// 列出,`ensure_mpc` 据此 fail-closed。
fn mpc_plan(&self, params: &MpcParams, parties: &[String], self_party: &str) -> MpcPlan {
let c = &self.config;
let sites: Vec<String> = parties
.iter()
.filter_map(|p| c.site_of_party.get(p).cloned())
.collect();
let peers = sites
.iter()
.filter_map(|s| c.endpoint_of_site.get(s).map(|e| format!("{}={}", s, e)))
.collect::<Vec<_>>()
.join(",");
let site = c
.site_of_party
.get(self_party)
.cloned()
.unwrap_or_else(|| self_party.to_string());
let wd = &c.mpc_workdir;
let mut args = vec![
params.operator.clone(),
"--site".to_string(),
site.clone(),
"--sites".to_string(),
sites.join(","),
"--listen".to_string(),
format!("0.0.0.0:{}", c.mpc_port),
"--peers".to_string(),
peers.clone(),
"--report".to_string(),
"json".to_string(),
];
let mut needed = Vec::new();
let mut extra_notes = Vec::new();
if params.is_lr() {
// 纵向 LR:列块 + 标签方 + 本端特征/键 +(仅标签方)标签
// 🔴 `--seed` 必须三方同一份(`sess-master.txt`):它定 PRG / 初始化,
// 各端自取随机种子会让三方跑出不同模型,「结果一致」就成了假象。
// 与 build/verify-real-machine-3party.sh 的已验证口径保持一字不差。
let master_p = wd.join("sess-master.txt");
needed.push(master_p.clone());
args.push("--seed".into());
args.push(read_or_placeholder(&master_p));
match params.col_blocks.as_deref() {
Some(v) => {
args.push("--col-blocks".into());
args.push(v.to_string());
}
None => extra_notes.push("缺 colBlocks(LR 必需:site:offset:len,须铺满 0..d_total)".into()),
}
match params.label_owner.as_deref() {
Some(v) => {
args.push("--label-owner".into());
args.push(v.to_string());
}
None => extra_notes.push("缺 labelOwner(LR 必需:恰好一方给 --data-y)".into()),
}
let ids_p = wd.join(format!("{}.ids", site));
let x_p = wd.join(format!("{}.x", site));
needed.push(ids_p.clone());
needed.push(x_p.clone());
args.push("--data-ids".into());
args.push(read_or_placeholder(&ids_p));
args.push("--data-x".into());
args.push(read_or_placeholder(&x_p));
if params.label_owner.as_deref() == Some(site.as_str()) {
let y_p = wd.join(format!("{}.y", site));
needed.push(y_p.clone());
args.push("--data-y".into());
args.push(read_or_placeholder(&y_p));
} else {
extra_notes.push(format!("非标签方(label-owner={})⇒ 不给 --data-y",
params.label_owner.as_deref().unwrap_or("?")));
}
if let Some(e) = params.epochs {
args.push("--epochs".into());
args.push(e.to_string());
}
if let Some(l) = params.lr {
args.push("--lr".into());
args.push(l.to_string());
}
} else {
// PSI:会话 seed + 本端 ID 清单(`--ids`);物料三方必须同一份 seed
let ids_p = wd.join(format!("{}.ids", site));
let master_p = wd.join("sess-master.txt");
needed.push(ids_p.clone());
needed.push(master_p.clone());
args.push("--ids".into());
args.push(read_or_placeholder(&ids_p));
args.push("--seed".into());
args.push(read_or_placeholder(&master_p));
}
let missing: Vec<String> = needed
.iter()
.filter(|p| !p.exists())
.map(|p| p.display().to_string())
.collect();
MpcPlan {
args: args.clone(),
peers,
needed,
missing,
site,
operator: params.operator.clone(),
params_source: params.source,
notes: extra_notes,
}
}
fn ensure_mpc(&self, exec_id: &str, self_site: &str, parties: &[String], payload: &Value) -> Result<String, String> {
let c = &self.config;
if c.endpoint_of_site.is_empty() {
return Err(
"未配置 TDC_EXEC_SITE_ENDPOINTS ⇒ 不回落 K8s Service DNS 死地址,拒绝点火".to_string(),
);
}
for p in parties {
match c.site_of_party.get(p) {
None => return Err(format!("party {} 无 TDC_EXEC_PARTY_SITE_MAP 映射", p)),
Some(s) if !c.endpoint_of_site.contains_key(s) => {
return Err(format!("site {} 无端点配置", s))
}
_ => {}
}
}
// 🔴 先解析算子:未知算子/参数不全 ⇒ 这里就拒,**绝不**回落到默认 psi 点火
let params = self.effective_mpc_params(payload)?;
let plan = self.mpc_plan(&params, parties, &self_party_of(c, self_site));
if c.dry_run {
return Ok(format!(
"dry-run ⇒ 未点火;算子={}(来源 {});peers={};物料目录 {}({} 项就绪,缺 {:?})",
plan.operator,
plan.params_source,
plan.peers,
c.mpc_workdir.display(),
plan.needed.len() - plan.missing.len(),
plan.missing
));
}
if !c.mpc_bin.exists() {
return Err(format!("mpc 二进制不存在: {}", c.mpc_bin.display()));
}
if !plan.missing.is_empty() {
return Err(format!(
"MPC 物料缺失(算子={}):{}",
plan.operator,
plan.missing.join(", ")
));
}
if !plan.notes.is_empty() {
// 参数不全属于**配置错误**,不降级执行
let fatal: Vec<&String> = plan.notes.iter().filter(|n| n.starts_with("缺 ")).collect();
if !fatal.is_empty() {
let msg: Vec<String> = fatal.iter().map(|s| (*s).clone()).collect();
return Err(format!("算子参数不全(算子={}):{}", plan.operator, msg.join(";")));
}
}
// `mpc lr-vertical` 的密态通道端口 +1,PSK 由会话密钥统一供给
let psk = c.mpc_psk.clone().unwrap_or_else(|| {
std::fs::read_to_string(c.mpc_workdir.join("sess-auth.txt"))
.map(|s| s.trim().to_string())
.unwrap_or_default()
});
// 每次执行独立日志文件(exec_id 是 16 位 hex 短哈希,路径安全)。
let mpc_tag = format!("mpc-{}", exec_id);
let req = SpawnRequest::new(&c.mpc_bin, plan.args.clone(), &c.state_dir, &mpc_tag)
.with_env("MPC_PSK", &psk);
let pid = spawn_detached(&req)?;
let mut extra = String::new();
if plan.is_lr() {
if let Some(cb) = params.col_blocks.as_deref() {
extra.push_str(&format!(",colBlocks={}", cb));
}
if let Some(lo) = params.label_owner.as_deref() {
extra.push_str(&format!(",labelOwner={}", lo));
}
}
Ok(format!(
"已点火 mpc {}(pid={},site={},peers={}{},参数来源 {});跨机启动顺序与 LISTEN 就绪由部署侧保证",
plan.operator, pid, plan.site, plan.peers, extra, plan.params_source
))
}
// ---- 状态写入小工具 ----
fn ok_step(&self, id: &str, phase: ExecPhase, name: &str, detail: impl Into<String>) {
self.store
.mutate(id, |r| r.step(phase, name, true, detail.into()));
}
fn fail(&self, id: &str, phase: ExecPhase, name: &str, reason: impl Into<String>) {
self.store
.mutate(id, |r| r.step(phase, name, false, reason.into()));
}
}
/// `ensure_mpc` 用 site 反查 selfParty(映射双向唯一,缺省回落 site 本身)。
fn self_party_of(c: &ExecConfig, site: &str) -> String {
c.site_of_party
.iter()
.find(|(_, s)| s.as_str() == site)
.map(|(p, _)| p.clone())
.unwrap_or_else(|| site.to_string())
}
fn text(v: &Value, key: &str) -> String {
v.get(key)
.and_then(Value::as_str)
.unwrap_or("")
.to_string()
}
#[cfg(test)]
mod tests {
use super::*;
fn cfg(overrides: &[(&str, &str)]) -> ExecConfig {
let env: BTreeMap<String, String> = overrides
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect();
ExecConfig::from_raw(&env, Path::new("/tmp/tdc-exec-test"))
}
fn runtime(c: ExecConfig, tag: &str) -> ExecRuntime {
let dir = std::env::temp_dir().join(format!("tdc-exec-{}-{}", tag, std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
ExecRuntime {
config: c,
store: ExecStore::new(&dir),
control_plane: MpcControlPlaneClient::new(None, String::new()),
tds: TdsClient::new(None, String::new()),
}
}
#[test]
fn plan_never_contains_the_vnet_secret() {
let r = runtime(cfg(&[(config::ENV_LOCAL_VIP, "10.12.0.2")]), "plan");
let nax = json!({
"vnetName": "ssw-SPACE", "vnetSecret": "ssw-TOP-SECRET-123",
"namespace": "nax-1-v1", "rayHead": "10.12.0.2:30417",
"rayHeadName": "nax-head-1-v1", "parties": ["p1","p2"],
"consumerParty": "p1",
});
let payload = json!({"clusterConfig": {"nax": nax.clone(), "parties": {}}, "selfParty": "p1"});
let plan = r.build_plan(
&payload,
&nax,
&["p1".into(), "p2".into()],
"p1",
"10.12.0.2:30417",
);
let s = serde_json::to_string(&plan).unwrap();
assert!(!s.contains("ssw-TOP-SECRET-123"), "计划泄漏了 vnetSecret");
assert_eq!(plan["contract"]["vnetSecretConfigured"], true);
assert_eq!(plan["contract"]["k8sServiceDnsUnusable"], true);
}
#[test]
fn mpc_peers_come_from_explicit_config_not_k8s_dns() {
let r = runtime(
cfg(&[(
config::ENV_SITE_ENDPOINTS,
"alice=10.12.0.1:9449,carol=10.12.0.2:9449",
)]),
"peers",
);
// 🔴 原测试调的 `mpc_args` 早已被 `mpc_plan` 取代(测试编译不过)⇒ 改用真实计划推导
let params = MpcParams::parse(&json!({ "operator": "psi" }), "test").unwrap();
let plan = r.mpc_plan(&params, &["p1".into(), "p2".into()], "p1");
assert_eq!(plan.peers, "alice=10.12.0.1:9449,carol=10.12.0.2:9449");
}
#[test]
fn ray_head_port_is_interpreted_as_gcs() {
let r = runtime(
cfg(&[(config::ENV_RAY_ENABLED, "1"), (config::ENV_RAY_IS_HEAD, "1")]),
"ray",
);
let (args, note) = r.ray_args("10.12.0.2:30417", "p1");
assert!(args.iter().any(|a| a == "--port=30417"));
assert!(args.iter().any(|a| a == "--ray-client-server-port=10001"));
assert!(note.contains("30417"));
}
#[test]
fn accept_is_idempotent_per_namespace_vnet_task() {
let r = runtime(cfg(&[]), "accept");
let payload = json!({"clusterConfig":{"nax":{"namespace":"ns","vnetName":"vn"}}});
let (a, dup1) = r.accept("t1", &payload);
let (b, dup2) = r.accept("t1", &payload);
assert!(!dup1);
assert!(dup2);
assert_eq!(a.exec_id, b.exec_id);
}
#[test]
fn dry_run_state_is_planned_not_succeeded() {
// 状态机收口阶段的语义:Planned 与 Succeeded 是两个不同枚举值。
assert_ne!(ExecPhase::Planned, ExecPhase::Succeeded);
assert_eq!(ExecPhase::Planned.as_str(), "PLANNED");
}
}