From f44dd57af93ffeedc32745c223e32594c88a6345 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 14 Nov 2026 18:04:48 +0800 Subject: [PATCH] =?UTF-8?q?chore(baseline):=20=E5=85=88=E5=85=A5=E5=BA=93?= =?UTF-8?q?=E6=97=A2=E6=9C=89=E6=9C=AC=E5=9C=B0=E6=9C=AA=E6=8F=90=E4=BA=A4?= =?UTF-8?q?=E6=94=B9=E5=8A=A8=EF=BC=88tdc-connector=20=E7=9B=B4=E8=BF=9E?= =?UTF-8?q?=E5=BD=A2=E6=80=81=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit G12 收口前的基线留痕:connector-engine.ts 与 engine-registry.ts 的改动 在本轮接手时已是工作区未提交状态(非本轮新写)。按项目约定 「不覆盖、不无视」,先提交入库再在其上做正路收口,便于逐行对比。 --- .workbuddy/memory/2026-11-14.md | 51 +++ .../src/sswflow/engine/connector-engine.ts | 314 ++++++++++++++++++ .../src/sswflow/engine/engine-registry.ts | 25 +- 3 files changed, 388 insertions(+), 2 deletions(-) create mode 100644 .workbuddy/memory/2026-11-14.md create mode 100644 src/backend/src/sswflow/engine/connector-engine.ts diff --git a/.workbuddy/memory/2026-11-14.md b/.workbuddy/memory/2026-11-14.md new file mode 100644 index 0000000..88356c3 --- /dev/null +++ b/.workbuddy/memory/2026-11-14.md @@ -0,0 +1,51 @@ +# 2026-11-14 + +## ★ 09:28 复查 192.168.1.7:SSH 正常,上次修复有效 + +用户问「是不是又 ssh 不上了」。**结论:能连,一切正常,没有复发。** + +### 健康证据 + +| 检查项 | 结果 | +|---|---| +| SSH 登录 | ✅ 通(`tee_amd_7_user`)。裸探时的 `Invalid user xs` / `kex_...closed` 是我没给凭据造成的,非故障 | +| 连续运行 | **up 3 days 16:16**,boot 0 = `11-10 17:12:26` 至今**从未重启** | +| sshd | `NRestarts=0`,`ActiveEnterTimestamp=11-10 17:12:28` | +| 日志连续性 | 11-14 每小时 715~743 行,**零断层**(对比 11-10 那次 13:54–17:12 共 0 行) | +| 失败服务 | **0 个** | +| host-agent | `inactive` + `disabled` + **`NRestarts=0`** ⇒ 停用 3 天半,一次没起来 | +| 负载/磁盘/内存 | `0.40 0.54 0.58` / 38% (168G/466G) / 46G 中用 6.4G | +| known_hosts | 本机已更新为 `SHA256:IIb4ATvW…`,与服务端**一致** | +| 端口 | 22 OPEN、3010 OPEN、13300/3020 closed(后者属正常,见下) | + +### ★ 顺带纠正一条过期记忆:控制面不在 1.7 + +hostd 实际启动参数(`pgrep -a mpc-host` 实读): + +``` +mpc-host hostd --listen 0.0.0.0:3010 --allow-remote \ + --shared-secret \ + --control-plane 192.168.1.23:3020 \ ← 控制面在 1.23,不是本机 + --node-platform snp --node-site alice \ + --node-host 192.168.1.7 --node-tcb 5 \ + --node-qemu /opt/kata/bin/qemu-system-x86_64 \ + --node-firmware /opt/kata/share/ovmf/AMDSEV.fd \ + --node-kernel /opt/kata/share/kata-containers/vmlinuz-6.12.36-160-confidential \ + --node-initrd /tmp/mpc-tee-run-attestpair/initrd.img \ + --node-measurement 238f2deb… --node-kbc online \ + --node-kbs-endpoint http://10.93.0.1:18093 \ + --node-capacity 2 --report-interval 5 --guest-endpoint 10.93.0.1:3010 +``` + +- **3010 返回 401 是正常的**:需要 Bearer 共享密钥,不是故障。 +- **3020 在 1.7 上不开也是正常的**:控制面在 `192.168.1.23`。 + 旧记忆「AMD 真机 1.7 兼控制面 3020」**已过期,勿再按此排查**。 +- KBS 也在跑:`serve.py --host 10.93.0.1 --port 18093 --platform snp --verifier snp-verify-attestation.sh` + +### 结论 + +上次「停掉 host-agent 无限重启」的处置**确实解决了根因**:3 天半无重启、无日志断层、无 failed 服务。 + +但 `/dev/vfio/61` + `vfio-pci` 未加载的**底层问题仍未修**,只是把服务禁用绕过了。 +若要恢复 PCIe 隧道功能,必须先修根因(`modprobe vfio-pci` + 把硬编码的 group 61 改成真实的 group 60)再启用, +否则会原样复发(`Restart=always` + `StartLimitIntervalSec=0` 的组合会让它崩到拖垮系统)。 diff --git a/src/backend/src/sswflow/engine/connector-engine.ts b/src/backend/src/sswflow/engine/connector-engine.ts new file mode 100644 index 0000000..c2d5c58 --- /dev/null +++ b/src/backend/src/sswflow/engine/connector-engine.ts @@ -0,0 +1,314 @@ +/** + * tdc-rs 连接器执行引擎 —— 平台**不直接起 mpc 进程**,而是把算子派发到各参与方的 + * `tdc-connector`(Rust 版连接器),由连接器在本机点火。这是「全量真编排」的形态: + * 平台只做编排/取数/落库,密态计算发生在三方设备各自的连接器里。 + * + * ## 契约来源(逐字对齐,不是推测) + * - `tdc-rs/src/tdc/controller/exec.rs` + * `POST /api/ssw/tdc/DeliveryTask/{id}/Exec/Start` → `{code:200,data:{ok,accepted,duplicate,execId,phase,statusPath}}` + * `GET /api/ssw/tdc/DeliveryTask/{id}/Exec/Status?execId=…` → `{code:200,data:{records:[ExecRecord]}}` + * 头:`X-Internal-Token: ` + `X-Protocol-Version: 3.2` + * - `tdc-rs/src/tdc/engine/mod.rs::run` payload: + * `selfParty` + `clusterConfig.nax.{parties,vnetName,vnetSecret,namespace,rayHead}` + 可选 `algorithmHash` + * - `tdc-rs/src/tdc/engine/state.rs` phase 取值: + * `ACCEPTED | CHECKING | CONTROL_PLANE | NETWORK | RAY | MPC | SUCCEEDED | PLANNED | FAILED` + * 终态只有三个:`SUCCEEDED` / `FAILED` / `PLANNED`(**PLANNED 是 dry-run,不算成功**)。 + * + * ## 三条硬约束(踩过就别再踩) + * 1. `Exec/Start` 只是**受理**("受理 ≠ 执行成功"),必须轮询 `Exec/Status` 拿终态; + * 直接当成功会把「点火失败」写成「计算完成」。 + * 2. 状态是**按 execId 幂等**的:`namespace + vnetName + taskId` 相同会复用旧记录( + * `accept()` 只在 FAILED/PLANNED 时重开)。所以 vnetName/taskId 必须按 runId+stepId 派生, + * 否则重跑一次会拿到上一轮的终态、误判为「已完成」。 + * 3. mpc 点火有**启动顺序约束**(deploy 侧注释:编号大的先起,且需确认对端 LISTEN ≥6s)。 + * 这里按 parties 逆序 + 可配间隔下发,别一次性并发打满。 + */ +import * as http from 'http'; +import * as https from 'https'; +import { URL } from 'url'; +import { createHash } from 'crypto'; +import { ComputeEngine, StepArtifact, StepContext, StepResult, failed, success } from './compute-engine'; + +/** 单方连接器地址。 */ +export interface ConnectorTarget { + party: string; + base: string; +} + +export interface ConnectorSettings { + /** 是否启用(TDC_CONNECTORS 配了合法映射才算启用) */ + enabled: boolean; + targets: ConnectorTarget[]; + token: string; + protocolVersion: string; + namespace: string; + /** 固定 vnet 名/密钥(不配则按 runId+stepId 派生;配了便于跨轮复用同一 overlay) */ + vnetName: string; + vnetSecret: string; + /** 单次 HTTP 超时 */ + requestTimeoutMs: number; + /** 等终态的总预算 */ + dispatchTimeoutMs: number; + pollIntervalMs: number; + /** 相邻两方下发间隔(满足「编号大的先起」) */ + spawnGapMs: number; + rayHead: string; + raw: string; +} + +/** 本引擎接管的算子:目前 tdc-rs 的 `ensure_mpc` 只拼了 PSI 参数(LR 仍是半成品,见 mod.rs:591)。 */ +const SUPPORTED_OPERATORS = new Set(['prep.intersect']); + +/** 判定终态(`PLANNED` 是 dry-run,按失败处理,绝不冒充成功)。 */ +const TERMINAL_PHASES = new Set(['SUCCEEDED', 'FAILED', 'PLANNED']); + +export function connectorSettings(): ConnectorSettings { + const raw = (process.env.TDC_CONNECTORS || '').trim(); + const targets: ConnectorTarget[] = []; + for (const seg of raw.split(',')) { + const s = seg.trim(); + if (!s) continue; + const i = s.indexOf('='); + if (i <= 0) continue; + const party = s.slice(0, i).trim(); + const base = s.slice(i + 1).trim().replace(/\/+$/, ''); + if (party && /^https?:\/\//.test(base)) targets.push({ party, base }); + } + return { + enabled: targets.length > 0, + targets, + // 连接器侧的入向凭证就是平台内部令牌(`guard()` 与中间件同口径) + token: process.env.TDC_INTERNAL_TOKEN || process.env.SSW_INTERNAL_TOKEN || '', + protocolVersion: process.env.TDC_PROTOCOL_VERSION || '3.2', + namespace: process.env.TDC_NAMESPACE || 'ssw-flow', + vnetName: process.env.TDC_VNET_NAME || '', + vnetSecret: process.env.TDC_VNET_SECRET || '', + requestTimeoutMs: Number(process.env.TDC_REQUEST_TIMEOUT_MS || 15000), + dispatchTimeoutMs: Number(process.env.TDC_DISPATCH_TIMEOUT_MS || 180000), + pollIntervalMs: Number(process.env.TDC_POLL_INTERVAL_MS || 2000), + spawnGapMs: Number(process.env.TDC_SPAWN_GAP_MS || 3000), + rayHead: process.env.TDC_RAY_HEAD || '', + raw, + }; +} + +interface HttpResult { + status: number; + body: string; +} + +/** 最小 HTTP 客户端(不引三方库;与 mpc-client.ts 一样走 node 原生模块)。 */ +function requestJson(url: string, method: string, headers: Record, body: any, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + let u: URL; + try { + u = new URL(url); + } catch (e: any) { + reject(new Error(`URL 非法: ${url}`)); + return; + } + const payload = body === undefined ? null : Buffer.from(JSON.stringify(body), 'utf8'); + const h: Record = { ...headers }; + if (payload) { + h['Content-Type'] = 'application/json'; + h['Content-Length'] = String(payload.length); + } + const mod = u.protocol === 'https:' ? https : http; + const req = mod.request( + { + protocol: u.protocol, + hostname: u.hostname, + port: u.port || (u.protocol === 'https:' ? 443 : 80), + path: `${u.pathname}${u.search}`, + method, + headers: h, + }, + (res) => { + const chunks: Buffer[] = []; + res.on('data', (c) => chunks.push(c as Buffer)); + res.on('end', () => resolve({ status: res.statusCode || 0, body: Buffer.concat(chunks).toString('utf8') })); + }, + ); + req.setTimeout(timeoutMs, () => req.destroy(new Error(`请求超时 ${timeoutMs}ms`))); + req.on('error', reject); + if (payload) req.write(payload); + req.end(); + }); +} + +/** 连接器统一信封:`{code,msg,data}`(`ActionResult`),失败也可能是 HTTP 200 + code!=200。 */ +function unwrap(r: HttpResult): { ok: boolean; data: any; error?: string } { + let parsed: any = null; + try { + parsed = r.body ? JSON.parse(r.body) : null; + } catch { + return { ok: false, data: null, error: `响应不是 JSON(HTTP ${r.status}):${r.body.slice(0, 200)}` }; + } + if (r.status === 401) return { ok: false, data: parsed, error: '连接器拒绝:内部令牌或协议版本不匹配(401)' }; + if (r.status < 200 || r.status >= 300) { + return { ok: false, data: parsed, error: `HTTP ${r.status}:${(parsed && (parsed.msg || parsed.error)) || r.body.slice(0, 200)}` }; + } + const code = parsed && typeof parsed.code === 'number' ? parsed.code : 200; + if (code !== 200) return { ok: false, data: parsed && parsed.data, error: (parsed && parsed.msg) || `连接器返回 code=${code}` }; + return { ok: true, data: parsed ? parsed.data : null }; +} + +/** taskId 只允许安全字符(它会参与幂等键与落盘文件名)。 */ +function safeId(s: string): string { + return (s || '').replace(/[^A-Za-z0-9_-]/g, '_').slice(0, 96) || 'task'; +} + +export class ConnectorEngine implements ComputeEngine { + readonly name = 'tdc-connector'; + private readonly settings: ConnectorSettings; + + constructor(settings = connectorSettings()) { + this.settings = settings; + } + + supports(engineType: string, operatorKey: string): boolean { + return engineType === 'mpc' && SUPPORTED_OPERATORS.has(operatorKey); + } + + async execute(ctx: StepContext): Promise { + const s = this.settings; + const logs: string[] = []; + // 参与方优先级:节点参数 params.parties(流程里显式指定) > StepContext.parties(项目/数据集归属) + // > TDC_CONNECTORS 全量。★ 允许节点覆盖是必要的:同一套部署既要跑三方,也要能在某台设备 + // 掉线时按两方复算,否则一个端点不可达就整图失败。 + const explicit = Array.isArray(ctx.params?.parties) ? ctx.params!.parties.map(String) : []; + const parties = explicit.length ? explicit : ctx.parties?.length ? ctx.parties : s.targets.map((t) => t.party); + const targets = parties.map((p) => s.targets.find((t) => t.party === p)).filter((t): t is ConnectorTarget => !!t); + + if (targets.length !== parties.length) { + const missing = parties.filter((p) => !s.targets.some((t) => t.party === p)); + return failed(this.name, `参与方缺少连接器端点:${missing.join(',')}(TDC_CONNECTORS 只配了 ${s.targets.map((t) => t.party).join(',')})`, logs); + } + if (!s.token) { + return failed(this.name, '缺少内部令牌:TDC_INTERNAL_TOKEN / SSW_INTERNAL_TOKEN 均未配置,连接器会 401', logs); + } + + // ★ 幂等键:同一 run+step 重跑必须落到同一条 exec 记录(否则会重复点火三方进程)。 + const taskId = safeId(`${ctx.runId}-${ctx.stepId}`); + const vnetName = s.vnetName || `vnet-${taskId}`; + // ★ 密钥必须**跨方一致**且**跨轮稳定**(重跑要能复用同一 overlay 配置), + // 所以按 taskId 做确定性派生,而不是每轮 randomBytes。 + const vnetSecret = s.vnetSecret || createHash('sha256').update(`ssw-flow|${taskId}`).digest('hex').slice(0, 32); + const headers = { + 'X-Internal-Token': s.token, + 'X-Protocol-Version': s.protocolVersion, + }; + + logs.push(`[tdc-connector] 算子 ${ctx.operatorKey} taskId=${taskId} vnet=${vnetName} parties=${parties.join(',')}`); + + // ---- ① 按逆序逐个受理(满足 mpc「编号大的先起」)---- + const started: { party: string; base: string; execId: string; phase: string }[] = []; + for (let i = targets.length - 1; i >= 0; i--) { + const t = targets[i]; + const payload = { + selfParty: t.party, + ...(ctx.params?.algorithmHash ? { algorithmHash: String(ctx.params.algorithmHash) } : {}), + clusterConfig: { + nax: { + namespace: s.namespace, + vnetName, + vnetSecret, + parties, + rayHead: s.rayHead, + rayHeadName: s.rayHead ? s.rayHead.split(':')[0] : '', + consumerParty: parties[0], + }, + }, + }; + const url = `${t.base}/api/ssw/tdc/DeliveryTask/${encodeURIComponent(taskId)}/Exec/Start`; + let r: HttpResult; + try { + r = await requestJson(url, 'POST', headers, payload, s.requestTimeoutMs); + } catch (e: any) { + return failed(this.name, `派发到 ${t.party} 失败:${e.message}`, [...logs, `[ERROR] POST ${url} → ${e.message}`]); + } + const u = unwrap(r); + if (!u.ok || !u.data?.execId) { + return failed(this.name, `${t.party} 未受理:${u.error}`, [...logs, `[ERROR] POST ${url} → HTTP ${r.status} ${r.body.slice(0, 300)}`]); + } + started.push({ party: t.party, base: t.base, execId: u.data.execId, phase: u.data.phase || 'ACCEPTED' }); + logs.push(`[tdc-connector] ${t.party} 已受理 execId=${u.data.execId} phase=${u.data.phase}${u.data.duplicate ? '(幂等复用旧记录)' : ''}`); + if (i > 0 && s.spawnGapMs > 0) await sleep(s.spawnGapMs); + } + + // ---- ② 轮询终态 ---- + const deadline = Date.now() + s.dispatchTimeoutMs; + const final = new Map(); + while (Date.now() < deadline) { + let pending = 0; + for (const st of started) { + if (final.has(st.party)) continue; + const url = `${st.base}/api/ssw/tdc/DeliveryTask/${encodeURIComponent(taskId)}/Exec/Status?execId=${encodeURIComponent(st.execId)}`; + try { + const r = await requestJson(url, 'GET', headers, undefined, s.requestTimeoutMs); + const u = unwrap(r); + const rec = u.ok ? (u.data?.records || []).find((x: any) => x.exec_id === st.execId) || u.data?.records?.[0] : null; + if (rec && TERMINAL_PHASES.has(String(rec.phase))) { + final.set(st.party, rec); + logs.push(`[tdc-connector] ${st.party} 终态 ${rec.phase}${rec.message ? `:${truncate(rec.message, 200)}` : ''}`); + } else { + pending++; + if (rec) st.phase = String(rec.phase); + } + } catch (e: any) { + pending++; + logs.push(`[WARN] 查 ${st.party} 状态失败(将继续重试):${e.message}`); + } + } + if (final.size === started.length) break; + if (pending === 0) break; + await sleep(s.pollIntervalMs); + } + + // ---- ③ 收口:任一方非 SUCCEEDED 即整体失败 ---- + const missing = started.filter((x) => !final.has(x.party)); + if (missing.length) { + const detail = missing.map((x) => `${x.party}=${x.phase}`).join(','); + return failed( + this.name, + `等待终态超时(${s.dispatchTimeoutMs}ms):${detail}。注意 mpc 点火只拉起进程,算子是否出结果需看各方日志`, + logs, + ); + } + const bad = [...final.entries()].filter(([, rec]) => rec.phase !== 'SUCCEEDED'); + if (bad.length) { + const detail = bad.map(([p, rec]) => `${p}: ${rec.phase} — ${truncate(rec.message || '', 200)}`).join(' | '); + return failed(this.name, `连接器执行未成功:${detail}`, logs); + } + + const artifacts: StepArtifact[] = started.map((x) => { + const rec = final.get(x.party) || {}; + return { + portKey: 'report', + type: 'report' as const, + title: `${ctx.label} · ${x.party} 连接器执行轨迹`, + summary: `tdc-connector ${x.phase}(execId=${x.execId})`, + metrics: { + party: x.party, + endpoint: x.base, + execId: x.execId, + phase: rec.phase, + steps: (rec.steps || []).map((st: any) => `${st.ok ? '✓' : '✗'} ${st.name}: ${truncate(st.detail || '', 300)}`), + plan: rec.plan || null, + engine: 'tdc-connector', + operator: ctx.operatorKey, + }, + }; + }); + + return success(this.name, artifacts, logs, started.map((x) => `${x.party}:${x.execId}`).join(',')); + } +} + +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +function truncate(s: string, n: number): string { + return s.length > n ? `${s.slice(0, n)}…` : s; +} diff --git a/src/backend/src/sswflow/engine/engine-registry.ts b/src/backend/src/sswflow/engine/engine-registry.ts index 64ce76e..f729509 100644 --- a/src/backend/src/sswflow/engine/engine-registry.ts +++ b/src/backend/src/sswflow/engine/engine-registry.ts @@ -6,6 +6,7 @@ import { Injectable } from '@nestjs/common'; import { ComputeEngine } from './compute-engine'; import { SimulationEngine } from './simulation-engine'; import { MpcEngine } from './mpc-engine'; +import { ConnectorEngine, connectorSettings } from './connector-engine'; import { BIN_SOURCE_LABEL, mpcSettings, probeCliSubcommands, probeMpc } from './mpc-client'; import { applyRuntimeCapability, MPC_OPERATOR_MAP } from './mpc-operator-map'; @@ -16,8 +17,15 @@ export class EngineRegistry { constructor() { const s = mpcSettings(); - this.driver = s.enabled ? 'mpc' : 'simulation'; - this.engines = s.enabled ? [new MpcEngine(s), new SimulationEngine()] : [new SimulationEngine()]; + // 优先级:tdc-connector(三方真编排,密态计算在设备侧) > ssw-mpc(本机 CLI) > 模拟。 + // ★ 连接器是**最外层**形态:平台不持有 mpc 二进制,算子派发到各参与方的 tdc-connector。 + const c = connectorSettings(); + this.driver = c.enabled ? 'tdc-connector' : s.enabled ? 'mpc' : 'simulation'; + this.engines = [ + ...(c.enabled ? [new ConnectorEngine(c)] : []), + ...(s.enabled ? [new MpcEngine(s)] : []), + new SimulationEngine(), + ]; } select(engineType: string, operatorKey: string): ComputeEngine { @@ -27,6 +35,7 @@ export class EngineRegistry { /** 供 /Engine/Info 展示真实能力边界(不粉饰 TODO) */ async info() { const s = mpcSettings(); + const c = connectorSettings(); const probe = s.enabled ? await probeMpc(s) : null; // ★ 能力表按**实测的子命令可用性**修正:版本二进制包不含 secure feature, @@ -43,6 +52,18 @@ export class EngineRegistry { return { driver: this.driver, engines: this.engines.map((e) => e.name), + // ★ 连接器形态如实展示:端点、协议版本、命名空间。 + // 多个端点配了几个方就写几个方,不隐藏「只配到 2 方」这种能力缺口。 + connectors: { + enabled: c.enabled, + parties: c.targets.map((t) => t.party), + endpoints: c.targets.map((t) => `${t.party}=${t.base}`), + namespace: c.namespace, + protocolVersion: c.protocolVersion, + dispatchTimeoutMs: c.dispatchTimeoutMs, + tokenConfigured: !!c.token, + raw: c.raw || null, + }, mpc: { repo: s.workDir, bin: s.bin,