ci / gate (push) Waiting to run
- 项目与文档标题:nax-rs → ssw-mpc(尚数网-多方安全计算),二进制 nax → mpc - 18 个 crate 及目录 nax-* → mpc-*,Rust 引用 nax_* → mpc_* - 环境变量、C 符号、构建/部署脚本:NAX_* → MPC_*、libnax_ray.so → libmpc_ray.so - 保留协议域分隔常量与 refs/nax 参考实现路径,维持 golden 向量与历史数据兼容 - 新增 [profile.dev] incremental=false
283 lines
10 KiB
Python
Executable File
283 lines
10 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""mpc 集群编排:用 Ray 把 `mpc node` 钉到各方自己的机器上。
|
||
|
||
Ray 在这里**只做进程编排**,一行计算都不经过它:
|
||
不用 object store、不用 cloudpickle 传数据、不用 actor 做通信。
|
||
节点之间跑的是 mpc 自己的帧协议(全网状 TCP 直连)。
|
||
|
||
之所以还要 Ray,只有一条不可替代的能力:**自定义资源标签**。
|
||
|
||
ray start --resources='{"alice_node": 1}' # alice 的机器
|
||
|
||
driver 提交 actor 时声明 `resources={"alice_node": 1}`,Ray 就只会把它
|
||
调度到那台机器上。"数据不出域"这条要求因此变成调度层的硬约束,
|
||
而不是靠运维记得改配置文件。
|
||
|
||
三个执行后端:
|
||
|
||
--dry-run 只打印计划,不起进程(审阅用)
|
||
--local 本机并行起 n 个 mpc node(127.0.0.1 + 不同端口)
|
||
用来验证"计划拼出来的参数是对的",不需要 Ray 集群
|
||
(默认) 真连 Ray 集群,actor 按资源标签落机器
|
||
|
||
`--local` 存在的理由:编排逻辑(端口分配、peers 字符串、命令拼装)
|
||
是唯一可能写错的地方,而那部分和 Ray 无关。能在本机验证的先在本机验证,
|
||
别把"参数拼错"和"Ray 集群没配好"混在一起排查。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import shlex
|
||
import subprocess
|
||
import sys
|
||
|
||
DEFAULT_PORT_BASE = 9440
|
||
DEFAULT_MPC = "mpc"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 纯逻辑:计划生成
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def parse_data_map(raw: str) -> dict[str, str]:
|
||
"""解析 `alice=2,bob=3,carol=5` -> {site: "2"}。
|
||
|
||
允许值里带逗号(矩阵场景),所以按 `name=` 切分而不是按逗号切。
|
||
"""
|
||
out: dict[str, str] = {}
|
||
if not raw:
|
||
return out
|
||
for chunk in raw.split(","):
|
||
chunk = chunk.strip()
|
||
if not chunk:
|
||
continue
|
||
if "=" not in chunk:
|
||
raise SystemExit(f"--data 里 {chunk!r} 不是 site=values 形式")
|
||
name, values = chunk.split("=", 1)
|
||
name = name.strip()
|
||
if not name:
|
||
raise SystemExit(f"--data 里 {chunk!r} 的站点名为空")
|
||
if name in out:
|
||
raise SystemExit(f"--data 里站点 {name} 重复")
|
||
out[name] = values.strip()
|
||
return out
|
||
|
||
|
||
def build_plan(
|
||
sites: list[str],
|
||
hosts: dict[str, str],
|
||
ports: dict[str, int],
|
||
data: dict[str, str],
|
||
mpc_bin: str = DEFAULT_MPC,
|
||
fxp: int = 18,
|
||
seed: str | None = None,
|
||
) -> dict[str, dict]:
|
||
"""生成每个 site 的启动命令。
|
||
|
||
**字典序排序是协议要求,不是美观**:`Vault` 内部按字典序给参与方编号,
|
||
第 p 片分片投递给"排序后第 p 个"。如果这里按别的顺序分配端口和 peers,
|
||
各方对 party 编号的理解就会不一致 —— 分片投递对象整体错位,
|
||
结果静默算错(不崩、不报警,就是数不对)。
|
||
|
||
`--peers` 里给自己那条也要给:CLI 会忽略自己那一格,
|
||
但保留它让参数表完整、可读,也方便人工核对。
|
||
"""
|
||
ordered = sorted(sites)
|
||
missing = [s for s in ordered if s not in hosts]
|
||
if missing:
|
||
raise SystemExit(f"缺少站点地址: {missing}")
|
||
missing = [s for s in ordered if s not in ports]
|
||
if missing:
|
||
raise SystemExit(f"缺少站点端口: {missing}")
|
||
missing = [s for s in ordered if s not in data]
|
||
if missing:
|
||
raise SystemExit(f"--data 缺少站点: {missing}")
|
||
|
||
peers = ",".join(f"{s}={hosts[s]}:{ports[s]}" for s in ordered)
|
||
sites_arg = ",".join(ordered)
|
||
|
||
plan: dict[str, dict] = {}
|
||
for s in ordered:
|
||
cmd = [
|
||
mpc_bin, "node",
|
||
"--site", s,
|
||
"--sites", sites_arg,
|
||
"--listen", f"0.0.0.0:{ports[s]}",
|
||
"--peers", peers,
|
||
"--data", data[s],
|
||
"--fxp", str(fxp),
|
||
]
|
||
if seed:
|
||
cmd += ["--seed", seed]
|
||
plan[s] = {
|
||
"host": hosts[s],
|
||
"port": ports[s],
|
||
"party_index": ordered.index(s),
|
||
"cmd": cmd,
|
||
}
|
||
return plan
|
||
|
||
|
||
def format_plan(plan: dict[str, dict]) -> str:
|
||
lines = []
|
||
for s, item in plan.items():
|
||
lines.append(f"[{item['party_index']}] {s} @ {item['host']}:{item['port']}")
|
||
lines.append(" " + " ".join(shlex.quote(c) for c in item["cmd"]))
|
||
return "\n".join(lines)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 执行后端一:本机并行(不需要 Ray)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def run_local(plan: dict[str, dict], timeout: int) -> int:
|
||
procs = {}
|
||
for s, item in plan.items():
|
||
env = dict(os.environ)
|
||
print(f"-- {s} 启动", flush=True)
|
||
procs[s] = subprocess.Popen(
|
||
item["cmd"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=env
|
||
)
|
||
|
||
failed = 0
|
||
for s, p in procs.items():
|
||
try:
|
||
out, _ = p.communicate(timeout=timeout)
|
||
except subprocess.TimeoutExpired:
|
||
p.kill()
|
||
out, _ = p.communicate()
|
||
print(f"== {s} 超时({timeout}s)", file=sys.stderr)
|
||
failed += 1
|
||
print(f"\n===== {s} (rc={p.returncode}) =====")
|
||
print(out.rstrip())
|
||
if p.returncode != 0:
|
||
failed += 1
|
||
return 1 if failed else 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 执行后端二:Ray(钉机器)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def run_ray(plan: dict[str, dict], address: str | None, timeout: int) -> int:
|
||
try:
|
||
import ray
|
||
except ImportError:
|
||
raise SystemExit("未安装 ray。pip install 'ray[default]',或先用 --local / --dry-run")
|
||
|
||
if address:
|
||
ray.init(address=address)
|
||
else:
|
||
ray.init()
|
||
|
||
@ray.remote
|
||
class NodeSlot:
|
||
"""占位 actor:只做两件事 —— 报本机地址、起进程。
|
||
|
||
两阶段是必须的:第一轮各 actor 先报 IP,driver 才能拼出完整的
|
||
peers 表;第二轮才带着 peers 启动进程。若一轮就起进程,
|
||
先启动的方不知道自己该连谁。
|
||
"""
|
||
|
||
def __init__(self, site: str):
|
||
self.site = site
|
||
|
||
def where(self) -> str:
|
||
return ray.util.get_node_ip_address()
|
||
|
||
def run(self, cmd: list[str]) -> dict:
|
||
p = subprocess.run(cmd, capture_output=True, text=True)
|
||
return {"rc": p.returncode, "out": p.stdout, "err": p.stderr}
|
||
|
||
# 第一轮:按资源标签占位,拿到每台机器的真实 IP
|
||
slots = {}
|
||
for s in plan:
|
||
slots[s] = NodeSlot.options(resources={f"{s}_node": 1.0}).remote(s)
|
||
|
||
hosts = {s: ray.get(slots[s].where.remote()) for s in plan}
|
||
print("各站点落点:", json.dumps(hosts, ensure_ascii=False), flush=True)
|
||
|
||
# 第二轮:用真实 IP 重建计划(actor 里 listen 仍是 0.0.0.0)
|
||
ports = {s: plan[s]["port"] for s in plan}
|
||
data = {}
|
||
for s in plan:
|
||
cmd = plan[s]["cmd"]
|
||
data[s] = cmd[cmd.index("--data") + 1]
|
||
rebuilt = build_plan(
|
||
list(plan.keys()),
|
||
hosts,
|
||
ports,
|
||
data,
|
||
mpc_bin=plan[next(iter(plan))]["cmd"][0],
|
||
fxp=int(plan[next(iter(plan))]["cmd"][plan[next(iter(plan))]["cmd"].index("--fxp") + 1]),
|
||
)
|
||
|
||
print(format_plan(rebuilt), flush=True)
|
||
|
||
futures = {s: slots[s].run.remote(rebuilt[s]["cmd"]) for s in rebuilt}
|
||
failed = 0
|
||
for s, fut in futures.items():
|
||
r = ray.get(fut, timeout=timeout)
|
||
print(f"\n===== {s} (rc={r['rc']}) =====")
|
||
print(r["out"].rstrip())
|
||
if r["err"].strip():
|
||
print("[stderr]", r["err"].rstrip(), file=sys.stderr)
|
||
if r["rc"] != 0:
|
||
failed += 1
|
||
|
||
ray.shutdown()
|
||
return 1 if failed else 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser(description="mpc 集群编排(Ray 只管进程,不管数据)")
|
||
ap.add_argument("--sites", required=True, help="参与方,逗号分隔,如 alice,bob,carol")
|
||
ap.add_argument("--data", required=True, help="各方数据,如 alice=2,bob=3,carol=5")
|
||
ap.add_argument("--mpc", default=DEFAULT_MPC, help=f"mpc 二进制路径(默认 {DEFAULT_MPC})")
|
||
ap.add_argument("--port-base", type=int, default=DEFAULT_PORT_BASE, help="端口基准(按字典序递增)")
|
||
ap.add_argument("--fxp", type=int, default=18, help="定点小数位")
|
||
ap.add_argument("--seed", default=None, help="会话主种子(64 位十六进制;生产应来自 TEE)")
|
||
ap.add_argument("--address", default=None, help="Ray 集群地址,如 ray://head:10001")
|
||
ap.add_argument("--timeout", type=int, default=300, help="单方执行超时秒数")
|
||
ap.add_argument("--dry-run", action="store_true", help="只打印计划,不起进程")
|
||
ap.add_argument("--local", action="store_true", help="本机并行起进程(127.0.0.1),不需要 Ray")
|
||
args = ap.parse_args()
|
||
|
||
sites = [s.strip() for s in args.sites.split(",") if s.strip()]
|
||
if len(sites) < 2:
|
||
raise SystemExit("--sites 至少 2 方")
|
||
|
||
data = parse_data_map(args.data)
|
||
ordered = sorted(sites)
|
||
|
||
if args.local:
|
||
hosts = {s: "127.0.0.1" for s in ordered}
|
||
else:
|
||
# Ray 模式下 host 由 actor 报;这里先占位,run_ray 会重建计划
|
||
hosts = {s: "0.0.0.0" for s in ordered}
|
||
ports = {s: args.port_base + i for i, s in enumerate(ordered)}
|
||
|
||
plan = build_plan(
|
||
sites, hosts, ports, data,
|
||
mpc_bin=args.mpc, fxp=args.fxp, seed=args.seed,
|
||
)
|
||
|
||
print("== mpc 集群计划 ==")
|
||
print(format_plan(plan))
|
||
print()
|
||
|
||
if args.dry_run:
|
||
return 0
|
||
if args.local:
|
||
return run_local(plan, args.timeout)
|
||
return run_ray(plan, args.address, args.timeout)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|