Files
ssw-mpc/deploy/ray/mpc_cluster.py
T
admin 54781b3f83
ci / gate (push) Waiting to run
refactor: 项目改名 ssw-mpc,crate 统一到 mpc-* 前缀
- 项目与文档标题:nax-rs → ssw-mpc(尚数网-多方安全计算),二进制 nax → mpc
- 18 个 crate 及目录 nax-* → mpc-*,Rust 引用 nax_* → mpc_*
- 环境变量、C 符号、构建/部署脚本:NAX_* → MPC_*、libnax_ray.so → libmpc_ray.so
- 保留协议域分隔常量与 refs/nax 参考实现路径,维持 golden 向量与历史数据兼容
- 新增 [profile.dev] incremental=false
2026-11-05 15:06:12 +08:00

283 lines
10 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""mpc 集群编排:用 Ray 把 `mpc node` 钉到各方自己的机器上。
Ray 在这里**只做进程编排**,一行计算都不经过它:
不用 object store、不用 cloudpickle 传数据、不用 actor 做通信。
节点之间跑的是 mpc 自己的帧协议(全网状 TCP 直连)。
之所以还要 Ray,只有一条不可替代的能力:**自定义资源标签**。
ray start --resources='{"alice_node": 1}' # alice 的机器
driver 提交 actor 时声明 `resources={"alice_node": 1}`,Ray 就只会把它
调度到那台机器上。"数据不出域"这条要求因此变成调度层的硬约束,
而不是靠运维记得改配置文件。
三个执行后端:
--dry-run 只打印计划,不起进程(审阅用)
--local 本机并行起 n 个 mpc node(127.0.0.1 + 不同端口)
用来验证"计划拼出来的参数是对的",不需要 Ray 集群
(默认) 真连 Ray 集群,actor 按资源标签落机器
`--local` 存在的理由:编排逻辑(端口分配、peers 字符串、命令拼装)
是唯一可能写错的地方,而那部分和 Ray 无关。能在本机验证的先在本机验证,
别把"参数拼错"和"Ray 集群没配好"混在一起排查。
"""
from __future__ import annotations
import argparse
import json
import os
import shlex
import subprocess
import sys
DEFAULT_PORT_BASE = 9440
DEFAULT_MPC = "mpc"
# ---------------------------------------------------------------------------
# 纯逻辑:计划生成
# ---------------------------------------------------------------------------
def parse_data_map(raw: str) -> dict[str, str]:
"""解析 `alice=2,bob=3,carol=5` -> {site: "2"}。
允许值里带逗号(矩阵场景),所以按 `name=` 切分而不是按逗号切。
"""
out: dict[str, str] = {}
if not raw:
return out
for chunk in raw.split(","):
chunk = chunk.strip()
if not chunk:
continue
if "=" not in chunk:
raise SystemExit(f"--data 里 {chunk!r} 不是 site=values 形式")
name, values = chunk.split("=", 1)
name = name.strip()
if not name:
raise SystemExit(f"--data 里 {chunk!r} 的站点名为空")
if name in out:
raise SystemExit(f"--data 里站点 {name} 重复")
out[name] = values.strip()
return out
def build_plan(
sites: list[str],
hosts: dict[str, str],
ports: dict[str, int],
data: dict[str, str],
mpc_bin: str = DEFAULT_MPC,
fxp: int = 18,
seed: str | None = None,
) -> dict[str, dict]:
"""生成每个 site 的启动命令。
**字典序排序是协议要求,不是美观**:`Vault` 内部按字典序给参与方编号,
第 p 片分片投递给"排序后第 p 个"。如果这里按别的顺序分配端口和 peers,
各方对 party 编号的理解就会不一致 —— 分片投递对象整体错位,
结果静默算错(不崩、不报警,就是数不对)。
`--peers` 里给自己那条也要给:CLI 会忽略自己那一格,
但保留它让参数表完整、可读,也方便人工核对。
"""
ordered = sorted(sites)
missing = [s for s in ordered if s not in hosts]
if missing:
raise SystemExit(f"缺少站点地址: {missing}")
missing = [s for s in ordered if s not in ports]
if missing:
raise SystemExit(f"缺少站点端口: {missing}")
missing = [s for s in ordered if s not in data]
if missing:
raise SystemExit(f"--data 缺少站点: {missing}")
peers = ",".join(f"{s}={hosts[s]}:{ports[s]}" for s in ordered)
sites_arg = ",".join(ordered)
plan: dict[str, dict] = {}
for s in ordered:
cmd = [
mpc_bin, "node",
"--site", s,
"--sites", sites_arg,
"--listen", f"0.0.0.0:{ports[s]}",
"--peers", peers,
"--data", data[s],
"--fxp", str(fxp),
]
if seed:
cmd += ["--seed", seed]
plan[s] = {
"host": hosts[s],
"port": ports[s],
"party_index": ordered.index(s),
"cmd": cmd,
}
return plan
def format_plan(plan: dict[str, dict]) -> str:
lines = []
for s, item in plan.items():
lines.append(f"[{item['party_index']}] {s} @ {item['host']}:{item['port']}")
lines.append(" " + " ".join(shlex.quote(c) for c in item["cmd"]))
return "\n".join(lines)
# ---------------------------------------------------------------------------
# 执行后端一:本机并行(不需要 Ray)
# ---------------------------------------------------------------------------
def run_local(plan: dict[str, dict], timeout: int) -> int:
procs = {}
for s, item in plan.items():
env = dict(os.environ)
print(f"-- {s} 启动", flush=True)
procs[s] = subprocess.Popen(
item["cmd"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, env=env
)
failed = 0
for s, p in procs.items():
try:
out, _ = p.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
p.kill()
out, _ = p.communicate()
print(f"== {s} 超时({timeout}s)", file=sys.stderr)
failed += 1
print(f"\n===== {s} (rc={p.returncode}) =====")
print(out.rstrip())
if p.returncode != 0:
failed += 1
return 1 if failed else 0
# ---------------------------------------------------------------------------
# 执行后端二:Ray(钉机器)
# ---------------------------------------------------------------------------
def run_ray(plan: dict[str, dict], address: str | None, timeout: int) -> int:
try:
import ray
except ImportError:
raise SystemExit("未安装 ray。pip install 'ray[default]',或先用 --local / --dry-run")
if address:
ray.init(address=address)
else:
ray.init()
@ray.remote
class NodeSlot:
"""占位 actor:只做两件事 —— 报本机地址、起进程。
两阶段是必须的:第一轮各 actor 先报 IP,driver 才能拼出完整的
peers 表;第二轮才带着 peers 启动进程。若一轮就起进程,
先启动的方不知道自己该连谁。
"""
def __init__(self, site: str):
self.site = site
def where(self) -> str:
return ray.util.get_node_ip_address()
def run(self, cmd: list[str]) -> dict:
p = subprocess.run(cmd, capture_output=True, text=True)
return {"rc": p.returncode, "out": p.stdout, "err": p.stderr}
# 第一轮:按资源标签占位,拿到每台机器的真实 IP
slots = {}
for s in plan:
slots[s] = NodeSlot.options(resources={f"{s}_node": 1.0}).remote(s)
hosts = {s: ray.get(slots[s].where.remote()) for s in plan}
print("各站点落点:", json.dumps(hosts, ensure_ascii=False), flush=True)
# 第二轮:用真实 IP 重建计划(actor 里 listen 仍是 0.0.0.0)
ports = {s: plan[s]["port"] for s in plan}
data = {}
for s in plan:
cmd = plan[s]["cmd"]
data[s] = cmd[cmd.index("--data") + 1]
rebuilt = build_plan(
list(plan.keys()),
hosts,
ports,
data,
mpc_bin=plan[next(iter(plan))]["cmd"][0],
fxp=int(plan[next(iter(plan))]["cmd"][plan[next(iter(plan))]["cmd"].index("--fxp") + 1]),
)
print(format_plan(rebuilt), flush=True)
futures = {s: slots[s].run.remote(rebuilt[s]["cmd"]) for s in rebuilt}
failed = 0
for s, fut in futures.items():
r = ray.get(fut, timeout=timeout)
print(f"\n===== {s} (rc={r['rc']}) =====")
print(r["out"].rstrip())
if r["err"].strip():
print("[stderr]", r["err"].rstrip(), file=sys.stderr)
if r["rc"] != 0:
failed += 1
ray.shutdown()
return 1 if failed else 0
# ---------------------------------------------------------------------------
def main() -> int:
ap = argparse.ArgumentParser(description="mpc 集群编排(Ray 只管进程,不管数据)")
ap.add_argument("--sites", required=True, help="参与方,逗号分隔,如 alice,bob,carol")
ap.add_argument("--data", required=True, help="各方数据,如 alice=2,bob=3,carol=5")
ap.add_argument("--mpc", default=DEFAULT_MPC, help=f"mpc 二进制路径(默认 {DEFAULT_MPC})")
ap.add_argument("--port-base", type=int, default=DEFAULT_PORT_BASE, help="端口基准(按字典序递增)")
ap.add_argument("--fxp", type=int, default=18, help="定点小数位")
ap.add_argument("--seed", default=None, help="会话主种子(64 位十六进制;生产应来自 TEE)")
ap.add_argument("--address", default=None, help="Ray 集群地址,如 ray://head:10001")
ap.add_argument("--timeout", type=int, default=300, help="单方执行超时秒数")
ap.add_argument("--dry-run", action="store_true", help="只打印计划,不起进程")
ap.add_argument("--local", action="store_true", help="本机并行起进程(127.0.0.1),不需要 Ray")
args = ap.parse_args()
sites = [s.strip() for s in args.sites.split(",") if s.strip()]
if len(sites) < 2:
raise SystemExit("--sites 至少 2 方")
data = parse_data_map(args.data)
ordered = sorted(sites)
if args.local:
hosts = {s: "127.0.0.1" for s in ordered}
else:
# Ray 模式下 host 由 actor 报;这里先占位,run_ray 会重建计划
hosts = {s: "0.0.0.0" for s in ordered}
ports = {s: args.port_base + i for i, s in enumerate(ordered)}
plan = build_plan(
sites, hosts, ports, data,
mpc_bin=args.mpc, fxp=args.fxp, seed=args.seed,
)
print("== mpc 集群计划 ==")
print(format_plan(plan))
print()
if args.dry_run:
return 0
if args.local:
return run_local(plan, args.timeout)
return run_ray(plan, args.address, args.timeout)
if __name__ == "__main__":
sys.exit(main())