Files
ssw-mpc/build/measure_ray_core3.py
T
rong 4bb573d98c feat: Ray C++ 内核接进 nax Rust + armv7 真交叉编译
Ray 这条线不绕 bazel:
- wheel 里的 ray/_raylet.so 就是编好的 C++ core(34113 个导出符号,
  3638 个 ray:: 符号 default 可见,含 CoreWorkerProcess::Initialize)
- ray/nax_ray_shim.cc:259 行 C ABI shim,复用 process_helper 的初始化路径
- 实测跑通:Rust 二进制作为 driver 接入真实 GCS+raylet,
  拿到 raylet 分配的 node_id(d190801c…),probe 通过

踩坑(都已固化进 ray/build.sh):
- raylet 只编了 Python worker pool → 注册时语言声明 PYTHON
- Ray C++ 是旧 ABI → -D_GLIBCXX_USE_CXX11_ABI=0
- fmt v9/v11 撞车 → FMT_HEADER_ONLY
- Debian absl 固定自家 optional → 影子头直通 std::optional
- _raylet.so 是 CPython 扩展(253 个 Py 符号)→ build.rs 自动链 libpython
- GcsNodeInfo 手写 wire format 解析,避免 protobuf 跨 ABI

armv7 交叉编译修正(原结论有误):
- 换 armv7-unknown-linux-musleabihf + rust-lld/link-self-contained
- 不再需要外部交叉 gcc,macOS 本机 83 秒出 687 KB 静态二进制
- .cargo/config.toml 固化,无需 export 任何变量

其他:nax-orch 告警清理;README/ZYNQ 文档对齐实测。
2026-10-30 11:31:02 +08:00

141 lines
4.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""第三轮:真正跑 bazel 构建并量出时间/磁盘/体积/ABI。
沙箱连不上 github(42 个依赖 URL 全指向它),但能连到本机。
所以本进程内起一个 HTTP 服务把 mac 上拉好的 distdir 喂给沙箱,
sandbox 再 curl 到本地 /distdir,bazel 用 --distdir 避免联网。
"""
import http.server
import json
import os
import socketserver
import sys
import threading
import time
sys.path.insert(0, "/tmp")
import sbfix # noqa: F401
from cubesandbox import Sandbox
SID = "ccb00b05e9f045d99fd99a6e2aecd20b"
MAC_IP = "192.168.1.19"
PORT = 18080
DIST = "/tmp/raydist"
JOBS = os.environ.get("RAY_BAZEL_JOBS", "8")
def log(m=""):
print(m, flush=True)
class Handler(http.server.SimpleHTTPRequestHandler):
def __init__(self, *a, **kw):
super().__init__(*a, directory=DIST, **kw)
def log_message(self, *a):
pass
def serve():
socketserver.TCPServer.allow_reuse_address = True
httpd = socketserver.ThreadingTCPServer(("0.0.0.0", PORT), Handler)
threading.Thread(target=httpd.serve_forever, daemon=True).start()
log(f"[dist] serving {DIST} on 0.0.0.0:{PORT}")
def main():
t_all = time.time()
serve()
manifest = [l.strip() for l in open("/tmp/raydist_manifest.txt") if l.strip()]
log(f"[dist] {len(manifest)} 个包,"
f"{sum(os.path.getsize(os.path.join(DIST, f)) for f in manifest)/1e6:.0f} MB")
sb = Sandbox.connect(SID)
log(f"connected {SID}")
def sh(cmd, t=120, label=""):
if label:
log(f"\n===== {label} =====")
try:
r = sb.commands.run(cmd, timeout=t)
except Exception as e:
log(f"[EXC] {type(e).__name__}: {e}")
return ""
out = (r.stdout or "").strip()
log(out if out else "(无输出)")
err = (r.stderr or "").strip()
if err:
log("STDERR: " + err[:400])
return r.stdout or ""
# 1) 放开到本机的出站
try:
r = sb.update_network({"allow_out": ["192.168.1.0/24"], "allow_internet_access": True})
log(f"update_network -> {r}")
except Exception as e:
log(f"update_network EXC {type(e).__name__}: {e}")
sh(f"curl -s -o /dev/null -w 'mac:%{{http_code}} %{{time_total}}s\\n' --max-time 10 "
f"http://{MAC_IP}:{PORT}/", 40, "STEP1 沙箱 -> mac distdir 服务")
# 2) 装 bazel(releases.bazel.build 可达)
sh(
"curl -sSL --http1.1 --max-time 600 -o /usr/local/bin/bazel "
"https://releases.bazel.build/7.5.0/release/bazel-7.5.0-linux-x86_64 "
"&& chmod +x /usr/local/bin/bazel && /usr/local/bin/bazel --version",
900,
"STEP2 安装 bazel 7.5.0",
)
# 3) 从 mac 取 distdir
t0 = time.time()
names = " ".join(manifest)
sh(
f"rm -rf /distdir && mkdir -p /distdir && cd /distdir && "
f"for f in {names}; do curl -sS --max-time 300 -O http://{MAC_IP}:{PORT}/$f || echo MISS $f; done; "
f"ls /distdir | wc -l; du -sh /distdir",
1800,
"STEP3 从 mac 拉 distdir",
)
log(f">>> distdir 传输 {round(time.time()-t0)}s")
# 4) 构建
t0 = time.time()
build = sh(
"cd /ray && S=$(date +%s); "
"bazel build -c opt --jobs=" + JOBS + " --noshow_progress "
"--distdir=/distdir --repository_cache=/root/bazelrepo "
"//:raylet //:ray_core_worker_lib 2>&1 | tail -35; "
"E=$(date +%s); echo BUILD_SECS=$((E-S))",
5400,
"STEP4 bazel build //:raylet //:ray_core_worker_lib",
)
wall = round(time.time() - t0)
log(f"\n>>> 构建墙钟 {wall}s = {wall/60:.1f} 分钟(jobs={JOBS}, 16 vCPU / 15GB)")
# 5) 量产物
sh(
"cd /ray && echo '--- 产物体积 ---'; "
"ls -lh bazel-bin/raylet bazel-bin/libray_core_worker_lib.so 2>/dev/null | awk '{print $5, $9}'; "
"echo '--- 磁盘 ---'; du -sh /root/.cache/bazel 2>/dev/null; du -sh bazel-bin 2>/dev/null; "
"df -h / | tail -1; "
"echo '--- 动态依赖条数 ---'; ldd bazel-bin/raylet 2>/dev/null | wc -l; "
"echo '--- 依赖清单 ---'; ldd bazel-bin/raylet 2>/dev/null | awk '{print $1}' | tr '\\n' ' '; echo; "
"echo '--- 导出符号 T 总数 / C++ 修饰 _Z 数 ---'; "
"nm -D --defined-only bazel-bin/libray_core_worker_lib.so 2>/dev/null | grep -c ' T '; "
"nm -D --defined-only bazel-bin/libray_core_worker_lib.so 2>/dev/null | grep ' T ' | grep -c ' _Z'; "
"echo '--- 有无 C 入口 ---'; "
"nm -D --defined-only bazel-bin/libray_core_worker_lib.so 2>/dev/null | grep -E ' T (ray_|ray_init|ray_shutdown)' | head -3; "
"echo end",
300,
"STEP5 产物 / 依赖 / ABI",
)
log(f"\n总耗时 {round(time.time()-t_all)}s")
log("SANDBOX " + SID)
time.sleep(2)
main()