Ray 这条线不绕 bazel: - wheel 里的 ray/_raylet.so 就是编好的 C++ core(34113 个导出符号, 3638 个 ray:: 符号 default 可见,含 CoreWorkerProcess::Initialize) - ray/nax_ray_shim.cc:259 行 C ABI shim,复用 process_helper 的初始化路径 - 实测跑通:Rust 二进制作为 driver 接入真实 GCS+raylet, 拿到 raylet 分配的 node_id(d190801c…),probe 通过 踩坑(都已固化进 ray/build.sh): - raylet 只编了 Python worker pool → 注册时语言声明 PYTHON - Ray C++ 是旧 ABI → -D_GLIBCXX_USE_CXX11_ABI=0 - fmt v9/v11 撞车 → FMT_HEADER_ONLY - Debian absl 固定自家 optional → 影子头直通 std::optional - _raylet.so 是 CPython 扩展(253 个 Py 符号)→ build.rs 自动链 libpython - GcsNodeInfo 手写 wire format 解析,避免 protobuf 跨 ABI armv7 交叉编译修正(原结论有误): - 换 armv7-unknown-linux-musleabihf + rust-lld/link-self-contained - 不再需要外部交叉 gcc,macOS 本机 83 秒出 687 KB 静态二进制 - .cargo/config.toml 固化,无需 export 任何变量 其他:nax-orch 告警清理;README/ZYNQ 文档对齐实测。
141 lines
4.8 KiB
Python
141 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
||
"""第三轮:真正跑 bazel 构建并量出时间/磁盘/体积/ABI。
|
||
|
||
沙箱连不上 github(42 个依赖 URL 全指向它),但能连到本机。
|
||
所以本进程内起一个 HTTP 服务把 mac 上拉好的 distdir 喂给沙箱,
|
||
sandbox 再 curl 到本地 /distdir,bazel 用 --distdir 避免联网。
|
||
"""
|
||
|
||
import http.server
|
||
import json
|
||
import os
|
||
import socketserver
|
||
import sys
|
||
import threading
|
||
import time
|
||
|
||
sys.path.insert(0, "/tmp")
|
||
import sbfix # noqa: F401
|
||
from cubesandbox import Sandbox
|
||
|
||
SID = "ccb00b05e9f045d99fd99a6e2aecd20b"
|
||
MAC_IP = "192.168.1.19"
|
||
PORT = 18080
|
||
DIST = "/tmp/raydist"
|
||
JOBS = os.environ.get("RAY_BAZEL_JOBS", "8")
|
||
|
||
|
||
def log(m=""):
|
||
print(m, flush=True)
|
||
|
||
|
||
class Handler(http.server.SimpleHTTPRequestHandler):
|
||
def __init__(self, *a, **kw):
|
||
super().__init__(*a, directory=DIST, **kw)
|
||
|
||
def log_message(self, *a):
|
||
pass
|
||
|
||
|
||
def serve():
|
||
socketserver.TCPServer.allow_reuse_address = True
|
||
httpd = socketserver.ThreadingTCPServer(("0.0.0.0", PORT), Handler)
|
||
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
||
log(f"[dist] serving {DIST} on 0.0.0.0:{PORT}")
|
||
|
||
|
||
def main():
|
||
t_all = time.time()
|
||
serve()
|
||
manifest = [l.strip() for l in open("/tmp/raydist_manifest.txt") if l.strip()]
|
||
log(f"[dist] {len(manifest)} 个包,"
|
||
f"{sum(os.path.getsize(os.path.join(DIST, f)) for f in manifest)/1e6:.0f} MB")
|
||
|
||
sb = Sandbox.connect(SID)
|
||
log(f"connected {SID}")
|
||
|
||
def sh(cmd, t=120, label=""):
|
||
if label:
|
||
log(f"\n===== {label} =====")
|
||
try:
|
||
r = sb.commands.run(cmd, timeout=t)
|
||
except Exception as e:
|
||
log(f"[EXC] {type(e).__name__}: {e}")
|
||
return ""
|
||
out = (r.stdout or "").strip()
|
||
log(out if out else "(无输出)")
|
||
err = (r.stderr or "").strip()
|
||
if err:
|
||
log("STDERR: " + err[:400])
|
||
return r.stdout or ""
|
||
|
||
# 1) 放开到本机的出站
|
||
try:
|
||
r = sb.update_network({"allow_out": ["192.168.1.0/24"], "allow_internet_access": True})
|
||
log(f"update_network -> {r}")
|
||
except Exception as e:
|
||
log(f"update_network EXC {type(e).__name__}: {e}")
|
||
|
||
sh(f"curl -s -o /dev/null -w 'mac:%{{http_code}} %{{time_total}}s\\n' --max-time 10 "
|
||
f"http://{MAC_IP}:{PORT}/", 40, "STEP1 沙箱 -> mac distdir 服务")
|
||
|
||
# 2) 装 bazel(releases.bazel.build 可达)
|
||
sh(
|
||
"curl -sSL --http1.1 --max-time 600 -o /usr/local/bin/bazel "
|
||
"https://releases.bazel.build/7.5.0/release/bazel-7.5.0-linux-x86_64 "
|
||
"&& chmod +x /usr/local/bin/bazel && /usr/local/bin/bazel --version",
|
||
900,
|
||
"STEP2 安装 bazel 7.5.0",
|
||
)
|
||
|
||
# 3) 从 mac 取 distdir
|
||
t0 = time.time()
|
||
names = " ".join(manifest)
|
||
sh(
|
||
f"rm -rf /distdir && mkdir -p /distdir && cd /distdir && "
|
||
f"for f in {names}; do curl -sS --max-time 300 -O http://{MAC_IP}:{PORT}/$f || echo MISS $f; done; "
|
||
f"ls /distdir | wc -l; du -sh /distdir",
|
||
1800,
|
||
"STEP3 从 mac 拉 distdir",
|
||
)
|
||
log(f">>> distdir 传输 {round(time.time()-t0)}s")
|
||
|
||
# 4) 构建
|
||
t0 = time.time()
|
||
build = sh(
|
||
"cd /ray && S=$(date +%s); "
|
||
"bazel build -c opt --jobs=" + JOBS + " --noshow_progress "
|
||
"--distdir=/distdir --repository_cache=/root/bazelrepo "
|
||
"//:raylet //:ray_core_worker_lib 2>&1 | tail -35; "
|
||
"E=$(date +%s); echo BUILD_SECS=$((E-S))",
|
||
5400,
|
||
"STEP4 bazel build //:raylet //:ray_core_worker_lib",
|
||
)
|
||
wall = round(time.time() - t0)
|
||
log(f"\n>>> 构建墙钟 {wall}s = {wall/60:.1f} 分钟(jobs={JOBS}, 16 vCPU / 15GB)")
|
||
|
||
# 5) 量产物
|
||
sh(
|
||
"cd /ray && echo '--- 产物体积 ---'; "
|
||
"ls -lh bazel-bin/raylet bazel-bin/libray_core_worker_lib.so 2>/dev/null | awk '{print $5, $9}'; "
|
||
"echo '--- 磁盘 ---'; du -sh /root/.cache/bazel 2>/dev/null; du -sh bazel-bin 2>/dev/null; "
|
||
"df -h / | tail -1; "
|
||
"echo '--- 动态依赖条数 ---'; ldd bazel-bin/raylet 2>/dev/null | wc -l; "
|
||
"echo '--- 依赖清单 ---'; ldd bazel-bin/raylet 2>/dev/null | awk '{print $1}' | tr '\\n' ' '; echo; "
|
||
"echo '--- 导出符号 T 总数 / C++ 修饰 _Z 数 ---'; "
|
||
"nm -D --defined-only bazel-bin/libray_core_worker_lib.so 2>/dev/null | grep -c ' T '; "
|
||
"nm -D --defined-only bazel-bin/libray_core_worker_lib.so 2>/dev/null | grep ' T ' | grep -c ' _Z'; "
|
||
"echo '--- 有无 C 入口 ---'; "
|
||
"nm -D --defined-only bazel-bin/libray_core_worker_lib.so 2>/dev/null | grep -E ' T (ray_|ray_init|ray_shutdown)' | head -3; "
|
||
"echo end",
|
||
300,
|
||
"STEP5 产物 / 依赖 / ABI",
|
||
)
|
||
|
||
log(f"\n总耗时 {round(time.time()-t_all)}s")
|
||
log("SANDBOX " + SID)
|
||
time.sleep(2)
|
||
|
||
|
||
main()
|