Ray 这条线不绕 bazel: - wheel 里的 ray/_raylet.so 就是编好的 C++ core(34113 个导出符号, 3638 个 ray:: 符号 default 可见,含 CoreWorkerProcess::Initialize) - ray/nax_ray_shim.cc:259 行 C ABI shim,复用 process_helper 的初始化路径 - 实测跑通:Rust 二进制作为 driver 接入真实 GCS+raylet, 拿到 raylet 分配的 node_id(d190801c…),probe 通过 踩坑(都已固化进 ray/build.sh): - raylet 只编了 Python worker pool → 注册时语言声明 PYTHON - Ray C++ 是旧 ABI → -D_GLIBCXX_USE_CXX11_ABI=0 - fmt v9/v11 撞车 → FMT_HEADER_ONLY - Debian absl 固定自家 optional → 影子头直通 std::optional - _raylet.so 是 CPython 扩展(253 个 Py 符号)→ build.rs 自动链 libpython - GcsNodeInfo 手写 wire format 解析,避免 protobuf 跨 ABI armv7 交叉编译修正(原结论有误): - 换 armv7-unknown-linux-musleabihf + rust-lld/link-self-contained - 不再需要外部交叉 gcc,macOS 本机 83 秒出 687 KB 静态二进制 - .cargo/config.toml 固化,无需 export 任何变量 其他:nax-orch 告警清理;README/ZYNQ 文档对齐实测。
135 lines
4.8 KiB
Python
135 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
||
"""第二轮:真正跑起 bazel 构建,量时间/磁盘/产物体积/ABI。
|
||
|
||
上轮两处失误已修:
|
||
- bazelisk 要走 GCS 解析 "latest"(沙箱连不上),改成直下 7.5.0 二进制
|
||
- 用 /usr/bin/time 计时(镜像里没有),改成 date +%s
|
||
"""
|
||
|
||
import sys
|
||
import os
|
||
import time
|
||
import json
|
||
|
||
sys.path.insert(0, "/tmp")
|
||
import sbfix # noqa: F401
|
||
from cubesandbox import Sandbox
|
||
|
||
SID = "ccb00b05e9f045d99fd99a6e2aecd20b"
|
||
JOBS = os.environ.get("RAY_BAZEL_JOBS", "8")
|
||
|
||
|
||
def log(m=""):
|
||
print(m, flush=True)
|
||
|
||
|
||
def main():
|
||
t_all = time.time()
|
||
sb = None
|
||
reused = False
|
||
if SID:
|
||
try:
|
||
sb = Sandbox.connect(SID)
|
||
r = sb.commands.run("cat /ray/.bazelversion 2>/dev/null; nproc", timeout=30)
|
||
if "/ray" in (r.stdout or "") or (r.stdout or "").strip():
|
||
reused = True
|
||
log(f"复用沙箱 {SID}")
|
||
except Exception as e:
|
||
log(f"connect 失败: {type(e).__name__}: {e}")
|
||
sb = None
|
||
|
||
if sb is None:
|
||
sb = Sandbox.create(
|
||
template="toolchain",
|
||
timeout=7200,
|
||
network={"allow_public_traffic": True},
|
||
)
|
||
log(f"新建沙箱 {sb.sandbox_id}")
|
||
open("/tmp/ray_cost_sid.txt", "w").write(sb.sandbox_id)
|
||
|
||
def sh(cmd, t=120, label=""):
|
||
if label:
|
||
log(f"\n===== {label} =====")
|
||
try:
|
||
r = sb.commands.run(cmd, timeout=t)
|
||
except Exception as e:
|
||
log(f"[EXC] {type(e).__name__}: {e}")
|
||
return ""
|
||
out = (r.stdout or "").strip()
|
||
log(out if out else "(无输出)")
|
||
err = (r.stderr or "").strip()
|
||
if err:
|
||
log("STDERR: " + err[:500])
|
||
return r.stdout or ""
|
||
|
||
# ---------- 0: 现状 ----------
|
||
sh("cat /ray/.bazelversion; du -sh /ray; df -h / | tail -1; which bazel clang", 60, "STEP0 沙箱现状")
|
||
|
||
# ---------- 1: 网络探测(决定 bazel 能不能自己拉依赖)----------
|
||
sh(
|
||
"for u in https://github.com https://storage.googleapis.com "
|
||
"https://ghfast.top https://mirrors.ustc.edu.cn; do "
|
||
" printf '%-34s ' $u; "
|
||
" curl -s -o /dev/null -w '%{http_code} %{time_total}s\\n' --max-time 8 $u || echo FAIL; done",
|
||
90,
|
||
"STEP1 外网可达性",
|
||
)
|
||
|
||
# ---------- 2: 装 bazel 7.5.0(直下二进制,绕开 GCS)----------
|
||
sh(
|
||
"curl -sSL --http1.1 --max-time 600 -o /usr/local/bin/bazel "
|
||
"https://ghfast.top/https://github.com/bazelbuild/bazel/releases/download/7.5.0/bazel-7.5.0-linux-x86_64 "
|
||
"&& chmod +x /usr/local/bin/bazel && bazel --version",
|
||
900,
|
||
"STEP2 安装 bazel 7.5.0",
|
||
)
|
||
|
||
# ---------- 3: 依赖预热(bazel fetch 会在这里暴露网络问题)----------
|
||
t0 = time.time()
|
||
fetch = sh(
|
||
"cd /ray && timeout 1500 bazel fetch //:raylet 2>&1 | tail -25; echo FETCH_RC=$?",
|
||
1800,
|
||
"STEP3 bazel fetch //:raylet(拉 34 个依赖)",
|
||
)
|
||
fetch_secs = round(time.time() - t0)
|
||
log(f">>> fetch 耗时 {fetch_secs}s")
|
||
|
||
# ---------- 4: 构建 ----------
|
||
t0 = time.time()
|
||
build = sh(
|
||
"cd /ray && S=$(date +%s) && "
|
||
"bazel build -c opt --jobs=" + JOBS + " --noshow_progress "
|
||
"--disk_cache=/root/bazelcache //:raylet //:ray_core_worker_lib 2>&1 | tail -30; "
|
||
"RC=$?; E=$(date +%s); echo BUILD_RC=$RC BUILD_SECS=$((E-S))",
|
||
5400,
|
||
"STEP4 bazel build(jobs=" + JOBS + ")",
|
||
)
|
||
wall = round(time.time() - t0)
|
||
log(f"\n>>> 构建墙钟 {wall}s({wall/60:.1f} 分钟)")
|
||
|
||
# ---------- 5: 量产物 ----------
|
||
sh(
|
||
"cd /ray && echo '--- 产物体积 ---'; "
|
||
"ls -lh bazel-bin/raylet bazel-bin/libray_core_worker_lib.so bazel-bin/gcs_server 2>/dev/null "
|
||
"| awk '{print $5, $9}'; "
|
||
"echo '--- 磁盘占用 ---'; du -sh /root/.cache/bazel 2>/dev/null; du -sh bazel-bin 2>/dev/null; "
|
||
"df -h / | tail -1; "
|
||
"echo '--- 动态依赖条数 ---'; "
|
||
"ldd bazel-bin/raylet 2>/dev/null | wc -l; "
|
||
"echo '--- 动态依赖清单(前 25)---'; ldd bazel-bin/raylet 2>/dev/null | head -25 | awk '{print $1}'; "
|
||
"echo '--- 导出符号:总 T / C++ 修饰 _Z ---'; "
|
||
"nm -D --defined-only bazel-bin/libray_core_worker_lib.so 2>/dev/null | grep -c ' T '; "
|
||
"nm -D --defined-only bazel-bin/libray_core_worker_lib.so 2>/dev/null | grep ' T ' | grep -c ' _Z'; "
|
||
"echo '--- 是否有 C 入口 ---'; "
|
||
"nm -D --defined-only bazel-bin/libray_core_worker_lib.so 2>/dev/null | grep -E ' T (ray_|ray_init|ray_shutdown)' | head -3; "
|
||
"echo end",
|
||
300,
|
||
"STEP5 产物 / 依赖 / ABI",
|
||
)
|
||
|
||
log(f"\n总耗时 {round(time.time()-t_all)}s")
|
||
log("SANDBOX " + sb.sandbox_id)
|
||
|
||
|
||
main()
|