Files
ssw-mpc/build/measure_ray_core.py
T
rong 4bb573d98c feat: Ray C++ 内核接进 nax Rust + armv7 真交叉编译
Ray 这条线不绕 bazel:
- wheel 里的 ray/_raylet.so 就是编好的 C++ core(34113 个导出符号,
  3638 个 ray:: 符号 default 可见,含 CoreWorkerProcess::Initialize)
- ray/nax_ray_shim.cc:259 行 C ABI shim,复用 process_helper 的初始化路径
- 实测跑通:Rust 二进制作为 driver 接入真实 GCS+raylet,
  拿到 raylet 分配的 node_id(d190801c…),probe 通过

踩坑(都已固化进 ray/build.sh):
- raylet 只编了 Python worker pool → 注册时语言声明 PYTHON
- Ray C++ 是旧 ABI → -D_GLIBCXX_USE_CXX11_ABI=0
- fmt v9/v11 撞车 → FMT_HEADER_ONLY
- Debian absl 固定自家 optional → 影子头直通 std::optional
- _raylet.so 是 CPython 扩展(253 个 Py 符号)→ build.rs 自动链 libpython
- GcsNodeInfo 手写 wire format 解析,避免 protobuf 跨 ABI

armv7 交叉编译修正(原结论有误):
- 换 armv7-unknown-linux-musleabihf + rust-lld/link-self-contained
- 不再需要外部交叉 gcc,macOS 本机 83 秒出 687 KB 静态二进制
- .cargo/config.toml 固化,无需 export 任何变量

其他:nax-orch 告警清理;README/ZYNQ 文档对齐实测。
2026-10-30 11:31:02 +08:00

219 lines
8.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""量清 Ray C++ core 的真实代价。
回答四个问题,不猜:
1. 它到底要哪些第三方依赖(个数 / 版本)
2. bazel 从零构建要多久、吃多少内存和磁盘
3. 产物多大、动态依赖多少、有没有 C ABI 可供 Rust 直接调
4. 有没有 32 位 ARM(armv7)的构建目标 —— 决定 Zynq-7015 能不能用
必须在单进程内跑完:沙箱离开创建它的进程就会被回收。
"""
import sys
import os
import time
import json
sys.path.insert(0, "/tmp")
import sbfix # noqa: F401 (cube-auth-gate gzip bug 绕过)
from cubesandbox import Sandbox
JOBS = os.environ.get("RAY_BAZEL_JOBS", "8")
OUT = "/tmp/ray_core_cost.json"
def log(msg=""):
print(msg, flush=True)
def main():
t_all = time.time()
sb = Sandbox.create(
template="toolchain",
timeout=7200,
network={"allow_public_traffic": True},
)
sid = sb.sandbox_id
tok = sb._data.get("trafficAccessToken") or ""
log(f"SANDBOX {sid} token_len={len(tok)}")
open("/tmp/ray_cost_sid.txt", "w").write(sid)
open(f"/tmp/sb_{sid}.token", "w").write(tok)
result = {"sandbox": sid, "steps": {}}
def sh(cmd, t=120, label="", quiet=False):
"""跑一条命令,返回 stdout。所有调用都必须显式给 timeout。"""
if label:
log(f"\n===== {label} =====")
try:
r = sb.commands.run(cmd, timeout=t)
except Exception as e: # 超时不等于没执行,但这里如实记录
log(f"[EXC] {type(e).__name__}: {e}")
return ""
if not quiet:
out = (r.stdout or "").strip()
log(out if out else "(无输出)")
err = (r.stderr or "").strip()
if err:
log("STDERR: " + err[:600])
return r.stdout or ""
# ---------- STEP 0: 机器规格 ----------
sh(
"echo cpus=$(nproc); "
"free -g | awk 'NR==2{print \"mem_gb=\"$2}'; "
"df -h / /root | tail -2; "
"which gcc clang bazel git curl python3 2>/dev/null; "
"cat /etc/os-release | grep PRETTY",
60,
"STEP0 机器规格",
)
# ---------- STEP 1: 构建依赖(Ray 官方要求 clang + psmisc + unzip)----------
sh(
"sed -i 's|http://deb.debian.org|https://mirrors.ustc.edu.cn|g' "
"/etc/apt/sources.list.d/debian.sources 2>/dev/null; "
"apt-get update -qq 2>&1 | tail -2; "
"DEBIAN_FRONTEND=noninteractive apt-get install -y -qq "
"clang lld unzip zip pkg-config psmisc patch file 2>&1 | tail -3; "
"clang --version | head -1",
900,
"STEP1 构建依赖",
)
# ---------- STEP 2: 取 Ray 源码 ----------
tag = sh(
"git ls-remote --tags https://ghfast.top/https://github.com/ray-project/ray.git "
"2>/dev/null | sed 's#.*refs/tags/##' | grep -E '^ray-[0-9]+\\.[0-9]+\\.[0-9]+$' "
"| sort -V | tail -1",
180,
"STEP2 查最新 release tag",
).strip()
if not tag:
tag = "ray-2.9.3"
log(f"选定 tag: {tag}")
result["tag"] = tag
t0 = time.time()
sh(
f"rm -rf /ray && mkdir -p /ray && "
f"curl -sSL --http1.1 --max-time 900 "
f"https://ghfast.top/https://github.com/ray-project/ray/archive/refs/tags/{tag}.tar.gz "
f"-o /tmp/ray.tgz && ls -lh /tmp/ray.tgz && "
f"tar xzf /tmp/ray.tgz -C /ray --strip-components=1 && "
f"du -sh /ray && cat /ray/.bazelversion 2>/dev/null",
1200,
"STEP2 下载并解压源码",
)
result["steps"]["fetch_secs"] = round(time.time() - t0)
# ---------- STEP 3: 静态分析(不构建也能定案的部分)----------
arm = sh(
"cd /ray && echo '--- bazel 平台定义 ---'; ls bazel/ 2>/dev/null; "
"echo '--- 平台 target ---'; grep -rn 'name = \"linux_' bazel/BUILD* 2>/dev/null | head -20; "
"echo '--- 任何 armv7/armhf/32位 痕迹 ---'; "
"grep -rniE 'armv7|armhf|arm_32|arm32|aarch32' --include='*.bzl' --include='BUILD*' "
"--include='.bazelrc' . 2>/dev/null | grep -v thirdparty/ | head -10; "
"echo 'end'",
120,
"STEP3 armv7 支持度",
)
result["armv7_hits"] = [l for l in arm.splitlines() if "armv7" in l.lower() or "armhf" in l.lower()]
deps = sh(
"cd /ray && echo '--- http_archive 依赖个数 ---'; "
"grep -c ' http_archive(' bazel/ray_deps_setup.bzl 2>/dev/null; "
"echo '--- 依赖名 ---'; "
"grep -oE 'name = \"[a-z0-9_]+' bazel/ray_deps_setup.bzl 2>/dev/null | sed 's/.*\"//' | tr '\\n' ' '; "
"echo; echo '--- 源码规模 ---'; "
"find . -name '*.cc' -o -name '*.cpp' | wc -l; "
"du -sh src/ cpp/ 2>/dev/null",
120,
"STEP3 第三方依赖清单",
)
result["deps_raw"] = deps[:2000]
# ---------- STEP 4: 装 bazel ----------
baz = sh(
"curl -sSL --http1.1 --max-time 300 -o /usr/local/bin/bazel "
"https://ghfast.top/https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-linux-amd64 "
"&& chmod +x /usr/local/bin/bazel && bazel version 2>&1 | head -3",
600,
"STEP4 安装 bazelisk",
)
result["bazel"] = baz.strip().splitlines()[:2]
# ---------- STEP 5: 构建(真金白银)----------
targets = sh(
"cd /ray && grep -nE 'name = \"(raylet|gcs_server|ray_core_worker_lib|libray_agent)\"' BUILD.bazel | head",
60,
"STEP5 确认构建目标",
)
log(targets)
t0 = time.time()
build = sh(
f"cd /ray && sync && "
f"/usr/bin/time -v bazel build -c opt --jobs={JOBS} "
f"--disk_cache=/root/bazelcache --noshow_progress "
f"//:raylet //:ray_core_worker_lib 2>&1 | tail -40",
5400,
"STEP5 bazel 构建 raylet + core worker lib",
)
build_secs = round(time.time() - t0)
result["steps"]["build_secs"] = build_secs
log(f"\n>>> 构建耗时 {build_secs}s({(build_secs/60):.1f} 分钟, jobs={JOBS})")
result["build_ok"] = "Build completed successfully" in build or "target //:raylet" in build
result["build_tail"] = build[-1500:]
# ---------- STEP 6: 量产物 ----------
art = sh(
"cd /ray && echo '--- 产物体积 ---'; "
"ls -lh bazel-bin/raylet bazel-bin/libray_core_worker_lib.so bazel-bin/gcs_server 2>/dev/null "
"| awk '{print $5, $9}'; "
"echo '--- bazel 缓存/输出目录 ---'; du -sh /root/.cache/bazel 2>/dev/null; "
"du -sh bazel-bin 2>/dev/null; "
"echo '--- 动态依赖条数 ---'; "
"ldd bazel-bin/raylet 2>/dev/null | wc -l; "
"ldd bazel-bin/libray_core_worker_lib.so 2>/dev/null | wc -l; "
"echo '--- 是否有 C ABI(extern C / 未修饰符号)---'; "
"nm -D --defined-only bazel-bin/libray_core_worker_lib.so 2>/dev/null | grep -c ' T ' ; "
"nm -D --defined-only bazel-bin/libray_core_worker_lib.so 2>/dev/null | grep ' T ' | grep -c ' _Z' ; "
"echo '--- 是否有 ray:: 相关的 C 入口(若有说明存在 C shim)---'; "
"nm -D --defined-only bazel-bin/libray_core_worker_lib.so 2>/dev/null | grep -iE ' T (ray_|ray_init|ray_shutdown)' | head -5; "
"echo end",
180,
"STEP6 产物与 ABI",
)
result["artifacts_raw"] = art
# ---------- STEP 7: C++ API 是否需要已在运行的集群 ----------
api = sh(
"cd /ray && echo '--- cpp/include/ray/api.h 的 Init 声明 ---'; "
"grep -nE 'void Init|bool IsInitialized|class Ray|RAY_API' cpp/include/ray/api.h 2>/dev/null | head -10; "
"echo '--- Init 实现里连什么 ---'; "
"grep -rn 'CoreWorkerProcess::Initialize\\|gcs_address\\|address' cpp/src/ray/runtime/abstract_ray_runtime.cc 2>/dev/null | head -8",
120,
"STEP7 C++ API 运行前提",
)
result["api_raw"] = api[:1200]
result["total_secs"] = round(time.time() - t_all)
log("\n===== SUMMARY =====")
summary = {
"tag": result["tag"],
"build_secs": result["steps"].get("build_secs"),
"build_ok": result.get("build_ok"),
"jobs": JOBS,
"armv7_hits": result["armv7_hits"][:5],
}
log(json.dumps(summary, ensure_ascii=False, indent=2))
open(OUT, "w").write(json.dumps(result, ensure_ascii=False, indent=2))
log("SAVED " + OUT)
log("SANDBOX_KEPT " + sid)
main()