Ray 这条线不绕 bazel: - wheel 里的 ray/_raylet.so 就是编好的 C++ core(34113 个导出符号, 3638 个 ray:: 符号 default 可见,含 CoreWorkerProcess::Initialize) - ray/nax_ray_shim.cc:259 行 C ABI shim,复用 process_helper 的初始化路径 - 实测跑通:Rust 二进制作为 driver 接入真实 GCS+raylet, 拿到 raylet 分配的 node_id(d190801c…),probe 通过 踩坑(都已固化进 ray/build.sh): - raylet 只编了 Python worker pool → 注册时语言声明 PYTHON - Ray C++ 是旧 ABI → -D_GLIBCXX_USE_CXX11_ABI=0 - fmt v9/v11 撞车 → FMT_HEADER_ONLY - Debian absl 固定自家 optional → 影子头直通 std::optional - _raylet.so 是 CPython 扩展(253 个 Py 符号)→ build.rs 自动链 libpython - GcsNodeInfo 手写 wire format 解析,避免 protobuf 跨 ABI armv7 交叉编译修正(原结论有误): - 换 armv7-unknown-linux-musleabihf + rust-lld/link-self-contained - 不再需要外部交叉 gcc,macOS 本机 83 秒出 687 KB 静态二进制 - .cargo/config.toml 固化,无需 export 任何变量 其他:nax-orch 告警清理;README/ZYNQ 文档对齐实测。
219 lines
8.3 KiB
Python
219 lines
8.3 KiB
Python
#!/usr/bin/env python3
|
||
"""量清 Ray C++ core 的真实代价。
|
||
|
||
回答四个问题,不猜:
|
||
1. 它到底要哪些第三方依赖(个数 / 版本)
|
||
2. bazel 从零构建要多久、吃多少内存和磁盘
|
||
3. 产物多大、动态依赖多少、有没有 C ABI 可供 Rust 直接调
|
||
4. 有没有 32 位 ARM(armv7)的构建目标 —— 决定 Zynq-7015 能不能用
|
||
|
||
必须在单进程内跑完:沙箱离开创建它的进程就会被回收。
|
||
"""
|
||
|
||
import sys
|
||
import os
|
||
import time
|
||
import json
|
||
|
||
sys.path.insert(0, "/tmp")
|
||
import sbfix # noqa: F401 (cube-auth-gate gzip bug 绕过)
|
||
from cubesandbox import Sandbox
|
||
|
||
JOBS = os.environ.get("RAY_BAZEL_JOBS", "8")
|
||
OUT = "/tmp/ray_core_cost.json"
|
||
|
||
|
||
def log(msg=""):
|
||
print(msg, flush=True)
|
||
|
||
|
||
def main():
|
||
t_all = time.time()
|
||
sb = Sandbox.create(
|
||
template="toolchain",
|
||
timeout=7200,
|
||
network={"allow_public_traffic": True},
|
||
)
|
||
sid = sb.sandbox_id
|
||
tok = sb._data.get("trafficAccessToken") or ""
|
||
log(f"SANDBOX {sid} token_len={len(tok)}")
|
||
open("/tmp/ray_cost_sid.txt", "w").write(sid)
|
||
open(f"/tmp/sb_{sid}.token", "w").write(tok)
|
||
|
||
result = {"sandbox": sid, "steps": {}}
|
||
|
||
def sh(cmd, t=120, label="", quiet=False):
|
||
"""跑一条命令,返回 stdout。所有调用都必须显式给 timeout。"""
|
||
if label:
|
||
log(f"\n===== {label} =====")
|
||
try:
|
||
r = sb.commands.run(cmd, timeout=t)
|
||
except Exception as e: # 超时不等于没执行,但这里如实记录
|
||
log(f"[EXC] {type(e).__name__}: {e}")
|
||
return ""
|
||
if not quiet:
|
||
out = (r.stdout or "").strip()
|
||
log(out if out else "(无输出)")
|
||
err = (r.stderr or "").strip()
|
||
if err:
|
||
log("STDERR: " + err[:600])
|
||
return r.stdout or ""
|
||
|
||
# ---------- STEP 0: 机器规格 ----------
|
||
sh(
|
||
"echo cpus=$(nproc); "
|
||
"free -g | awk 'NR==2{print \"mem_gb=\"$2}'; "
|
||
"df -h / /root | tail -2; "
|
||
"which gcc clang bazel git curl python3 2>/dev/null; "
|
||
"cat /etc/os-release | grep PRETTY",
|
||
60,
|
||
"STEP0 机器规格",
|
||
)
|
||
|
||
# ---------- STEP 1: 构建依赖(Ray 官方要求 clang + psmisc + unzip)----------
|
||
sh(
|
||
"sed -i 's|http://deb.debian.org|https://mirrors.ustc.edu.cn|g' "
|
||
"/etc/apt/sources.list.d/debian.sources 2>/dev/null; "
|
||
"apt-get update -qq 2>&1 | tail -2; "
|
||
"DEBIAN_FRONTEND=noninteractive apt-get install -y -qq "
|
||
"clang lld unzip zip pkg-config psmisc patch file 2>&1 | tail -3; "
|
||
"clang --version | head -1",
|
||
900,
|
||
"STEP1 构建依赖",
|
||
)
|
||
|
||
# ---------- STEP 2: 取 Ray 源码 ----------
|
||
tag = sh(
|
||
"git ls-remote --tags https://ghfast.top/https://github.com/ray-project/ray.git "
|
||
"2>/dev/null | sed 's#.*refs/tags/##' | grep -E '^ray-[0-9]+\\.[0-9]+\\.[0-9]+$' "
|
||
"| sort -V | tail -1",
|
||
180,
|
||
"STEP2 查最新 release tag",
|
||
).strip()
|
||
if not tag:
|
||
tag = "ray-2.9.3"
|
||
log(f"选定 tag: {tag}")
|
||
result["tag"] = tag
|
||
|
||
t0 = time.time()
|
||
sh(
|
||
f"rm -rf /ray && mkdir -p /ray && "
|
||
f"curl -sSL --http1.1 --max-time 900 "
|
||
f"https://ghfast.top/https://github.com/ray-project/ray/archive/refs/tags/{tag}.tar.gz "
|
||
f"-o /tmp/ray.tgz && ls -lh /tmp/ray.tgz && "
|
||
f"tar xzf /tmp/ray.tgz -C /ray --strip-components=1 && "
|
||
f"du -sh /ray && cat /ray/.bazelversion 2>/dev/null",
|
||
1200,
|
||
"STEP2 下载并解压源码",
|
||
)
|
||
result["steps"]["fetch_secs"] = round(time.time() - t0)
|
||
|
||
# ---------- STEP 3: 静态分析(不构建也能定案的部分)----------
|
||
arm = sh(
|
||
"cd /ray && echo '--- bazel 平台定义 ---'; ls bazel/ 2>/dev/null; "
|
||
"echo '--- 平台 target ---'; grep -rn 'name = \"linux_' bazel/BUILD* 2>/dev/null | head -20; "
|
||
"echo '--- 任何 armv7/armhf/32位 痕迹 ---'; "
|
||
"grep -rniE 'armv7|armhf|arm_32|arm32|aarch32' --include='*.bzl' --include='BUILD*' "
|
||
"--include='.bazelrc' . 2>/dev/null | grep -v thirdparty/ | head -10; "
|
||
"echo 'end'",
|
||
120,
|
||
"STEP3 armv7 支持度",
|
||
)
|
||
result["armv7_hits"] = [l for l in arm.splitlines() if "armv7" in l.lower() or "armhf" in l.lower()]
|
||
|
||
deps = sh(
|
||
"cd /ray && echo '--- http_archive 依赖个数 ---'; "
|
||
"grep -c ' http_archive(' bazel/ray_deps_setup.bzl 2>/dev/null; "
|
||
"echo '--- 依赖名 ---'; "
|
||
"grep -oE 'name = \"[a-z0-9_]+' bazel/ray_deps_setup.bzl 2>/dev/null | sed 's/.*\"//' | tr '\\n' ' '; "
|
||
"echo; echo '--- 源码规模 ---'; "
|
||
"find . -name '*.cc' -o -name '*.cpp' | wc -l; "
|
||
"du -sh src/ cpp/ 2>/dev/null",
|
||
120,
|
||
"STEP3 第三方依赖清单",
|
||
)
|
||
result["deps_raw"] = deps[:2000]
|
||
|
||
# ---------- STEP 4: 装 bazel ----------
|
||
baz = sh(
|
||
"curl -sSL --http1.1 --max-time 300 -o /usr/local/bin/bazel "
|
||
"https://ghfast.top/https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-linux-amd64 "
|
||
"&& chmod +x /usr/local/bin/bazel && bazel version 2>&1 | head -3",
|
||
600,
|
||
"STEP4 安装 bazelisk",
|
||
)
|
||
result["bazel"] = baz.strip().splitlines()[:2]
|
||
|
||
# ---------- STEP 5: 构建(真金白银)----------
|
||
targets = sh(
|
||
"cd /ray && grep -nE 'name = \"(raylet|gcs_server|ray_core_worker_lib|libray_agent)\"' BUILD.bazel | head",
|
||
60,
|
||
"STEP5 确认构建目标",
|
||
)
|
||
log(targets)
|
||
|
||
t0 = time.time()
|
||
build = sh(
|
||
f"cd /ray && sync && "
|
||
f"/usr/bin/time -v bazel build -c opt --jobs={JOBS} "
|
||
f"--disk_cache=/root/bazelcache --noshow_progress "
|
||
f"//:raylet //:ray_core_worker_lib 2>&1 | tail -40",
|
||
5400,
|
||
"STEP5 bazel 构建 raylet + core worker lib",
|
||
)
|
||
build_secs = round(time.time() - t0)
|
||
result["steps"]["build_secs"] = build_secs
|
||
log(f"\n>>> 构建耗时 {build_secs}s({(build_secs/60):.1f} 分钟, jobs={JOBS})")
|
||
result["build_ok"] = "Build completed successfully" in build or "target //:raylet" in build
|
||
result["build_tail"] = build[-1500:]
|
||
|
||
# ---------- STEP 6: 量产物 ----------
|
||
art = sh(
|
||
"cd /ray && echo '--- 产物体积 ---'; "
|
||
"ls -lh bazel-bin/raylet bazel-bin/libray_core_worker_lib.so bazel-bin/gcs_server 2>/dev/null "
|
||
"| awk '{print $5, $9}'; "
|
||
"echo '--- bazel 缓存/输出目录 ---'; du -sh /root/.cache/bazel 2>/dev/null; "
|
||
"du -sh bazel-bin 2>/dev/null; "
|
||
"echo '--- 动态依赖条数 ---'; "
|
||
"ldd bazel-bin/raylet 2>/dev/null | wc -l; "
|
||
"ldd bazel-bin/libray_core_worker_lib.so 2>/dev/null | wc -l; "
|
||
"echo '--- 是否有 C ABI(extern C / 未修饰符号)---'; "
|
||
"nm -D --defined-only bazel-bin/libray_core_worker_lib.so 2>/dev/null | grep -c ' T ' ; "
|
||
"nm -D --defined-only bazel-bin/libray_core_worker_lib.so 2>/dev/null | grep ' T ' | grep -c ' _Z' ; "
|
||
"echo '--- 是否有 ray:: 相关的 C 入口(若有说明存在 C shim)---'; "
|
||
"nm -D --defined-only bazel-bin/libray_core_worker_lib.so 2>/dev/null | grep -iE ' T (ray_|ray_init|ray_shutdown)' | head -5; "
|
||
"echo end",
|
||
180,
|
||
"STEP6 产物与 ABI",
|
||
)
|
||
result["artifacts_raw"] = art
|
||
|
||
# ---------- STEP 7: C++ API 是否需要已在运行的集群 ----------
|
||
api = sh(
|
||
"cd /ray && echo '--- cpp/include/ray/api.h 的 Init 声明 ---'; "
|
||
"grep -nE 'void Init|bool IsInitialized|class Ray|RAY_API' cpp/include/ray/api.h 2>/dev/null | head -10; "
|
||
"echo '--- Init 实现里连什么 ---'; "
|
||
"grep -rn 'CoreWorkerProcess::Initialize\\|gcs_address\\|address' cpp/src/ray/runtime/abstract_ray_runtime.cc 2>/dev/null | head -8",
|
||
120,
|
||
"STEP7 C++ API 运行前提",
|
||
)
|
||
result["api_raw"] = api[:1200]
|
||
|
||
result["total_secs"] = round(time.time() - t_all)
|
||
|
||
log("\n===== SUMMARY =====")
|
||
summary = {
|
||
"tag": result["tag"],
|
||
"build_secs": result["steps"].get("build_secs"),
|
||
"build_ok": result.get("build_ok"),
|
||
"jobs": JOBS,
|
||
"armv7_hits": result["armv7_hits"][:5],
|
||
}
|
||
log(json.dumps(summary, ensure_ascii=False, indent=2))
|
||
open(OUT, "w").write(json.dumps(result, ensure_ascii=False, indent=2))
|
||
log("SAVED " + OUT)
|
||
log("SANDBOX_KEPT " + sid)
|
||
|
||
|
||
main()
|