Files
ssw-mpc/ray/build.sh
T
rong 4bb573d98c feat: Ray C++ 内核接进 nax Rust + armv7 真交叉编译
Ray 这条线不绕 bazel:
- wheel 里的 ray/_raylet.so 就是编好的 C++ core(34113 个导出符号,
  3638 个 ray:: 符号 default 可见,含 CoreWorkerProcess::Initialize)
- ray/nax_ray_shim.cc:259 行 C ABI shim,复用 process_helper 的初始化路径
- 实测跑通:Rust 二进制作为 driver 接入真实 GCS+raylet,
  拿到 raylet 分配的 node_id(d190801c…),probe 通过

踩坑(都已固化进 ray/build.sh):
- raylet 只编了 Python worker pool → 注册时语言声明 PYTHON
- Ray C++ 是旧 ABI → -D_GLIBCXX_USE_CXX11_ABI=0
- fmt v9/v11 撞车 → FMT_HEADER_ONLY
- Debian absl 固定自家 optional → 影子头直通 std::optional
- _raylet.so 是 CPython 扩展(253 个 Py 符号)→ build.rs 自动链 libpython
- GcsNodeInfo 手写 wire format 解析,避免 protobuf 跨 ABI

armv7 交叉编译修正(原结论有误):
- 换 armv7-unknown-linux-musleabihf + rust-lld/link-self-contained
- 不再需要外部交叉 gcc,macOS 本机 83 秒出 687 KB 静态二进制
- .cargo/config.toml 固化,无需 export 任何变量

其他:nax-orch 告警清理;README/ZYNQ 文档对齐实测。
2026-10-30 11:31:02 +08:00

153 lines
6.3 KiB
Bash
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/bash
# 把 Ray 的 C++ core(core_worker)编成 libnax_ray.so,供 Rust 链接。
#
# 不走 bazel:直接复用 pip wheel 里预编译的 `_raylet.so`(自包含、导出 ray:: 符号),
# 自己只需要编一个 ~250 行的 C ABI shim。背景与踩坑见 docs/RAY_CORE.md。
#
# 依赖(Debian 12 实测;apt 源换成 mirrors.ustc.edu.cn 后可用):
# g++ protobuf-compiler libprotobuf-dev libgrpc++-dev protobuf-compiler-grpc
# libabsl-dev libboost-dev libssl-dev libgtest-dev libunwind-dev
# libmsgpack-cxx-dev nlohmann-json3-dev libflatbuffers-dev flatbuffers-compiler
# libspdlog-dev libfmt-dev
#
# 用法:
# ./ray/build.sh # 全自动(需要能访问 pypi 与 github)
# RAY_CACHE=/path ./ray/build.sh # 复用已下好的 wheel / 头文件压缩包
set -o pipefail
set -u
RAY_VERSION="${RAY_VERSION:-2.59.0}"
PYVER="${PYVER:-3.12}"
OUT_DIR="${OUT_DIR:-$(cd "$(dirname "$0")/.." && pwd)/target/ray}"
CACHE="${RAY_CACHE:-/tmp}"
HERE="$(cd "$(dirname "$0")" && pwd)"
mkdir -p "$OUT_DIR" "$CACHE"
echo "== ray $RAY_VERSION / py$PYVER -> $OUT_DIR"
# --- 1. wheel:里面有 _raylet.so,以及 gcs_server / raylet 两个可执行文件 ------
WHEEL="$CACHE/ray-$RAY_VERSION-cp$PYVER-cp$PYVER-manylinux2014_x86_64.whl"
if [ ! -f "$WHEEL" ]; then
python3 -m pip download "ray==$RAY_VERSION" --no-deps --only-binary=:all: \
--platform manylinux2014_x86_64 --python-version "$PYVER" --implementation cp \
-d "$CACHE"
fi
RAYSO=""
if python3 -c 'import ray' 2>/dev/null; then
SP=$(python3 -c 'import ray,os;print(os.path.dirname(ray.__file__))')
RAYSO="$SP/_raylet.so"
fi
if [ ! -f "$RAYSO" ]; then
# 没装 ray 也能干:直接从 wheel 里抽
rm -rf "$CACHE/rawextract" && mkdir -p "$CACHE/rawextract"
python3 - "$WHEEL" "$CACHE/rawextract" <<'PY'
import sys, zipfile
zipfile.ZipFile(sys.argv[1]).extract("ray/_raylet.so", sys.argv[2])
PY
RAYSO="$CACHE/rawextract/ray/_raylet.so"
fi
echo " _raylet.so: $RAYSO"
# --- 2. 头文件:ray 源码里的 src/ray/**(只要 .h 和 .proto) -----------------
HDR_TGZ="$CACHE/raysel.tgz"
if [ ! -f "$HDR_TGZ" ]; then
SRC="$CACHE/ray-$RAY_VERSION.tar.gz"
[ -f "$SRC" ] || curl -sSL --http1.1 -m 900 \
"https://ghfast.top/https://github.com/ray-project/ray/archive/refs/tags/ray-$RAY_VERSION.tar.gz" \
-o "$SRC"
rm -rf "$CACHE/raysel" && mkdir -p "$CACHE/raysel"
tar xzf "$SRC" -C "$CACHE/raysel" --strip-components=1 "ray-ray-$RAY_VERSION/src/ray"
tar czf "$HDR_TGZ" -C "$CACHE" raysel
fi
# --- 3. opencensus 头文件(Debian 没有这个包) ------------------------------
OC_TGZ="$CACHE/oc.tgz"
if [ ! -f "$OC_TGZ" ]; then
curl -sSL --http1.1 -m 300 \
"https://ghfast.top/https://github.com/census-instrumentation/opencensus-cpp/archive/refs/tags/v0.3.0.tar.gz" \
-o "$OC_TGZ"
fi
rm -rf "$CACHE/rayhdr" "$CACHE/gen" "$CACHE/oc"
mkdir -p "$CACHE/rayhdr" "$CACHE/gen" "$CACHE/oc"
tar xzf "$HDR_TGZ" -C "$CACHE/rayhdr" --strip-components=1
tar xzf "$OC_TGZ" -C "$CACHE/oc" --strip-components=1
echo " 头文件: $(find "$CACHE/rayhdr" -name '*.h' | wc -l) 个"
# --- 4. 生成 protobuf / flatbuffers 头 -------------------------------------
cd "$CACHE/rayhdr"
# reporter.proto 依赖 bazel 里才有的 opencensus-proto,nax 用不到,跳过
PROTOS=$(find src/ray/protobuf -name '*.proto' | grep -v reporter.proto)
protoc -I. --cpp_out="$CACHE/gen" --grpc_out="$CACHE/gen" \
--plugin=protoc-gen-grpc="$(which grpc_cpp_plugin)" \
$PROTOS 2>&1 | grep -v unused | tail -3
echo " pb.h: $(ls "$CACHE"/gen/src/ray/protobuf/*.pb.h 2>/dev/null | wc -l) 个"
for f in $(find src -name '*.fbs'); do
flatc --cpp --scoped-enums -o "$(dirname "$f")" "$f" || exit 1
done
echo " flatbuffers: $(find src -name '*_generated.h' | wc -l) 个"
# --- 5. 影子头文件:压掉两条用不到的依赖链 ---------------------------------
# (a) opentelemetry-cpp:ray 的指标头依赖它,Debian 没有,nax 也不用指标
# (b) absl optional:Debian 的 absl 为 ABI 稳定固定成自家实现,
# ray 却按 absl::optional == std::optional 编译
mkdir -p "$CACHE/shiminc/ray/observability" "$CACHE/shiminc/absl/types"
cat > "$CACHE/shiminc/ray/observability/open_telemetry_metric_recorder.h" <<'EOF'
#pragma once
#include <string>
#include <vector>
namespace ray { namespace observability {
class OpenTelemetryMetricRecorder {
public:
static OpenTelemetryMetricRecorder &GetInstance() {
static OpenTelemetryMetricRecorder inst;
return inst;
}
template <typename... A> void Start(A &&...) {}
void Shutdown() {}
void RegisterGaugeMetric(const std::string &, const std::string &) {}
void RegisterCounterMetric(const std::string &, const std::string &) {}
void RegisterSumMetric(const std::string &, const std::string &) {}
template <typename... A> void RegisterHistogramMetric(A &&...) {}
template <typename... A> void SetMetricValue(A &&...) {}
bool IsMetricRegistered(const std::string &) { return false; }
};
}}
EOF
cat > "$CACHE/shiminc/absl/types/optional.h" <<'EOF'
#pragma once
#include <optional>
namespace absl {
using std::bad_optional_access;
using std::in_place;
using std::in_place_t;
using std::make_optional;
using std::nullopt;
using std::nullopt_t;
using std::optional;
}
EOF
# --- 6. 编 shim -------------------------------------------------------------
# 关键宏,少一个都会在链接/加载期炸:
# _GLIBCXX_USE_CXX11_ABI=0 ray 的 C++ 是旧 ABI(std::string 不带 cxx11)
# FMT_HEADER_ONLY ray 自带 fmt(v11),和系统 fmt(v9) 会撞
# ABSL_* absl::optional 走 std::optional
g++ -std=c++17 -O2 -fPIC -shared \
-D_GLIBCXX_USE_CXX11_ABI=0 -DFMT_HEADER_ONLY=1 \
-DABSL_USES_STD_OPTIONAL=1 -DABSL_OPTION_USE_STD_OPTIONAL=1 \
-o "$OUT_DIR/libnax_ray.so" "$HERE/nax_ray_shim.cc" \
-I"$CACHE/shiminc" -I"$CACHE/rayhdr" -I"$CACHE/rayhdr/src" -I"$CACHE/gen" -I"$CACHE/oc" \
"$RAYSO" \
-labsl_synchronization -labsl_time -labsl_base -labsl_spinlock_wait -labsl_raw_logging_internal \
2>&1 | grep -E "error:" | head -20
if [ -f "$OUT_DIR/libnax_ray.so" ]; then
echo "== 成功: $OUT_DIR/libnax_ray.so ($(du -h "$OUT_DIR/libnax_ray.so" | cut -f1))"
echo " 自检: NAX_RAY_LIB_DIR=$OUT_DIR \\"
echo " cargo run -p nax-ray --features probe --bin rayprobe -- <gcs_ip:port>"
else
echo "== 失败"; exit 1
fi