Ray 这条线不绕 bazel: - wheel 里的 ray/_raylet.so 就是编好的 C++ core(34113 个导出符号, 3638 个 ray:: 符号 default 可见,含 CoreWorkerProcess::Initialize) - ray/nax_ray_shim.cc:259 行 C ABI shim,复用 process_helper 的初始化路径 - 实测跑通:Rust 二进制作为 driver 接入真实 GCS+raylet, 拿到 raylet 分配的 node_id(d190801c…),probe 通过 踩坑(都已固化进 ray/build.sh): - raylet 只编了 Python worker pool → 注册时语言声明 PYTHON - Ray C++ 是旧 ABI → -D_GLIBCXX_USE_CXX11_ABI=0 - fmt v9/v11 撞车 → FMT_HEADER_ONLY - Debian absl 固定自家 optional → 影子头直通 std::optional - _raylet.so 是 CPython 扩展(253 个 Py 符号)→ build.rs 自动链 libpython - GcsNodeInfo 手写 wire format 解析,避免 protobuf 跨 ABI armv7 交叉编译修正(原结论有误): - 换 armv7-unknown-linux-musleabihf + rust-lld/link-self-contained - 不再需要外部交叉 gcc,macOS 本机 83 秒出 687 KB 静态二进制 - .cargo/config.toml 固化,无需 export 任何变量 其他:nax-orch 告警清理;README/ZYNQ 文档对齐实测。
153 lines
6.3 KiB
Bash
153 lines
6.3 KiB
Bash
#!/bin/bash
|
||
# 把 Ray 的 C++ core(core_worker)编成 libnax_ray.so,供 Rust 链接。
|
||
#
|
||
# 不走 bazel:直接复用 pip wheel 里预编译的 `_raylet.so`(自包含、导出 ray:: 符号),
|
||
# 自己只需要编一个 ~250 行的 C ABI shim。背景与踩坑见 docs/RAY_CORE.md。
|
||
#
|
||
# 依赖(Debian 12 实测;apt 源换成 mirrors.ustc.edu.cn 后可用):
|
||
# g++ protobuf-compiler libprotobuf-dev libgrpc++-dev protobuf-compiler-grpc
|
||
# libabsl-dev libboost-dev libssl-dev libgtest-dev libunwind-dev
|
||
# libmsgpack-cxx-dev nlohmann-json3-dev libflatbuffers-dev flatbuffers-compiler
|
||
# libspdlog-dev libfmt-dev
|
||
#
|
||
# 用法:
|
||
# ./ray/build.sh # 全自动(需要能访问 pypi 与 github)
|
||
# RAY_CACHE=/path ./ray/build.sh # 复用已下好的 wheel / 头文件压缩包
|
||
set -o pipefail
|
||
set -u
|
||
|
||
RAY_VERSION="${RAY_VERSION:-2.59.0}"
|
||
PYVER="${PYVER:-3.12}"
|
||
OUT_DIR="${OUT_DIR:-$(cd "$(dirname "$0")/.." && pwd)/target/ray}"
|
||
CACHE="${RAY_CACHE:-/tmp}"
|
||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||
|
||
mkdir -p "$OUT_DIR" "$CACHE"
|
||
echo "== ray $RAY_VERSION / py$PYVER -> $OUT_DIR"
|
||
|
||
# --- 1. wheel:里面有 _raylet.so,以及 gcs_server / raylet 两个可执行文件 ------
|
||
WHEEL="$CACHE/ray-$RAY_VERSION-cp$PYVER-cp$PYVER-manylinux2014_x86_64.whl"
|
||
if [ ! -f "$WHEEL" ]; then
|
||
python3 -m pip download "ray==$RAY_VERSION" --no-deps --only-binary=:all: \
|
||
--platform manylinux2014_x86_64 --python-version "$PYVER" --implementation cp \
|
||
-d "$CACHE"
|
||
fi
|
||
RAYSO=""
|
||
if python3 -c 'import ray' 2>/dev/null; then
|
||
SP=$(python3 -c 'import ray,os;print(os.path.dirname(ray.__file__))')
|
||
RAYSO="$SP/_raylet.so"
|
||
fi
|
||
if [ ! -f "$RAYSO" ]; then
|
||
# 没装 ray 也能干:直接从 wheel 里抽
|
||
rm -rf "$CACHE/rawextract" && mkdir -p "$CACHE/rawextract"
|
||
python3 - "$WHEEL" "$CACHE/rawextract" <<'PY'
|
||
import sys, zipfile
|
||
zipfile.ZipFile(sys.argv[1]).extract("ray/_raylet.so", sys.argv[2])
|
||
PY
|
||
RAYSO="$CACHE/rawextract/ray/_raylet.so"
|
||
fi
|
||
echo " _raylet.so: $RAYSO"
|
||
|
||
# --- 2. 头文件:ray 源码里的 src/ray/**(只要 .h 和 .proto) -----------------
|
||
HDR_TGZ="$CACHE/raysel.tgz"
|
||
if [ ! -f "$HDR_TGZ" ]; then
|
||
SRC="$CACHE/ray-$RAY_VERSION.tar.gz"
|
||
[ -f "$SRC" ] || curl -sSL --http1.1 -m 900 \
|
||
"https://ghfast.top/https://github.com/ray-project/ray/archive/refs/tags/ray-$RAY_VERSION.tar.gz" \
|
||
-o "$SRC"
|
||
rm -rf "$CACHE/raysel" && mkdir -p "$CACHE/raysel"
|
||
tar xzf "$SRC" -C "$CACHE/raysel" --strip-components=1 "ray-ray-$RAY_VERSION/src/ray"
|
||
tar czf "$HDR_TGZ" -C "$CACHE" raysel
|
||
fi
|
||
|
||
# --- 3. opencensus 头文件(Debian 没有这个包) ------------------------------
|
||
OC_TGZ="$CACHE/oc.tgz"
|
||
if [ ! -f "$OC_TGZ" ]; then
|
||
curl -sSL --http1.1 -m 300 \
|
||
"https://ghfast.top/https://github.com/census-instrumentation/opencensus-cpp/archive/refs/tags/v0.3.0.tar.gz" \
|
||
-o "$OC_TGZ"
|
||
fi
|
||
|
||
rm -rf "$CACHE/rayhdr" "$CACHE/gen" "$CACHE/oc"
|
||
mkdir -p "$CACHE/rayhdr" "$CACHE/gen" "$CACHE/oc"
|
||
tar xzf "$HDR_TGZ" -C "$CACHE/rayhdr" --strip-components=1
|
||
tar xzf "$OC_TGZ" -C "$CACHE/oc" --strip-components=1
|
||
echo " 头文件: $(find "$CACHE/rayhdr" -name '*.h' | wc -l) 个"
|
||
|
||
# --- 4. 生成 protobuf / flatbuffers 头 -------------------------------------
|
||
cd "$CACHE/rayhdr"
|
||
# reporter.proto 依赖 bazel 里才有的 opencensus-proto,nax 用不到,跳过
|
||
PROTOS=$(find src/ray/protobuf -name '*.proto' | grep -v reporter.proto)
|
||
protoc -I. --cpp_out="$CACHE/gen" --grpc_out="$CACHE/gen" \
|
||
--plugin=protoc-gen-grpc="$(which grpc_cpp_plugin)" \
|
||
$PROTOS 2>&1 | grep -v unused | tail -3
|
||
echo " pb.h: $(ls "$CACHE"/gen/src/ray/protobuf/*.pb.h 2>/dev/null | wc -l) 个"
|
||
|
||
for f in $(find src -name '*.fbs'); do
|
||
flatc --cpp --scoped-enums -o "$(dirname "$f")" "$f" || exit 1
|
||
done
|
||
echo " flatbuffers: $(find src -name '*_generated.h' | wc -l) 个"
|
||
|
||
# --- 5. 影子头文件:压掉两条用不到的依赖链 ---------------------------------
|
||
# (a) opentelemetry-cpp:ray 的指标头依赖它,Debian 没有,nax 也不用指标
|
||
# (b) absl optional:Debian 的 absl 为 ABI 稳定固定成自家实现,
|
||
# ray 却按 absl::optional == std::optional 编译
|
||
mkdir -p "$CACHE/shiminc/ray/observability" "$CACHE/shiminc/absl/types"
|
||
cat > "$CACHE/shiminc/ray/observability/open_telemetry_metric_recorder.h" <<'EOF'
|
||
#pragma once
|
||
#include <string>
|
||
#include <vector>
|
||
namespace ray { namespace observability {
|
||
class OpenTelemetryMetricRecorder {
|
||
public:
|
||
static OpenTelemetryMetricRecorder &GetInstance() {
|
||
static OpenTelemetryMetricRecorder inst;
|
||
return inst;
|
||
}
|
||
template <typename... A> void Start(A &&...) {}
|
||
void Shutdown() {}
|
||
void RegisterGaugeMetric(const std::string &, const std::string &) {}
|
||
void RegisterCounterMetric(const std::string &, const std::string &) {}
|
||
void RegisterSumMetric(const std::string &, const std::string &) {}
|
||
template <typename... A> void RegisterHistogramMetric(A &&...) {}
|
||
template <typename... A> void SetMetricValue(A &&...) {}
|
||
bool IsMetricRegistered(const std::string &) { return false; }
|
||
};
|
||
}}
|
||
EOF
|
||
cat > "$CACHE/shiminc/absl/types/optional.h" <<'EOF'
|
||
#pragma once
|
||
#include <optional>
|
||
namespace absl {
|
||
using std::bad_optional_access;
|
||
using std::in_place;
|
||
using std::in_place_t;
|
||
using std::make_optional;
|
||
using std::nullopt;
|
||
using std::nullopt_t;
|
||
using std::optional;
|
||
}
|
||
EOF
|
||
|
||
# --- 6. 编 shim -------------------------------------------------------------
|
||
# 关键宏,少一个都会在链接/加载期炸:
|
||
# _GLIBCXX_USE_CXX11_ABI=0 ray 的 C++ 是旧 ABI(std::string 不带 cxx11)
|
||
# FMT_HEADER_ONLY ray 自带 fmt(v11),和系统 fmt(v9) 会撞
|
||
# ABSL_* absl::optional 走 std::optional
|
||
g++ -std=c++17 -O2 -fPIC -shared \
|
||
-D_GLIBCXX_USE_CXX11_ABI=0 -DFMT_HEADER_ONLY=1 \
|
||
-DABSL_USES_STD_OPTIONAL=1 -DABSL_OPTION_USE_STD_OPTIONAL=1 \
|
||
-o "$OUT_DIR/libnax_ray.so" "$HERE/nax_ray_shim.cc" \
|
||
-I"$CACHE/shiminc" -I"$CACHE/rayhdr" -I"$CACHE/rayhdr/src" -I"$CACHE/gen" -I"$CACHE/oc" \
|
||
"$RAYSO" \
|
||
-labsl_synchronization -labsl_time -labsl_base -labsl_spinlock_wait -labsl_raw_logging_internal \
|
||
2>&1 | grep -E "error:" | head -20
|
||
|
||
if [ -f "$OUT_DIR/libnax_ray.so" ]; then
|
||
echo "== 成功: $OUT_DIR/libnax_ray.so ($(du -h "$OUT_DIR/libnax_ray.so" | cut -f1))"
|
||
echo " 自检: NAX_RAY_LIB_DIR=$OUT_DIR \\"
|
||
echo " cargo run -p nax-ray --features probe --bin rayprobe -- <gcs_ip:port>"
|
||
else
|
||
echo "== 失败"; exit 1
|
||
fi
|