- ray/build.sh:下载缓存(wheel/源码 tgz/头文件)默认 /tmp 改为 ~/.cache/ssw-mpc/ray,可用 RAY_CACHE 覆盖;短时沙箱里应指向 host-mount 共享目录。/tmp 重启即丢,每次重下几百 MB - .gitignore 补 /artifacts/:build/build_optee.sh 的默认产物落点,产物仍留项目内但不再冒未跟踪文件 - CI 三条闸门全绿;sccache 实测 14 hits / 0 miss / 100%,Non-cacheable compilations 0
157 lines
6.7 KiB
Bash
157 lines
6.7 KiB
Bash
#!/bin/bash
|
||
# 把 Ray 的 C++ core(core_worker)编成 libmpc_ray.so,供 Rust 链接。
|
||
#
|
||
# 不走 bazel:直接复用 pip wheel 里预编译的 `_raylet.so`(自包含、导出 ray:: 符号),
|
||
# 自己只需要编一个 ~250 行的 C ABI shim。背景与踩坑见 docs/RAY_CORE.md。
|
||
#
|
||
# 依赖(Debian 12 实测;apt 源换成 mirrors.ustc.edu.cn 后可用):
|
||
# g++ protobuf-compiler libprotobuf-dev libgrpc++-dev protobuf-compiler-grpc
|
||
# libabsl-dev libboost-dev libssl-dev libgtest-dev libunwind-dev
|
||
# libmsgpack-cxx-dev nlohmann-json3-dev libflatbuffers-dev flatbuffers-compiler
|
||
# libspdlog-dev libfmt-dev
|
||
#
|
||
# 用法:
|
||
# ./ray/build.sh # 全自动(需要能访问 pypi 与 github)
|
||
# RAY_CACHE=/path ./ray/build.sh # 复用已下好的 wheel / 头文件压缩包
|
||
set -o pipefail
|
||
set -u
|
||
|
||
RAY_VERSION="${RAY_VERSION:-2.59.0}"
|
||
PYVER="${PYVER:-3.12}"
|
||
OUT_DIR="${OUT_DIR:-$(cd "$(dirname "$0")/.." && pwd)/target/ray}"
|
||
# 下载缓存(wheel / 源码 tgz / 解压出的头文件)是**包缓存**,按 dev-cache-layout
|
||
# 铁律 #2 统一放 ~/.cache/<tool>/,不要放 /tmp —— /tmp 重启即丢,每次重下几百 MB。
|
||
# 在 CubeSandbox 等短时沙箱里跑时用 RAY_CACHE 指到 host-mount 的共享缓存目录
|
||
# (宿主机 /data/shared/cache/ 下,别写 /data/shared/ 外面)。
|
||
CACHE="${RAY_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/ssw-mpc/ray}"
|
||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||
|
||
mkdir -p "$OUT_DIR" "$CACHE"
|
||
echo "== ray $RAY_VERSION / py$PYVER -> $OUT_DIR"
|
||
|
||
# --- 1. wheel:里面有 _raylet.so,以及 gcs_server / raylet 两个可执行文件 ------
|
||
WHEEL="$CACHE/ray-$RAY_VERSION-cp$PYVER-cp$PYVER-manylinux2014_x86_64.whl"
|
||
if [ ! -f "$WHEEL" ]; then
|
||
python3 -m pip download "ray==$RAY_VERSION" --no-deps --only-binary=:all: \
|
||
--platform manylinux2014_x86_64 --python-version "$PYVER" --implementation cp \
|
||
-d "$CACHE"
|
||
fi
|
||
RAYSO=""
|
||
if python3 -c 'import ray' 2>/dev/null; then
|
||
SP=$(python3 -c 'import ray,os;print(os.path.dirname(ray.__file__))')
|
||
RAYSO="$SP/_raylet.so"
|
||
fi
|
||
if [ ! -f "$RAYSO" ]; then
|
||
# 没装 ray 也能干:直接从 wheel 里抽
|
||
rm -rf "$CACHE/rawextract" && mkdir -p "$CACHE/rawextract"
|
||
python3 - "$WHEEL" "$CACHE/rawextract" <<'PY'
|
||
import sys, zipfile
|
||
zipfile.ZipFile(sys.argv[1]).extract("ray/_raylet.so", sys.argv[2])
|
||
PY
|
||
RAYSO="$CACHE/rawextract/ray/_raylet.so"
|
||
fi
|
||
echo " _raylet.so: $RAYSO"
|
||
|
||
# --- 2. 头文件:ray 源码里的 src/ray/**(只要 .h 和 .proto) -----------------
|
||
HDR_TGZ="$CACHE/raysel.tgz"
|
||
if [ ! -f "$HDR_TGZ" ]; then
|
||
SRC="$CACHE/ray-$RAY_VERSION.tar.gz"
|
||
[ -f "$SRC" ] || curl -sSL --http1.1 -m 900 \
|
||
"https://ghfast.top/https://github.com/ray-project/ray/archive/refs/tags/ray-$RAY_VERSION.tar.gz" \
|
||
-o "$SRC"
|
||
rm -rf "$CACHE/raysel" && mkdir -p "$CACHE/raysel"
|
||
tar xzf "$SRC" -C "$CACHE/raysel" --strip-components=1 "ray-ray-$RAY_VERSION/src/ray"
|
||
tar czf "$HDR_TGZ" -C "$CACHE" raysel
|
||
fi
|
||
|
||
# --- 3. opencensus 头文件(Debian 没有这个包) ------------------------------
|
||
OC_TGZ="$CACHE/oc.tgz"
|
||
if [ ! -f "$OC_TGZ" ]; then
|
||
curl -sSL --http1.1 -m 300 \
|
||
"https://ghfast.top/https://github.com/census-instrumentation/opencensus-cpp/archive/refs/tags/v0.3.0.tar.gz" \
|
||
-o "$OC_TGZ"
|
||
fi
|
||
|
||
rm -rf "$CACHE/rayhdr" "$CACHE/gen" "$CACHE/oc"
|
||
mkdir -p "$CACHE/rayhdr" "$CACHE/gen" "$CACHE/oc"
|
||
tar xzf "$HDR_TGZ" -C "$CACHE/rayhdr" --strip-components=1
|
||
tar xzf "$OC_TGZ" -C "$CACHE/oc" --strip-components=1
|
||
echo " 头文件: $(find "$CACHE/rayhdr" -name '*.h' | wc -l) 个"
|
||
|
||
# --- 4. 生成 protobuf / flatbuffers 头 -------------------------------------
|
||
cd "$CACHE/rayhdr"
|
||
# reporter.proto 依赖 bazel 里才有的 opencensus-proto,mpc 用不到,跳过
|
||
PROTOS=$(find src/ray/protobuf -name '*.proto' | grep -v reporter.proto)
|
||
protoc -I. --cpp_out="$CACHE/gen" --grpc_out="$CACHE/gen" \
|
||
--plugin=protoc-gen-grpc="$(which grpc_cpp_plugin)" \
|
||
$PROTOS 2>&1 | grep -v unused | tail -3
|
||
echo " pb.h: $(ls "$CACHE"/gen/src/ray/protobuf/*.pb.h 2>/dev/null | wc -l) 个"
|
||
|
||
for f in $(find src -name '*.fbs'); do
|
||
flatc --cpp --scoped-enums -o "$(dirname "$f")" "$f" || exit 1
|
||
done
|
||
echo " flatbuffers: $(find src -name '*_generated.h' | wc -l) 个"
|
||
|
||
# --- 5. 影子头文件:压掉两条用不到的依赖链 ---------------------------------
|
||
# (a) opentelemetry-cpp:ray 的指标头依赖它,Debian 没有,mpc 也不用指标
|
||
# (b) absl optional:Debian 的 absl 为 ABI 稳定固定成自家实现,
|
||
# ray 却按 absl::optional == std::optional 编译
|
||
mkdir -p "$CACHE/shiminc/ray/observability" "$CACHE/shiminc/absl/types"
|
||
cat > "$CACHE/shiminc/ray/observability/open_telemetry_metric_recorder.h" <<'EOF'
|
||
#pragma once
|
||
#include <string>
|
||
#include <vector>
|
||
namespace ray { namespace observability {
|
||
class OpenTelemetryMetricRecorder {
|
||
public:
|
||
static OpenTelemetryMetricRecorder &GetInstance() {
|
||
static OpenTelemetryMetricRecorder inst;
|
||
return inst;
|
||
}
|
||
template <typename... A> void Start(A &&...) {}
|
||
void Shutdown() {}
|
||
void RegisterGaugeMetric(const std::string &, const std::string &) {}
|
||
void RegisterCounterMetric(const std::string &, const std::string &) {}
|
||
void RegisterSumMetric(const std::string &, const std::string &) {}
|
||
template <typename... A> void RegisterHistogramMetric(A &&...) {}
|
||
template <typename... A> void SetMetricValue(A &&...) {}
|
||
bool IsMetricRegistered(const std::string &) { return false; }
|
||
};
|
||
}}
|
||
EOF
|
||
cat > "$CACHE/shiminc/absl/types/optional.h" <<'EOF'
|
||
#pragma once
|
||
#include <optional>
|
||
namespace absl {
|
||
using std::bad_optional_access;
|
||
using std::in_place;
|
||
using std::in_place_t;
|
||
using std::make_optional;
|
||
using std::nullopt;
|
||
using std::nullopt_t;
|
||
using std::optional;
|
||
}
|
||
EOF
|
||
|
||
# --- 6. 编 shim -------------------------------------------------------------
|
||
# 关键宏,少一个都会在链接/加载期炸:
|
||
# _GLIBCXX_USE_CXX11_ABI=0 ray 的 C++ 是旧 ABI(std::string 不带 cxx11)
|
||
# FMT_HEADER_ONLY ray 自带 fmt(v11),和系统 fmt(v9) 会撞
|
||
# ABSL_* absl::optional 走 std::optional
|
||
g++ -std=c++17 -O2 -fPIC -shared \
|
||
-D_GLIBCXX_USE_CXX11_ABI=0 -DFMT_HEADER_ONLY=1 \
|
||
-DABSL_USES_STD_OPTIONAL=1 -DABSL_OPTION_USE_STD_OPTIONAL=1 \
|
||
-o "$OUT_DIR/libmpc_ray.so" "$HERE/mpc_ray_shim.cc" \
|
||
-I"$CACHE/shiminc" -I"$CACHE/rayhdr" -I"$CACHE/rayhdr/src" -I"$CACHE/gen" -I"$CACHE/oc" \
|
||
"$RAYSO" \
|
||
-labsl_synchronization -labsl_time -labsl_base -labsl_spinlock_wait -labsl_raw_logging_internal \
|
||
2>&1 | grep -E "error:" | head -20
|
||
|
||
if [ -f "$OUT_DIR/libmpc_ray.so" ]; then
|
||
echo "== 成功: $OUT_DIR/libmpc_ray.so ($(du -h "$OUT_DIR/libmpc_ray.so" | cut -f1))"
|
||
echo " 自检: MPC_RAY_LIB_DIR=$OUT_DIR \\"
|
||
echo " cargo run -p mpc-ray --features probe --bin rayprobe -- <gcs_ip:port>"
|
||
else
|
||
echo "== 失败"; exit 1
|
||
fi
|