Files
ssw-mpc/build/fetch_ray_deps.py
rong 4bb573d98c feat: Ray C++ 内核接进 nax Rust + armv7 真交叉编译
Ray 这条线不绕 bazel:
- wheel 里的 ray/_raylet.so 就是编好的 C++ core(34113 个导出符号,
  3638 个 ray:: 符号 default 可见,含 CoreWorkerProcess::Initialize)
- ray/nax_ray_shim.cc:259 行 C ABI shim,复用 process_helper 的初始化路径
- 实测跑通:Rust 二进制作为 driver 接入真实 GCS+raylet,
  拿到 raylet 分配的 node_id(d190801c…),probe 通过

踩坑(都已固化进 ray/build.sh):
- raylet 只编了 Python worker pool → 注册时语言声明 PYTHON
- Ray C++ 是旧 ABI → -D_GLIBCXX_USE_CXX11_ABI=0
- fmt v9/v11 撞车 → FMT_HEADER_ONLY
- Debian absl 固定自家 optional → 影子头直通 std::optional
- _raylet.so 是 CPython 扩展(253 个 Py 符号)→ build.rs 自动链 libpython
- GcsNodeInfo 手写 wire format 解析,避免 protobuf 跨 ABI

armv7 交叉编译修正(原结论有误):
- 换 armv7-unknown-linux-musleabihf + rust-lld/link-self-contained
- 不再需要外部交叉 gcc,macOS 本机 83 秒出 687 KB 静态二进制
- .cargo/config.toml 固化,无需 export 任何变量

其他:nax-orch 告警清理;README/ZYNQ 文档对齐实测。
2026-10-30 11:31:02 +08:00

107 lines
3.6 KiB
Python

#!/usr/bin/env python3
"""把 Ray 的 38 个 bazel 依赖在 mac 上经 ghfast 拉成 distdir。
沙箱里 github.com 超时(12s),但 mac 走 ghfast 正常;沙箱到 mac 的出站
已经打通(allow_out 192.168.1.0/24),所以 mac 拉一次、sandbox 再从 mac 取。
"""
import hashlib
import os
import re
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
BZL = "/tmp/ray_deps.bzl"
DEST = "/tmp/raydist"
GHFAST = "https://ghfast.top/"
def parse_archives(path):
text = open(path).read()
out = []
# 按 http_archive( 切块,括号配平不可靠,这里用下一个 http_archive( 作边界
parts = text.split("http_archive(")[1:]
for p in parts:
name = re.search(r'name\s*=\s*"([^"]+)"', p)
sha = re.search(r'sha256\s*=\s*"([0-9a-f]{64})"', p)
url = re.search(r'\burl\s*=\s*"([^"]+)"', p)
urls = re.search(r"urls\s*=\s*\[(.*?)\]", p, re.S)
cand = []
if url:
cand.append(url.group(1))
if urls:
cand += re.findall(r'"([^"]+)"', urls.group(1))
out.append({
"name": name.group(1) if name else "?",
"sha256": sha.group(1) if sha else None,
"urls": cand,
})
return out
def sha256_file(p):
h = hashlib.sha256()
with open(p, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def download(item):
"""返回 (name, status, size, note)。"""
name = item["name"]
if not item["urls"]:
return (name, "skip", 0, "无 url")
last = ""
for raw in item["urls"]:
base = raw.rstrip("/").split("/")[-1].split("?")[0]
if not base:
continue
dst = os.path.join(DEST, base)
if os.path.exists(dst) and item["sha256"] and sha256_file(dst) == item["sha256"]:
return (name, "cached", os.path.getsize(dst), base)
# github 走 ghfast,其他直连
url = GHFAST + raw if "github.com" in raw or "sourceforge" in raw else raw
for attempt in (url, raw):
t0 = time.time()
r = subprocess.run(
["curl", "-sSL", "--http1.1", "--max-time", "600", "-o", dst, attempt],
capture_output=True,
)
if r.returncode == 0 and os.path.exists(dst) and os.path.getsize(dst) > 0:
if item["sha256"]:
got = sha256_file(dst)
if got != item["sha256"]:
last = f"sha 不符 {got[:8]}!={item['sha256'][:8]}"
continue
return (name, "ok", os.path.getsize(dst), f"{base} {time.time()-t0:.0f}s")
last = (r.stderr or b"").decode()[:80]
return (name, "fail", 0, last)
def main():
os.makedirs(DEST, exist_ok=True)
items = parse_archives(BZL)
print(f"解析到 {len(items)} 个 http_archive", flush=True)
total = 0
fails = []
with ThreadPoolExecutor(max_workers=4) as ex:
for name, st, size, note in ex.map(download, items):
total += size
mark = "OK " if st in ("ok", "cached") else st.upper()
print(f" [{mark}] {name:42s} {size/1e6:7.1f}MB {note}", flush=True)
if st == "fail":
fails.append(name)
print(f"\n合计 {total/1e6:.1f} MB,失败 {len(fails)} 个: {fails}", flush=True)
files = sorted(os.listdir(DEST))
print(f"distdir 文件数 {len(files)}", flush=True)
with open("/tmp/raydist_manifest.txt", "w") as f:
for fn in files:
f.write(fn + "\n")
main()