容量管理: zones.rs 直读 nginx file_cache 共享内存记账(双轴: 磁盘 + keys_zone 索引内存), ?scan=1 扫盘上限 20 万文件; POST /api/admin/cache/purge 整区清理(只删文件不删目录, 根治 [crit] mkdir failed); POST /api/admin/cache/invalidate 按制品精准失效(预览->确认两步式, glob/子串, 删除上限 5000/扫描上限 20 万); GET /api/admin/cache 新增 capacity 块; 前端容量面板 + 按制品失效工具条; 测试 test-cache-capacity.sh 48/48(容量 27 + 精准失效 21, 含 EXPECT_MIN 护栏与前端静态契约); docs 容量管理章节
283 lines
14 KiB
Bash
Executable File
283 lines
14 KiB
Bash
Executable File
#!/bin/bash
|
||
# ============================================================================
|
||
# dev-server.sh —— 本机可交互试跑「统一制品缓存」
|
||
#
|
||
# 与 run.sh(自动化回归)的区别:本脚本把实例**留在前台不关**,你可以自己 curl
|
||
# 观察 MISS→HIT、翻磁盘缓存文件、看管理 API 读数。上游是本地假 registry
|
||
# (mock_upstream.py),因为本机常连不上真实 registry / 不宜拿生产上游当靶子。
|
||
#
|
||
# 跑的 conf 是**生产 conf 的渲染副本**(借 lib-render.sh,与 e2e/Tier2 同源),
|
||
# 所以这里看到的路径剥离、变量上游、鉴权行为都是生产那套。
|
||
#
|
||
# bash tests/cache/dev-server.sh start # 起 mock 上游 + 加载 ipass 模块的 nginx
|
||
# bash tests/cache/dev-server.sh demo # 六生态各打两次,打印 MISS→HIT
|
||
# bash tests/cache/dev-server.sh stats # GET /api/v1/cache/stats(命中计数)
|
||
# bash tests/cache/dev-server.sh files # 列磁盘缓存文件(证明是本地文件缓存)
|
||
# bash tests/cache/dev-server.sh urls # 打印真实客户端接入示例
|
||
# bash tests/cache/dev-server.sh purge # 清空缓存(开源 nginx 无 proxy_cache_purge)
|
||
# bash tests/cache/dev-server.sh logs # tail error.log
|
||
# bash tests/cache/dev-server.sh status # 进程/端口状态
|
||
# bash tests/cache/dev-server.sh stop # 关掉(保留 data/ 与缓存文件)
|
||
# bash tests/cache/dev-server.sh clean # 关掉并删净 .run-dev
|
||
# bash tests/cache/dev-server.sh admin # 打印管理台地址、登录方式、能做什么
|
||
#
|
||
# 端口:http 127.0.0.1:18085 ,假上游 127.0.0.1:18998
|
||
# ⚠️ 不要与 bash tests/cache/run.sh 同时跑(测试会 pkill 掉 mock/nginx)。
|
||
# ⚠️ 制品路由现在**要鉴权**:`-H "Authorization: Bearer $SK"`(SK 见 fixtures/users.json,
|
||
# 本脚本用 alice 那把;stats 也是账号态,同样要带)。
|
||
# ============================================================================
|
||
set -uo pipefail
|
||
|
||
# 本机 HTTP_PROXY/HTTPS_PROXY 会把 127.0.0.1 的请求丢给代理 → 假 502
|
||
curl() { command curl --noproxy '*' "$@"; }
|
||
|
||
HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||
REPO=$(cd "$HERE/../.." && pwd)
|
||
RUN="$HERE/.run-dev"
|
||
|
||
NGX_SRC="$REPO/build/nginx-1.30.4"
|
||
NGINX_BIN=${NGINX_BIN:-$NGX_SRC/objs/nginx}
|
||
MODULE=${MODULE:-$REPO/rust/target/debug/libipass_gateway.dylib}
|
||
CARGO=${CARGO:-$HOME/.cargo/bin/cargo}
|
||
PY=${PY:-python3}
|
||
CACHE_PY="$PY"; export CACHE_PY
|
||
# shellcheck source=tests/cache/lib-render.sh
|
||
source "$HERE/lib-render.sh"
|
||
|
||
PORT=18085
|
||
UP_PORT=18998
|
||
BASE="http://127.0.0.1:$PORT"
|
||
SK=sk_alice_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||
AUTH="Authorization: Bearer $SK"
|
||
|
||
# 六生态 → **URI 前缀**(前缀 ≠ 路由名 ≠ 生态名:/v2/ 的前缀对应路由名 docker,
|
||
# 上游表按键是路由名 docker)
|
||
ROUTE_PAIRS="go:/go cargo:/cargo pypi:/pypi npm:/npm maven:/maven docker:/v2"
|
||
|
||
TDIR() { echo "$RUN" ; }
|
||
pid_of() { cat "$1" 2>/dev/null; }
|
||
|
||
# 端口探活:直到能建立连接再往下走(HEAD 探活,不会写进上游请求日志)。
|
||
wait_up() { # $1=port $2=最大尝试次数(默认 80,步长 0.1s)
|
||
local port="$1" n="${2:-80}" i=0
|
||
until curl -s -o /dev/null -I --max-time 1 "http://127.0.0.1:$port/" 2>/dev/null; do
|
||
i=$((i+1)); [ "$i" -ge "$n" ] && return 1; sleep 0.1
|
||
done
|
||
return 0
|
||
}
|
||
|
||
build_if_needed() {
|
||
[ -f "$MODULE" ] && [ "${FORCE_BUILD:-0}" != "1" ] && return 0
|
||
echo "── 编译 ipass 模块(首次/强制)──"
|
||
[ -x "$CARGO" ] || { echo "错误:找不到 cargo(用 CARGO= 指定)" >&2; exit 1; }
|
||
[ -z "${LIBCLANG_PATH:-}" ] && [ -d /Library/Developer/CommandLineTools/usr/lib ] \
|
||
&& export LIBCLANG_PATH=/Library/Developer/CommandLineTools/usr/lib
|
||
( cd "$REPO/rust" \
|
||
&& NGINX_SOURCE_DIR="$NGX_SRC" NGINX_BUILD_DIR="$NGX_SRC/objs" \
|
||
LIBCLANG_PATH="${LIBCLANG_PATH:-}" "$CARGO" build ) || return 1
|
||
echo " ✅ $MODULE"
|
||
}
|
||
|
||
start() {
|
||
build_if_needed || { echo "❌ 模块编译失败" >&2; exit 1; }
|
||
[ -x "$NGINX_BIN" ] || { echo "错误:nginx 不存在 $NGINX_BIN" >&2; exit 1; }
|
||
stop_quiet
|
||
|
||
mkdir -p "$RUN/logs" "$RUN/data" "$RUN/audit" "$RUN/cache"
|
||
cp "$HERE"/fixtures/*.json "$RUN/data/" 2>/dev/null
|
||
: > "$RUN/upstream.log"
|
||
|
||
# 上游表:六生态的 route 全指向假上游(route 名:docker 是 v2)。
|
||
# 不配 = 该路由 503(这是 access 层的闸门,不是 bug)。
|
||
cache_write_upstreams "$RUN" "http://127.0.0.1:${UP_PORT}" \
|
||
go cargo cargo-dl pypi pypi-files npm maven docker || exit 1
|
||
# 渲染生产 conf(端口错开自动回归,避免互相踩)
|
||
cache_render_prod_conf "$RUN" "$PORT" "$MODULE"
|
||
|
||
# ⚠️ 假上游必须**脱离本进程组**(start_new_session ≈ setsid)。
|
||
# 本脚本常被「起一次就返回」的外壳调用(CI、编辑器里的命令执行器)。这类外壳在
|
||
# 命令返回后会清理自己进程组里的后台任务——普通 `cmd &` 会连坐被杀,于是
|
||
# nginx(daemon 化,已脱离)活着、所有路由 502「Connection refused」、
|
||
# upstream.log 是 0 行,看着像缓存/网关坏了,其实只是上游没了。实测踩过。
|
||
UPSTREAM_LOG="$RUN/upstream.log" UPSTREAM_PORT="$UP_PORT" \
|
||
"$PY" - "$PY" "$HERE/mock_upstream.py" "$RUN/logs/upstream.out" "$RUN/upstream.pid" <<'P'
|
||
import os, subprocess, sys
|
||
py, script, out, pidf = sys.argv[1:5]
|
||
with open(out, "ab") as f:
|
||
p = subprocess.Popen([py, script], stdout=f, stderr=f,
|
||
env=dict(os.environ), start_new_session=True)
|
||
open(pidf, "w").write(str(p.pid))
|
||
P
|
||
wait_up "$UP_PORT" || { echo "❌ 假上游端口 $UP_PORT 未就绪"; exit 1; }
|
||
|
||
# 出厂管理员口令只在 users 表**为空**时播种。本脚本先拷了夹具 users.json(里面已有
|
||
# admin),所以下面这个默认值对本地试跑**不生效**——登录用的是夹具里那把口令,
|
||
# 别把它当成登录密码去找(`admin` 子命令会说明)。
|
||
export ADMIN_INIT_PASSWORD="${ADMIN_INIT_PASSWORD:-dev_admin_pw}"
|
||
if ! "$NGINX_BIN" -p "$RUN" -c conf/nginx.cache.conf -t 2>>"$RUN/logs/error.log"; then
|
||
echo "❌ nginx -t 失败:"; tail -15 "$RUN/logs/error.log"; exit 1
|
||
fi
|
||
"$NGINX_BIN" -p "$RUN" -c conf/nginx.cache.conf
|
||
wait_up "$PORT" || { echo "❌ nginx 端口 $PORT 未就绪"; tail -15 "$RUN/logs/error.log"; exit 1; }
|
||
|
||
echo "✅ 已启动"
|
||
echo " 管理台 $BASE/admin (在页面上配上游 / 一键复制下游地址)"
|
||
echo " 入口 $BASE"
|
||
echo " 制品 $BASE/<route>/<path> 例如 $BASE/go/@v/list $BASE/v2/library/alpine/manifests/latest"
|
||
echo " (要带 Key: -H \"$AUTH\")"
|
||
echo " 读数 $BASE/api/v1/cache/stats (账号态,同样要带 Key)"
|
||
echo " 缓存盘 $RUN/cache/<eco>/(真实文件)"
|
||
echo " 假上游 $(pid_of "$RUN/upstream.pid") nginx $(pid_of "$RUN/logs/nginx.pid")"
|
||
echo " 试一下: bash tests/cache/dev-server.sh demo"
|
||
}
|
||
|
||
stop_quiet() {
|
||
[ -f "$RUN/logs/nginx.pid" ] && kill "$(pid_of "$RUN/logs/nginx.pid")" 2>/dev/null
|
||
[ -f "$RUN/upstream.pid" ] && kill "$(pid_of "$RUN/upstream.pid")" 2>/dev/null
|
||
sleep 0.4
|
||
}
|
||
|
||
stop() {
|
||
stop_quiet
|
||
rm -f "$RUN/logs/nginx.pid" "$RUN/upstream.pid"
|
||
echo "✅ 已停止(缓存文件保留在 $RUN/cache)"
|
||
}
|
||
|
||
status() {
|
||
local np up
|
||
np=$(pid_of "$RUN/logs/nginx.pid"); up=$(pid_of "$RUN/upstream.pid")
|
||
if [ -n "$np" ] && kill -0 "$np" 2>/dev/null; then echo "nginx : 运行中 pid=$np"; else echo "nginx : 未运行"; fi
|
||
if [ -n "$up" ] && kill -0 "$up" 2>/dev/null; then echo "假上游 : 运行中 pid=$up"; else echo "假上游 : 未运行"; fi
|
||
curl -s -o /dev/null -w "探活 : HTTP %{http_code}\n" "$BASE/admin" 2>/dev/null || echo "探活 : 不可达"
|
||
}
|
||
|
||
demo() {
|
||
echo "── 六生态各请求两次(第一次 MISS 回源,第二次 HIT 走盘)──"
|
||
echo " (route 前缀 ≠ 生态名:docker 生态走 /v2/)"
|
||
printf "%-8s %-10s %-10s %s\n" 生态 首请求 二次 内容一致
|
||
for pair in $ROUTE_PAIRS; do
|
||
e=${pair%%:*}; pre=${pair#*:}
|
||
h1="$RUN/.h1"; b1="$RUN/.b1"; h2="$RUN/.h2"; b2="$RUN/.b2"
|
||
curl -s -D "$h1" -o "$b1" -H "$AUTH" "$BASE${pre}/pkg-a"
|
||
s1=$(grep -i '^X-Cache-Status:' "$h1" | tr -d '\r' | awk '{print $2}')
|
||
curl -s -D "$h2" -o "$b2" -H "$AUTH" "$BASE${pre}/pkg-a"
|
||
s2=$(grep -i '^X-Cache-Status:' "$h2" | tr -d '\r' | awk '{print $2}')
|
||
same=$([ "$(cat "$b1")" = "$(cat "$b2")" ] && echo yes || echo NO)
|
||
printf "%-8s %-10s %-10s %s\n" "$e" "${s1:-?}" "${s2:-?}" "$same"
|
||
done
|
||
rm -f "$RUN"/.h1 "$RUN"/.b1 "$RUN"/.h2 "$RUN"/.b2
|
||
echo
|
||
echo "本轮回源次数(空缓存时首轮应为 6;全 HIT 则为 0): $(wc -l < "$RUN/upstream.log" | tr -d ' ')"
|
||
echo "(首轮想看到 6→0 的对比:先 purge 再 demo)"
|
||
}
|
||
|
||
show_stats() {
|
||
local raw
|
||
raw=$(curl -s -H "$AUTH" "$BASE/api/v1/cache/stats")
|
||
[ -z "$raw" ] && { echo "拿不到响应,先 start"; return 1; }
|
||
echo "$raw" | grep -q '"code":401' && {
|
||
echo "❌ 账号态 stats 需要 Key;本机夹具里 alice 的 Key 应可用。原始响应:$raw"; return 1; }
|
||
# 注:用 % 格式化而非 f-string——shell 单引号内嵌双引号在旧版 python 会炸
|
||
STATS="$raw" "$PY" -c '
|
||
import json, os
|
||
d = json.loads(os.environ["STATS"])
|
||
data = d.get("data", d)
|
||
be = data.get("by_ecosystem", {})
|
||
print("%-9s%6s%6s%7s" % ("生态", "HIT", "MISS", "其他"))
|
||
for eco in ["go", "cargo", "pypi", "npm", "maven", "docker", "global"]:
|
||
m = be.get(eco, {})
|
||
other = sum(v for k, v in m.items() if k not in ("HIT", "MISS"))
|
||
print("%-9s%6d%6d%7d" % (eco, m.get("HIT", 0), m.get("MISS", 0), other))
|
||
' 2>/dev/null || echo "原始响应:$raw"
|
||
}
|
||
|
||
show_files() {
|
||
echo "── 缓存盘真实文件($RUN/cache,levels=1:2 两级散列)──"
|
||
for e in go cargo pypi npm maven docker; do
|
||
n=$(find "$RUN/cache/$e" -type f 2>/dev/null | wc -l | tr -d ' ')
|
||
sz=$(du -sh "$RUN/cache/$e" 2>/dev/null | awk '{print $1}')
|
||
printf " %-8s %3s 个文件 %s\n" "$e" "$n" "${sz:-0B}"
|
||
done
|
||
echo
|
||
echo " 样例(最深 3 条):"
|
||
find "$RUN/cache" -type f 2>/dev/null | head -3 | sed 's|^| |'
|
||
echo " 说明:文件名是 proxy_cache_key 的 md5,内容即上游响应体;"
|
||
echo " 计量计数**不在**这里,而在 ipass 共享内存(所以重启 nginx 会清零读数,缓存文件仍在)。"
|
||
}
|
||
|
||
purge() {
|
||
echo "── 清空缓存(开源 nginx 无 proxy_cache_purge,直接删目录;文件没了下次自动 MISS 回源)──"
|
||
for e in go cargo pypi npm maven docker; do
|
||
rm -rf "$RUN/cache/$e"; mkdir -p "$RUN/cache/$e"
|
||
done
|
||
echo "✅ 已清空 $RUN/cache/*(共享内存计数未清零,需 restart nginx)"
|
||
}
|
||
|
||
urls() {
|
||
cat <<EOF
|
||
── 真实客户端怎么把源指到这里(把 HOST 换成你的域名/网关:8080)──
|
||
制品路由要鉴权:统一 sk_ Key(Authorization: Bearer / x-api-key / api-key 三选一)
|
||
|
||
Go module proxy export GOPROXY=http://HOST/go # 全量走缓存 ✅
|
||
(go 不带自定义头,需用 .netrc 或前置代理注入 Authorization;
|
||
或把 /go/ 的上游配成「无凭据」并另有 IP 白名单——见 docs/cache-usage.md)
|
||
npm registry npm config set registry http://HOST/npm/ # packument+tarball 同源 ✅
|
||
npm 支持 //HOST/npm/:_authToken=<sk_...>
|
||
Maven mirror settings.xml <mirror><url>http://HOST/maven</url> # POM/JAR 同源 ✅
|
||
Docker /etc/docker/daemon.json registry-mirrors: ["https://HOST"]
|
||
⚠️ 上游 route 是 /v2/(registry v2 协议),且 registry-mirrors 只对
|
||
docker.io 官方镜像生效;docker 客户端会先向 auth.docker.io 换
|
||
短期 Bearer token,网关按 $cache_auth 透传给上游(自己的 sk_ 不外传)
|
||
cargo 稀疏索引 [registries.crates-io] index = "sparse+http://HOST/cargo/"
|
||
⚠️ 索引里 config.json 的 dl 是绝对地址 static.crates.io,
|
||
.crate 包本体不会自动走缓存(见 docs/cache-usage.md 的边界说明)
|
||
pip 索引 pip install -i http://HOST/pypi/simple/ <pkg>
|
||
⚠️ 包本体在 files.pythonhosted.org(绝对地址),同上属边界
|
||
EOF
|
||
}
|
||
|
||
logs() { tail -n "${1:-30}" "$RUN/logs/error.log"; }
|
||
|
||
admin() {
|
||
cat <<EOF
|
||
── 管理台(本机实例;改上游**不需要 reload**)──
|
||
地址 $BASE/admin
|
||
登录 用户名 admin
|
||
口令是**夹具内置**的那把(tests/cache/fixtures/users.json 里 admin 的 pwd_hash)。
|
||
注意它不是 ADMIN_INIT_PASSWORD——后者只在 users 表为空时才播种,本脚本已拷夹具,
|
||
故对本地试跑不生效(找不着密码时先看这句)。
|
||
页面上能做什么
|
||
· 「上游」表 每行一个路由(8 个:go / cargo / cargo-dl / pypi / pypi-files / npm / maven / docker)
|
||
· 编辑 → 改 base 与凭据(凭据要勾「修改凭据」才提交,空串=清空)
|
||
· 停用某行 → 该路由立刻 503「上游尚未配置」(不是 bug,是 access 闸门)
|
||
· 删除 → 同上;已落盘的缓存文件仍在
|
||
· ★ 保存后**立即生效**:变量每次求值都读共享内存,无需 reload/重启
|
||
· 复制接入命令 每个主路由一行按钮,命令文案由后端 config::CLIENT_HINTS 下发
|
||
(前缀与路由强耦合,放前端必然漂移,所以后端给)
|
||
· 测试 走 /_cache_test/<route>,与真实转发共用「拼目标 + 换凭据」逻辑,
|
||
所以这里通 = 真实请求能到上游
|
||
· 命中计数 共享内存读数(HIT/MISS/…)。**重启 nginx 归零**,但磁盘缓存文件还在,
|
||
所以「计数清零」不等于「缓存没了」
|
||
对照命令行
|
||
制品请求 curl -H "$AUTH" $BASE/go/@v/list
|
||
该请求读数 $BASE/api/v1/cache/stats (账号态,同样要带 Key)
|
||
磁盘缓存 $RUN/cache/<eco>/(真实文件,文件名 = proxy_cache_key 的 md5)
|
||
EOF
|
||
}
|
||
|
||
case "${1:-help}" in
|
||
start) start ;;
|
||
stop) stop ;;
|
||
status) status ;;
|
||
demo) demo ;;
|
||
stats) show_stats ;;
|
||
files) show_files ;;
|
||
urls) urls ;;
|
||
admin) admin ;;
|
||
purge) purge ;;
|
||
logs) logs "${2:-30}" ;;
|
||
clean) stop; rm -rf "$RUN"; echo "✅ 已删除 $RUN" ;;
|
||
help|"") sed -n '3,24p' "$0" | sed 's/^# \{0,1\}//' ;;
|
||
*) echo "用法: bash tests/cache/dev-server.sh {start|stop|status|demo|stats|files|urls|admin|purge|logs|clean}" >&2; exit 2 ;;
|
||
esac
|