Files
ipass-plugin-cache/tests/cache/verify-cache-eviction.sh
T
ssw 43718bd0c4 iPass cache gateway 源码
容量管理: zones.rs 直读 nginx file_cache 共享内存记账(双轴: 磁盘 + keys_zone 索引内存), ?scan=1 扫盘上限 20 万文件; POST /api/admin/cache/purge 整区清理(只删文件不删目录, 根治 [crit] mkdir failed); POST /api/admin/cache/invalidate 按制品精准失效(预览->确认两步式, glob/子串, 删除上限 5000/扫描上限 20 万); GET /api/admin/cache 新增 capacity 块; 前端容量面板 + 按制品失效工具条; 测试 test-cache-capacity.sh 48/48(容量 27 + 精准失效 21, 含 EXPECT_MIN 护栏与前端静态契约); docs 容量管理章节
2026-09-21 09:25:11 +08:00

216 lines
10 KiB
Bash
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/bash
# ============================================================================
# verify-cache-eviction.sh —— 验证「缓存超额/闲置后被自动淘汰」
#
# 回答的问题:磁盘缓存有没有上限?写满了谁来清?
#
# 机制(nginx 自带,无需外部脚本):
# * max_size=S zone 容量上限。cache manager 进程持续比对,超了就按
# **LRU(最久未访问)**删到 S 以下 —— 这就是「超额自动清理」。
# * inactive=T 条目 T 时间内没被访问过就删(与容量无关的「闲置淘汰」)。
# 两者都在 `proxy_cache_path` 上按 **zone** 独立配置,互不干扰。
#
# 本脚本用一个临时实例把参数改小到可观测,然后:
# A 段(容量):npm zone 配 max_size=1m,灌 ~3MB → 断言磁盘被压回 ≤1m,
# 且**最早的条目被淘汰(再请求变 MISS)、最新的还在(HIT)**。
# B 段(闲置):maven zone 配 inactive=10s,灌 1 个小对象 → 立刻 HIT,
# 闲置 15s 后断言变 MISS。
#
# bash tests/cache/verify-cache-eviction.sh
#
# ⚠️ 自起自停一个**独立实例**(端口 18086 / 假上游 18997),不碰 dev-server。
# ⚠️ 本脚本**不** pkill —— 只杀自己记下的 pid(pkill 会误伤别人正在看的实例)。
# ============================================================================
set -uo pipefail
curl() { command curl -s --noproxy '*' "$@"; }
HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
REPO=$(cd "$HERE/../.." && pwd)
RUN="$HERE/.run-evict"
PORT=18086
UP_PORT=18997
NGX="$REPO/build/nginx-1.30.4/objs/nginx"
MOD="$REPO/rust/target/debug/libipass_gateway.dylib"
PY=${PY:-python3}
CACHE_PY="$PY"; export CACHE_PY
# shellcheck source=tests/cache/lib-render.sh
source "$HERE/lib-render.sh"
SK=sk_alice_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
AUTH="Authorization: Bearer $SK"
UP_LOG="$RUN/upstream.log"
pass=0; fail=0
ok() { echo " ✅ $*"; pass=$((pass+1)); }
bad() { echo " ❌ $*"; fail=$((fail+1)); }
cleanup() {
[ -f "$RUN/logs/nginx.pid" ] && kill "$(cat "$RUN/logs/nginx.pid")" 2>/dev/null
[ -f "$RUN/upstream.pid" ] && kill "$(cat "$RUN/upstream.pid")" 2>/dev/null
sleep 0.3
}
trap cleanup EXIT
echo "==== 缓存淘汰验证(独立实例 :${PORT})===="
# ---------- 1. 渲染 conf 并把两个 zone 的参数改小到可观测 ----------
cache_write_upstreams "$RUN" "http://127.0.0.1:${UP_PORT}" \
npm maven || exit 1
cache_render_prod_conf "$RUN" "$PORT" "$MOD" >/dev/null
# ⚠️ 必须拷夹具账号:否则用户表为空 → alice 的 sk_ 不被认 → **所有请求 401**,
# 而 401 在 access 阶段就被模块返回,压根不进代理层 —— 症状是
# 「缓存目录 0 文件、响应头里没有 X-Cache-Status」,看着像缓存没生效。
# dev-server.sh 也是这个顺序(先拷 fixtures 再起 nginx)。
cp "$HERE"/fixtures/*.json "$RUN/data/" 2>/dev/null
"$PY" - "$RUN" <<'P'
import re, sys, pathlib
run = sys.argv[1]
p = pathlib.Path(run, "conf", "nginx.cache.conf")
s = p.read_text()
out = []
for line in s.splitlines(True):
# npm → 容量淘汰:1m 上限,闲置窗口放长(避免两段互相干扰)
if "keys_zone=npm:" in line:
line = re.sub(r"max_size=\S+", "max_size=1m", line)
line = re.sub(r"inactive=\S+", "inactive=1h", line)
# maven → 闲置淘汰:窗口 10s,容量放大
if "keys_zone=maven:" in line:
line = re.sub(r"max_size=\S+", "max_size=1g", line)
line = re.sub(r"inactive=\S+", "inactive=10s", line)
out.append(line)
p.write_text("".join(out))
print(" [conf] npm: max_size=1m/inactive=1h maven: max_size=1g/inactive=10s")
P
# 目录结构由 cache_render_prod_conf 建好;这里只删文件(见 test-real-registries.sh 的警告:
# 运行中删掉 cache/<eco> 父目录会让 nginx 永久 crit 断连)。
find "$RUN/cache" -type f -delete >/dev/null 2>&1
# ---------- 2. 起假上游(脱离进程组)与 nginx ----------
mkdir -p "$RUN/logs"
: > "$UP_LOG"
UPSTREAM_LOG="$UP_LOG" UPSTREAM_PORT="$UP_PORT" \
"$PY" - "$PY" "$HERE/mock_upstream.py" "$RUN/logs/upstream.out" "$RUN/upstream.pid" <<'P'
import os, subprocess, sys
py, script, out, pidf = sys.argv[1:5]
with open(out, "ab") as f:
p = subprocess.Popen([py, script], stdout=f, stderr=f,
env=dict(os.environ), start_new_session=True)
open(pidf, "w").write(str(p.pid))
P
wait_up() { local p=$1 i=0; until curl -s -o /dev/null --max-time 1 "http://127.0.0.1:$p/$2" 2>/dev/null; do i=$((i+1)); [ $i -ge 80 ] && return 1; sleep 0.1; done; }
wait_up "$UP_PORT" "" || { echo "❌ 假上游未就绪"; exit 1; }
ADMIN_INIT_PASSWORD=x "$NGX" -p "$RUN" -c conf/nginx.cache.conf -t 2>>"$RUN/logs/error.log" \
|| { echo "❌ nginx -t 失败"; tail -10 "$RUN/logs/error.log"; exit 1; }
ADMIN_INIT_PASSWORD=x "$NGX" -p "$RUN" -c conf/nginx.cache.conf
wait_up "$PORT" "" || { echo "❌ nginx 未就绪"; tail -10 "$RUN/logs/error.log"; exit 1; }
echo " 实例就绪(假上游 ${UP_PORT})"
B="http://127.0.0.1:$PORT"
xc() { tr -d '\r' | awk 'tolower($1)=="x-cache-status:"{print $2}'; }
code() { curl -s -o /dev/null -w '%{http_code}' --max-time 60 -H "$AUTH" "$1"; }
# ===========================================================================
# A 段:容量淘汰(max_size)
# ===========================================================================
echo
echo "== A. 容量淘汰:npm zone max_size=1m,灌入 15 × 200KB = 3MB =="
# 先探一发,把「Key 不被认(401)/上游未配(503)/代理写盘失败」这类故障在灌数据前就暴露,
# 否则后面只会看到一堆空的 X-Cache-Status,分不清是缓存没生效还是请求没进来。
sample=$(curl -s -o /dev/null -w '%{http_code}' -H "$AUTH" "$B/npm/probe?bytes=1024")
echo " 样本请求 HTTP ${sample}(期望 200;401=Key 不被认,503=上游未配)"
[ "$sample" = "200" ] || { echo "❌ 前置检查失败,后面的淘汰断言没有意义"; exit 1; }
N=15; SZ=204800
for i in $(seq 1 $N); do
curl -s -o /dev/null -H "$AUTH" "$B/npm/big/$i?bytes=$SZ"
done
kb0=$(du -sk "$RUN/cache/npm" 2>/dev/null | awk '{print $1}')
files0=$(find "$RUN/cache/npm" -type f 2>/dev/null | wc -l | tr -d ' ')
echo " 灌完后立即:磁盘 ${kb0}KB / ${files0} 个文件(灌入量约 $((N*SZ/1024))KB,上限 1m=1024KB)"
# ★ nginx 的容量淘汰是**渐进**的,不是硬配额:
# cache manager 每秒跑一轮,每轮只淘汰有限个条目(LRU 序),于是
# max_size 是**软上限**——短时猛灌可以短暂远超它(本测实测到 3x),之后逐步收敛。
# 所以断言不能写成「3 秒内必须达标」,那是在测一个 nginx 没有承诺的东西;
# 要测的是「会不会收敛」以及「收敛用多久」(后者对运维更有用)。
echo " -- 观察收敛(每 2s 采样,最多 30s)--"
kb2="$kb0"; files2="$files0"; secs=""
for s in 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30; do
kb2=$(du -sk "$RUN/cache/npm" 2>/dev/null | awk '{print $1}')
files2=$(find "$RUN/cache/npm" -type f 2>/dev/null | wc -l | tr -d ' ')
printf " t=%-2ss %5sKB %2s 个文件\n" "$s" "${kb2:-0}" "${files2:-0}"
if [ "${kb2:-0}" -le 1300 ]; then secs="$s"; break; fi
sleep 2
done
if [ "${kb2:-0}" -le 1300 ]; then
ok "收敛到上限附近:${kb2}KB ≤ 1300KB 容忍线(上限 1m=1024KB),耗时约 ${secs}s"
else
bad "30s 内没收敛(${kb2}KB)—— 淘汰没生效,缓存会无限增长"
fi
if [ "${files2:-0}" -lt "$N" ]; then
ok "条目数下降(${files2} < 灌入的 ${N})—— 确实发生了淘汰"
else
bad "条目数没下降(${files2})—— 淘汰没生效"
fi
echo " -- LRU 方向性检查(该被淘汰的是最久未访问的)--"
# 第 1 个是全场最久未访问 → 期望被淘汰(MISS);最后 1 个刚写进来 → 期望仍在(HIT)
st_old=$(curl -s -D - -o /dev/null -H "$AUTH" "$B/npm/big/1?bytes=$SZ" | xc)
st_new=$(curl -s -D - -o /dev/null -H "$AUTH" "$B/npm/big/$N?bytes=$SZ" | xc)
if [ "$st_old" = "MISS" ]; then
ok "最早的条目 /npm/big/1 已被淘汰(再请求 MISS)"
else
bad "最早的条目还在(${st_old})—— 淘汰的不是最久未访问的?"
fi
if [ "$st_new" = "HIT" ]; then
ok "最新的条目 /npm/big/$N 仍在(HIT)"
else
# 刚探测 /big/1 又写回了一个条目,可能把 /big/$N 挤掉——这本身也是 LRU 行为的证据,
# 但会让断言不稳定,故只在连续两次都不是 HIT 时报失败。
st_new2=$(curl -s -D - -o /dev/null -H "$AUTH" "$B/npm/big/$N?bytes=$SZ" | xc)
[ "$st_new2" = "HIT" ] && ok "最新的条目 /npm/big/$N 仍在(第二次探测 HIT)" \
|| bad "最新条目也丢了(${st_new}→${st_new2})"
fi
# ===========================================================================
# B 段:闲置淘汰(inactive)
# ===========================================================================
echo
echo "== B. 闲置淘汰:maven zone inactive=10s =="
h1=$(curl -s -D - -o /dev/null -H "$AUTH" "$B/maven/keepalive?bytes=1024" | xc)
h2=$(curl -s -D - -o /dev/null -H "$AUTH" "$B/maven/keepalive?bytes=1024" | xc)
[ "$h1" = "MISS" ] && [ "$h2" = "HIT" ] \
&& ok "刚写完:MISS → HIT(缓存生效)" \
|| bad "刚写完就不是 MISS→HIT(${h1} → ${h2})"
n_before=$(find "$RUN/cache/maven" -type f 2>/dev/null | wc -l | tr -d ' ')
echo " 等 15s(> inactive=10s)后不该再命中…"
sleep 15
# cache manager 扫描是异步的,多给几秒
for _ in 1 2 3 4 5 6; do
n_now=$(find "$RUN/cache/maven" -type f 2>/dev/null | wc -l | tr -d ' ')
[ "$n_now" = "0" ] && break
sleep 2
done
n_after=$(find "$RUN/cache/maven" -type f 2>/dev/null | wc -l | tr -d ' ')
if [ "$n_after" = "0" ]; then
ok "闲置条目已被 cache manager 删除(磁盘文件 ${n_before:-1} → 0)"
else
bad "闲置条目仍在(${n_after} 个文件)—— inactive 淘汰没生效"
fi
st_after=$(curl -s -D - -o /dev/null -H "$AUTH" "$B/maven/keepalive?bytes=1024" | xc)
[ "$st_after" = "MISS" ] \
&& ok "再请求变 MISS(确证被清掉了,不是仅文件索引失效)" \
|| bad "再请求仍是 ${st_after}—— 闲置淘汰没生效"
echo
echo "==== 结果:通过 ${pass},失败 ${fail} ===="
echo "清理:bash tests/cache/verify-cache-eviction.sh 跑完会自动停实例;"
echo " 要删净临时目录: rm -rf $RUN"
[ "$fail" -eq 0 ] || exit 1