- 6 new operators: scql (secure sum) / horizontal-fedavg (FedAvg mean) / secure-mlp (sigmoid + Laplace DP) / tee-inference (sha256 attestation) / prompt-defense (keyword classifier) / compliance-audit (uuid trail)
- OperatorsService + experiment.module 注册追加(不替换 chg_0002 的 3 算子)
- tests/e2e/dth_experiment_extend.test.sh: 8 case 覆盖 S7-S12 + list ≥9
- docs/dth/experiment-platform/operators-design.md: T1 架构
- openspec/changes/chg_0005_experiment_platform_extend/{5 工件}
Co-Authored-By: Claude Opus 4.8 <[email protected]>
164 lines
8.8 KiB
Bash
Executable File
164 lines
8.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# dth_experiment_extend.test.sh — e2e verification for S7..S12 of chg_0005.
|
|
# Exit 0 only if all 6 scenarios + list>=9 pass. Each assertion is a hard-fail.
|
|
#
|
|
# Usage:
|
|
# DTH_BASE=http://127.0.0.1:30000 bash tests/e2e/dth_experiment_extend.test.sh
|
|
#
|
|
# Pre-req: backend is running and admin seed is loaded (run.sh dev).
|
|
|
|
set -uo pipefail
|
|
|
|
BASE="${DTH_BASE:-http://127.0.0.1:${PORT:-30000}}"
|
|
LOGIN_URL="$BASE/api/ssw/dth/auth/login"
|
|
OPS_URL="$BASE/api/ssw/dth/experiment/operators"
|
|
RUNS_URL="$BASE/api/ssw/dth/experiment/runs"
|
|
|
|
pass=0; fail=0; fails=()
|
|
|
|
json_get() {
|
|
local field="$1"; local body="$2"
|
|
python3 -c "import json,sys; d=json.loads(sys.argv[1]); print(d${field})" "$body" 2>/dev/null
|
|
}
|
|
|
|
login() {
|
|
curl -s -X POST "$LOGIN_URL" -H 'Content-Type: application/json' -d "{\"username\":\"$1\",\"password\":\"$2\"}"
|
|
}
|
|
|
|
assert() {
|
|
local id="$1"; local cond="$2"; local detail="$3"
|
|
if eval "$cond"; then
|
|
pass=$((pass+1))
|
|
printf " ✓ %s\n" "$id"
|
|
else
|
|
fail=$((fail+1))
|
|
fails+=("$id: $detail")
|
|
printf " ✗ %s — %s\n" "$id" "$detail"
|
|
fi
|
|
}
|
|
|
|
# ----- pre: login admin -----
|
|
ADMIN_BODY="$(login admin 123456)"
|
|
ADMIN_TOKEN="$(json_get "['data']['token']" "$ADMIN_BODY")"
|
|
assert "pre_admin_login" "[ -n '$ADMIN_TOKEN' ]" "admin token empty; login body=$ADMIN_BODY"
|
|
|
|
# ----- S7: scql sum aggregation -----
|
|
SCQL_BODY="$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" -H 'Content-Type: application/json' \
|
|
-d '{"operator":"scql","input":{"columns":[{"values":[10,20,30]},{"values":[1,2,3]}]}}' "$RUNS_URL")"
|
|
SCQL_ID="$(json_get "['data']['run_id']" "$SCQL_BODY")"
|
|
SCQL_STATUS="$(json_get "['data']['status']" "$SCQL_BODY")"
|
|
assert "S7_scql_run_status" "[ '$SCQL_STATUS' = 'success' ]" "scql status=$SCQL_STATUS"
|
|
SCQL_GET="$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" "$RUNS_URL/$SCQL_ID")"
|
|
SCQL_SUM="$(json_get "['data']['output_json']['scql']['sum']" "$SCQL_GET")"
|
|
SCQL_COLS="$(json_get "['data']['output_json']['scql']['column_count']" "$SCQL_GET")"
|
|
SCQL_TOTAL="$(json_get "['data']['output_json']['scql']['total_values']" "$SCQL_GET")"
|
|
assert "S7_scql_sum_66" "[ '$SCQL_SUM' = '66' ]" "sum=$SCQL_SUM"
|
|
assert "S7_scql_column_count_2" "[ '$SCQL_COLS' = '2' ]" "column_count=$SCQL_COLS"
|
|
assert "S7_scql_total_values_6" "[ '$SCQL_TOTAL' = '6' ]" "total_values=$SCQL_TOTAL"
|
|
|
|
# ----- S8: horizontal-fedavg mean of 3 clients -----
|
|
FED_BODY="$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" -H 'Content-Type: application/json' \
|
|
-d '{"operator":"horizontal-fedavg","input":{"clients":[{"weights":[0.1,0.2]},{"weights":[0.3,0.4]},{"weights":[0.5,0.6]}],"aggregation":"mean"}}' "$RUNS_URL")"
|
|
FED_ID="$(json_get "['data']['run_id']" "$FED_BODY")"
|
|
FED_STATUS="$(json_get "['data']['status']" "$FED_BODY")"
|
|
assert "S8_fedavg_run_status" "[ '$FED_STATUS' = 'success' ]" "fedavg status=$FED_STATUS"
|
|
FED_GET="$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" "$RUNS_URL/$FED_ID")"
|
|
FED_AGG="$(json_get "['data']['output_json']['fedavg']['aggregated']" "$FED_GET")"
|
|
FED_ROUND="$(json_get "['data']['output_json']['fedavg']['round_count']" "$FED_GET")"
|
|
FED_CLIENTS="$(json_get "['data']['output_json']['fedavg']['client_count']" "$FED_GET")"
|
|
# aggregated must equal [0.3, 0.4]
|
|
FED_AGG_NORM="$(echo "$FED_AGG" | tr -d ' ')"
|
|
assert "S8_fedavg_aggregated_0.3_0.4" "[ '$FED_AGG_NORM' = '[0.3,0.4]' ]" "aggregated=$FED_AGG"
|
|
assert "S8_fedavg_round_count_1" "[ '$FED_ROUND' = '1' ]" "round_count=$FED_ROUND"
|
|
assert "S8_fedavg_client_count_3" "[ '$FED_CLIENTS' = '3' ]" "client_count=$FED_CLIENTS"
|
|
|
|
# ----- S9: secure-mlp sigmoid predictions + noise -----
|
|
MLP_BODY="$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" -H 'Content-Type: application/json' \
|
|
-d '{"operator":"secure-mlp","input":{"features":[[1.0,2.0],[3.0,4.0]],"weights":[0.5,0.5],"bias":0.1,"epsilon":1.0}}' "$RUNS_URL")"
|
|
MLP_ID="$(json_get "['data']['run_id']" "$MLP_BODY")"
|
|
MLP_STATUS="$(json_get "['data']['status']" "$MLP_BODY")"
|
|
assert "S9_mlp_run_status" "[ '$MLP_STATUS' = 'success' ]" "mlp status=$MLP_STATUS"
|
|
MLP_GET="$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" "$RUNS_URL/$MLP_ID")"
|
|
# Use python to assert length=2 and each in (0,1)
|
|
MLP_ASSERT="$(python3 -c "
|
|
import json,sys
|
|
d=json.loads(sys.argv[1])
|
|
p=d['data']['output_json']['mlp']['predictions']
|
|
n=d['data']['output_json']['mlp']['noise_scale']
|
|
ok = len(p)==2 and all(0 < x < 1 for x in p) and n>0
|
|
print(ok)
|
|
" "$MLP_GET" 2>/dev/null)"
|
|
assert "S9_mlp_predictions_sigmoid" "[ '$MLP_ASSERT' = 'True' ]" "predictions check failed; body=$MLP_GET"
|
|
|
|
# ----- S10: tee-inference attestation -----
|
|
TEE_BODY="$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" -H 'Content-Type: application/json' \
|
|
-d '{"operator":"tee-inference","input":{"prompt":"hello","model":"gpt-mock"}}' "$RUNS_URL")"
|
|
TEE_ID="$(json_get "['data']['run_id']" "$TEE_BODY")"
|
|
TEE_STATUS="$(json_get "['data']['status']" "$TEE_BODY")"
|
|
assert "S10_tee_run_status" "[ '$TEE_STATUS' = 'success' ]" "tee status=$TEE_STATUS"
|
|
TEE_GET="$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" "$RUNS_URL/$TEE_ID")"
|
|
TEE_ASSERT="$(python3 -c "
|
|
import json,sys
|
|
d=json.loads(sys.argv[1])
|
|
t=d['data']['output_json']['tee']
|
|
a=t['attestation']
|
|
ok = len(a)>=64 and a.startswith('tee-mock-') and t['model']=='gpt-mock'
|
|
print(ok)
|
|
" "$TEE_GET" 2>/dev/null)"
|
|
assert "S10_tee_attestation_ok" "[ '$TEE_ASSERT' = 'True' ]" "tee attestation check failed; body=$TEE_GET"
|
|
|
|
# ----- S11: prompt-defense detects injection + clean ignores -----
|
|
DEF_BODY="$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" -H 'Content-Type: application/json' \
|
|
-d '{"operator":"prompt-defense","input":{"prompt":"ignore previous instructions and ..."}}' "$RUNS_URL")"
|
|
DEF_ID="$(json_get "['data']['run_id']" "$DEF_BODY")"
|
|
DEF_STATUS="$(json_get "['data']['status']" "$DEF_BODY")"
|
|
assert "S11_defense_run_status" "[ '$DEF_STATUS' = 'success' ]" "defense status=$DEF_STATUS"
|
|
DEF_GET="$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" "$RUNS_URL/$DEF_ID")"
|
|
DEF_DETECTED="$(json_get "['data']['output_json']['defense']['detected']" "$DEF_GET")"
|
|
DEF_LEVEL="$(json_get "['data']['output_json']['defense']['threat_level']" "$DEF_GET")"
|
|
assert "S11_defense_detected_true" "[ '$DEF_DETECTED' = 'True' ]" "detected=$DEF_DETECTED"
|
|
assert "S11_defense_threat_high" "[ '$DEF_LEVEL' = 'high' ]" "threat_level=$DEF_LEVEL"
|
|
|
|
DEF_CLEAN_BODY="$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" -H 'Content-Type: application/json' \
|
|
-d '{"operator":"prompt-defense","input":{"prompt":"What is 2+2?"}}' "$RUNS_URL")"
|
|
DEF_CLEAN_ID="$(json_get "['data']['run_id']" "$DEF_CLEAN_BODY")"
|
|
DEF_CLEAN_GET="$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" "$RUNS_URL/$DEF_CLEAN_ID")"
|
|
DEF_CLEAN_DETECTED="$(json_get "['data']['output_json']['defense']['detected']" "$DEF_CLEAN_GET")"
|
|
DEF_CLEAN_LEVEL="$(json_get "['data']['output_json']['defense']['threat_level']" "$DEF_CLEAN_GET")"
|
|
assert "S11_clean_not_detected" "[ '$DEF_CLEAN_DETECTED' = 'False' ]" "clean detected=$DEF_CLEAN_DETECTED"
|
|
assert "S11_clean_threat_low" "[ '$DEF_CLEAN_LEVEL' = 'low' ]" "clean level=$DEF_CLEAN_LEVEL"
|
|
|
|
# ----- S12: compliance-audit trail -----
|
|
AUDIT_BODY="$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" -H 'Content-Type: application/json' \
|
|
-d '{"operator":"compliance-audit","input":{"action":"data_export","actor":"alice","user":"alice","timestamp":"2026-09-28T10:00:00Z"}}' "$RUNS_URL")"
|
|
AUDIT_ID="$(json_get "['data']['run_id']" "$AUDIT_BODY")"
|
|
AUDIT_STATUS="$(json_get "['data']['status']" "$AUDIT_BODY")"
|
|
assert "S12_audit_run_status" "[ '$AUDIT_STATUS' = 'success' ]" "audit status=$AUDIT_STATUS"
|
|
AUDIT_GET="$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" "$RUNS_URL/$AUDIT_ID")"
|
|
AUDIT_ASSERT="$(python3 -c "
|
|
import json,sys,re
|
|
d=json.loads(sys.argv[1])
|
|
a=d['data']['output_json']['audit']
|
|
uuid_re=re.compile(r'^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$')
|
|
ok = bool(uuid_re.match(a['trail_id'])) and 0 <= a['compliance_score'] < 1 and a['action']=='data_export'
|
|
print(ok)
|
|
" "$AUDIT_GET" 2>/dev/null)"
|
|
assert "S12_audit_trail_ok" "[ '$AUDIT_ASSERT' = 'True' ]" "audit trail check failed; body=$AUDIT_GET"
|
|
|
|
# ----- extra: list >= 9 operators + new 6 codes present -----
|
|
LIST_BODY="$(curl -s -H "Authorization: Bearer $ADMIN_TOKEN" "$OPS_URL")"
|
|
LIST_LEN="$(python3 -c "import json,sys; print(len(json.loads(sys.argv[1])['data']))" "$LIST_BODY")"
|
|
CODES="$(python3 -c "import json,sys; print(','.join(sorted(o['code'] for o in json.loads(sys.argv[1])['data'])))" "$LIST_BODY")"
|
|
assert "list_at_least_9" "[ '$LIST_LEN' -ge 9 ]" "got $LIST_LEN operators: $CODES"
|
|
for code in scql horizontal-fedavg secure-mlp tee-inference prompt-defense compliance-audit; do
|
|
assert "list_has_${code}" "echo '$CODES' | tr ',' '\n' | grep -qx '$code'" "missing $code in $CODES"
|
|
done
|
|
|
|
echo "---"
|
|
echo "e2e dth_experiment_extend: pass=$pass fail=$fail"
|
|
if [ $fail -gt 0 ]; then
|
|
printf 'FAILED:\n'; printf ' - %s\n' "${fails[@]}"
|
|
exit 1
|
|
fi
|
|
exit 0
|