feat(quality): sandbox preflight hardening + benchlocal-cli in report.sh
Community rigs hit two silent quality-test failure classes: (1) runs "skipping" the sandboxed packs because the Docker images were never built (the #492 warning exists but --sandboxed-only still warn-and-ran a guaranteed-useless run), and (2) rigs that pulled a benchlocal-cli update kept scoring on OLD sandbox images until told to rebuild manually — nothing detected the drift. And report.sh collected none of this, so triage had to ask. quality-test.sh: - --sandboxed-only + missing images/Docker → HARD FAIL up front with the build instructions (a run with zero runnable packs is never intended). --full keeps warn-and-continue (deterministic packs are still useful). - NEW staleness heuristic: each sandbox image's Created is compared against the benchlocal-cli console-script mtime (rewritten on every (re)install — portable across pip-from-git AND editable-checkout installs). Image older than CLI → warning naming the image + both dates + the rebuild command. Explicitly labelled heuristic (an unrelated reinstall trips it) → WARN, never abort. report.sh: - New "Quality tooling (benchlocal-cli + sandboxes)" section: CLI path / version (via the console-script's own interpreter) / install date, the 4 sandbox images with build dates + per-image OLDER-than-CLI flags + rebuild hint, and the newest results/quality/*.json. All best-effort; piped through redact (verified leak-clean live). Exact-version staleness (image labels + a `sandbox-status` command) needs upstream benchlocal-cli changes — tracked in the todo. Verified live: hard-fail branch (docker stub), warn-continue branch, staleness branch (fresh CLI vs real images), report section rendering. test-quality-baseline / test-quality-thinking / test-report-calib green. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EfF565T9eSLaqGzidyJ1Pm
This commit is contained in:
noonghunna
@@ -455,14 +455,48 @@ fi
|
||||
# instead of letting users discover a dead path mid-run (club-3090 #492).
|
||||
if [[ -z "$PACK" ]] && { [[ "$MODE" == "--full" && "$NO_SANDBOX" != "1" ]] || [[ "$SANDBOXED_ONLY" == "1" ]]; }; then
|
||||
_sb_missing=()
|
||||
_sb_stale=()
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
_sb_missing=("Docker not found on PATH")
|
||||
else
|
||||
# Install time of the benchlocal-cli console script — rewritten on every
|
||||
# (re)install, so it's a portable "CLI last updated" timestamp that works
|
||||
# for pip-from-git AND editable-checkout installs alike.
|
||||
_bl_bin="$(command -v benchlocal-cli || true)"
|
||||
_bl_mtime=0
|
||||
[[ -n "$_bl_bin" ]] && _bl_mtime="$(stat -c %Y "$_bl_bin" 2>/dev/null || echo 0)"
|
||||
for _img in benchlocal-sandbox-bugfind benchlocal-sandbox-cli benchlocal-sandbox-hermes; do
|
||||
docker image inspect "${_img}:latest" >/dev/null 2>&1 || _sb_missing+=("${_img}:latest")
|
||||
if ! docker image inspect "${_img}:latest" >/dev/null 2>&1; then
|
||||
_sb_missing+=("${_img}:latest")
|
||||
continue
|
||||
fi
|
||||
# Staleness heuristic: the image was built BEFORE the currently-installed
|
||||
# benchlocal-cli. If that update touched sandbox sources (verifiers,
|
||||
# harness, deps), results run against the OLD behavior — the exact
|
||||
# incident class where user rigs kept scoring on pre-fix sandboxes until
|
||||
# told to rebuild manually. Heuristic (an unrelated reinstall also trips
|
||||
# it), hence WARN not abort.
|
||||
_img_created="$(docker image inspect "${_img}:latest" --format '{{.Created}}' 2>/dev/null || true)"
|
||||
if [[ -n "$_img_created" && "$_bl_mtime" -gt 0 ]]; then
|
||||
_img_epoch="$(date -d "$_img_created" +%s 2>/dev/null || echo 0)"
|
||||
if [[ "$_img_epoch" -gt 0 && "$_img_epoch" -lt "$_bl_mtime" ]]; then
|
||||
_sb_stale+=("${_img}:latest ($(date -d "@${_img_epoch}" +%F) < CLI $(date -d "@${_bl_mtime}" +%F))")
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [[ ${#_sb_missing[@]} -gt 0 ]]; then
|
||||
if [[ "$SANDBOXED_ONLY" == "1" ]]; then
|
||||
# --sandboxed-only with nothing to run in is a guaranteed-useless run:
|
||||
# refuse up front instead of warn-and-skip-everything (#492 follow-up).
|
||||
echo "✗ --sandboxed-only requested but the sandbox prerequisites are missing: ${_sb_missing[*]}" >&2
|
||||
echo " The sandbox packs need pre-built Docker images that aren't auto-pulled. Build them once" >&2
|
||||
echo " from a benchlocal-cli CHECKOUT (the build tooling isn't in the pip package):" >&2
|
||||
echo " git clone https://github.com/noonghunna/benchlocal-cli" >&2
|
||||
echo " bash benchlocal-cli/tools/build-sandboxes.sh # ~30 GB free; prune if tight" >&2
|
||||
echo " then re-run. For a no-Docker run instead: bash scripts/quality-test.sh --medium" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[quality-test] ⚠ sandbox packs (BugFind / CLI / Hermes) will be SKIPPED — not available: ${_sb_missing[*]}" >&2
|
||||
echo " They need pre-built Docker images that aren't auto-pulled. Build them once from a" >&2
|
||||
echo " benchlocal-cli CHECKOUT (the build tooling isn't in the pip package):" >&2
|
||||
@@ -472,6 +506,15 @@ if [[ -z "$PACK" ]] && { [[ "$MODE" == "--full" && "$NO_SANDBOX" != "1" ]] || [[
|
||||
echo " (Continuing with the deterministic packs.)" >&2
|
||||
echo >&2
|
||||
fi
|
||||
if [[ ${#_sb_stale[@]} -gt 0 ]]; then
|
||||
echo "[quality-test] ⚠ sandbox image(s) OLDER than your installed benchlocal-cli:" >&2
|
||||
for _s in "${_sb_stale[@]}"; do echo " - ${_s}" >&2; done
|
||||
echo " If that benchlocal-cli update changed sandbox sources (verifiers/harness)," >&2
|
||||
echo " your scores will reflect the OLD sandbox behavior. Rebuild to be safe:" >&2
|
||||
echo " bash <benchlocal-cli-checkout>/tools/build-sandboxes.sh" >&2
|
||||
echo " (Heuristic — an unrelated CLI reinstall also trips this. Continuing.)" >&2
|
||||
echo >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---- run benchlocal-cli ------------------------------------------------------
|
||||
|
||||
@@ -555,6 +555,58 @@ if have python3 && [[ -f tools/kv-calc.py ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Quality tooling (benchlocal-cli + sandboxes)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Triage for "my quality run skipped packs / scored weird": is benchlocal-cli
|
||||
# installed, how fresh, are the sandbox images built, and do they PREDATE the
|
||||
# CLI (the rebuilt-CLI-stale-sandboxes incident class)? All best-effort — a
|
||||
# rig without any of this still produces a report.
|
||||
|
||||
section "Quality tooling (benchlocal-cli + sandboxes)"
|
||||
{
|
||||
bl_bin=$(command -v benchlocal-cli 2>/dev/null || true)
|
||||
if [[ -z "$bl_bin" ]]; then
|
||||
echo "- **benchlocal-cli:** not installed (quality-test.sh needs it — \`pip install git+https://github.com/noonghunna/benchlocal-cli.git\`)"
|
||||
else
|
||||
bl_mtime=$(stat -c %Y "$bl_bin" 2>/dev/null || echo 0)
|
||||
bl_when=$([[ "$bl_mtime" -gt 0 ]] && date -d "@${bl_mtime}" +%F 2>/dev/null || echo "unknown")
|
||||
# Version via the CLI's own interpreter (works for pip-from-git AND
|
||||
# editable-checkout installs; console-script shebang points at the env).
|
||||
bl_py=$(head -1 "$bl_bin" 2>/dev/null | sed 's/^#!//')
|
||||
bl_ver=$([[ -x "$bl_py" ]] && "$bl_py" -c 'import importlib.metadata as m; print(m.version("benchlocal-cli"))' 2>/dev/null || true)
|
||||
echo "- **benchlocal-cli:** \`${bl_bin}\` (version: \`${bl_ver:-unknown}\`, installed/updated: ${bl_when})"
|
||||
if have docker && docker info >/dev/null 2>&1; then
|
||||
stale_any=0
|
||||
echo "- **Sandbox images** (needed by the --full sandboxed packs):"
|
||||
for img in benchlocal-sandbox-bugfind benchlocal-sandbox-cli benchlocal-sandbox-hermes benchlocal-sandbox-aider-polyglot; do
|
||||
created=$(docker image inspect "${img}:latest" --format '{{.Created}}' 2>/dev/null || true)
|
||||
if [[ -z "$created" ]]; then
|
||||
echo " - \`${img}\`: ✗ not built"
|
||||
continue
|
||||
fi
|
||||
cdate=$(date -d "$created" +%F 2>/dev/null || echo "$created")
|
||||
cepoch=$(date -d "$created" +%s 2>/dev/null || echo 0)
|
||||
mark=""
|
||||
if [[ "$cepoch" -gt 0 && "$bl_mtime" -gt 0 && "$cepoch" -lt "$bl_mtime" ]]; then
|
||||
mark=" ⚠ OLDER than the installed CLI — rebuild if the update touched sandbox sources"
|
||||
stale_any=1
|
||||
fi
|
||||
echo " - \`${img}\`: built ${cdate}${mark}"
|
||||
done
|
||||
[[ "$stale_any" == "1" ]] && echo "- **Rebuild:** \`bash <benchlocal-cli-checkout>/tools/build-sandboxes.sh\` (heuristic — an unrelated reinstall also trips it)"
|
||||
else
|
||||
echo "- **Sandbox images:** docker unavailable — cannot inspect (sandboxed packs need Docker)"
|
||||
fi
|
||||
fi
|
||||
latest_q=$(ls -t results/quality/quality-*.json 2>/dev/null | head -1)
|
||||
if [[ -n "$latest_q" ]]; then
|
||||
echo "- **Latest quality result:** \`${latest_q}\` ($(date -d "@$(stat -c %Y "$latest_q")" +%F 2>/dev/null || echo '?'))"
|
||||
else
|
||||
echo "- **Latest quality result:** none found under results/quality/"
|
||||
fi
|
||||
} | redact
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Active container
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user