From ea9c233fefcb406c9553a326ffc5135fc6091e67 Mon Sep 17 00:00:00 2001 From: noonghunna <10742901+noonghunna@users.noreply.github.com> Date: Mon, 6 Jul 2026 21:56:04 +0000 Subject: [PATCH] feat(quality): sandbox preflight hardening + benchlocal-cli in report.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Community rigs hit two silent quality-test failure classes: (1) runs "skipping" the sandboxed packs because the Docker images were never built (the #492 warning exists but --sandboxed-only still warn-and-ran a guaranteed-useless run), and (2) rigs that pulled a benchlocal-cli update kept scoring on OLD sandbox images until told to rebuild manually — nothing detected the drift. And report.sh collected none of this, so triage had to ask. quality-test.sh: - --sandboxed-only + missing images/Docker → HARD FAIL up front with the build instructions (a run with zero runnable packs is never intended). --full keeps warn-and-continue (deterministic packs are still useful). - NEW staleness heuristic: each sandbox image's Created is compared against the benchlocal-cli console-script mtime (rewritten on every (re)install — portable across pip-from-git AND editable-checkout installs). Image older than CLI → warning naming the image + both dates + the rebuild command. Explicitly labelled heuristic (an unrelated reinstall trips it) → WARN, never abort. report.sh: - New "Quality tooling (benchlocal-cli + sandboxes)" section: CLI path / version (via the console-script's own interpreter) / install date, the 4 sandbox images with build dates + per-image OLDER-than-CLI flags + rebuild hint, and the newest results/quality/*.json. All best-effort; piped through redact (verified leak-clean live). Exact-version staleness (image labels + a `sandbox-status` command) needs upstream benchlocal-cli changes — tracked in the todo. Verified live: hard-fail branch (docker stub), warn-continue branch, staleness branch (fresh CLI vs real images), report section rendering. test-quality-baseline / test-quality-thinking / test-report-calib green. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01EfF565T9eSLaqGzidyJ1Pm --- scripts/quality-test.sh | 45 ++++++++++++++++++++++++++++++++++- scripts/report.sh | 52 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/scripts/quality-test.sh b/scripts/quality-test.sh index b2c45baf..0e131c8c 100755 --- a/scripts/quality-test.sh +++ b/scripts/quality-test.sh @@ -455,14 +455,48 @@ fi # instead of letting users discover a dead path mid-run (club-3090 #492). if [[ -z "$PACK" ]] && { [[ "$MODE" == "--full" && "$NO_SANDBOX" != "1" ]] || [[ "$SANDBOXED_ONLY" == "1" ]]; }; then _sb_missing=() + _sb_stale=() if ! command -v docker >/dev/null 2>&1; then _sb_missing=("Docker not found on PATH") else + # Install time of the benchlocal-cli console script — rewritten on every + # (re)install, so it's a portable "CLI last updated" timestamp that works + # for pip-from-git AND editable-checkout installs alike. + _bl_bin="$(command -v benchlocal-cli || true)" + _bl_mtime=0 + [[ -n "$_bl_bin" ]] && _bl_mtime="$(stat -c %Y "$_bl_bin" 2>/dev/null || echo 0)" for _img in benchlocal-sandbox-bugfind benchlocal-sandbox-cli benchlocal-sandbox-hermes; do - docker image inspect "${_img}:latest" >/dev/null 2>&1 || _sb_missing+=("${_img}:latest") + if ! docker image inspect "${_img}:latest" >/dev/null 2>&1; then + _sb_missing+=("${_img}:latest") + continue + fi + # Staleness heuristic: the image was built BEFORE the currently-installed + # benchlocal-cli. If that update touched sandbox sources (verifiers, + # harness, deps), results run against the OLD behavior — the exact + # incident class where user rigs kept scoring on pre-fix sandboxes until + # told to rebuild manually. Heuristic (an unrelated reinstall also trips + # it), hence WARN not abort. + _img_created="$(docker image inspect "${_img}:latest" --format '{{.Created}}' 2>/dev/null || true)" + if [[ -n "$_img_created" && "$_bl_mtime" -gt 0 ]]; then + _img_epoch="$(date -d "$_img_created" +%s 2>/dev/null || echo 0)" + if [[ "$_img_epoch" -gt 0 && "$_img_epoch" -lt "$_bl_mtime" ]]; then + _sb_stale+=("${_img}:latest ($(date -d "@${_img_epoch}" +%F) < CLI $(date -d "@${_bl_mtime}" +%F))") + fi + fi done fi if [[ ${#_sb_missing[@]} -gt 0 ]]; then + if [[ "$SANDBOXED_ONLY" == "1" ]]; then + # --sandboxed-only with nothing to run in is a guaranteed-useless run: + # refuse up front instead of warn-and-skip-everything (#492 follow-up). + echo "✗ --sandboxed-only requested but the sandbox prerequisites are missing: ${_sb_missing[*]}" >&2 + echo " The sandbox packs need pre-built Docker images that aren't auto-pulled. Build them once" >&2 + echo " from a benchlocal-cli CHECKOUT (the build tooling isn't in the pip package):" >&2 + echo " git clone https://github.com/noonghunna/benchlocal-cli" >&2 + echo " bash benchlocal-cli/tools/build-sandboxes.sh # ~30 GB free; prune if tight" >&2 + echo " then re-run. For a no-Docker run instead: bash scripts/quality-test.sh --medium" >&2 + exit 1 + fi echo "[quality-test] ⚠ sandbox packs (BugFind / CLI / Hermes) will be SKIPPED — not available: ${_sb_missing[*]}" >&2 echo " They need pre-built Docker images that aren't auto-pulled. Build them once from a" >&2 echo " benchlocal-cli CHECKOUT (the build tooling isn't in the pip package):" >&2 @@ -472,6 +506,15 @@ if [[ -z "$PACK" ]] && { [[ "$MODE" == "--full" && "$NO_SANDBOX" != "1" ]] || [[ echo " (Continuing with the deterministic packs.)" >&2 echo >&2 fi + if [[ ${#_sb_stale[@]} -gt 0 ]]; then + echo "[quality-test] ⚠ sandbox image(s) OLDER than your installed benchlocal-cli:" >&2 + for _s in "${_sb_stale[@]}"; do echo " - ${_s}" >&2; done + echo " If that benchlocal-cli update changed sandbox sources (verifiers/harness)," >&2 + echo " your scores will reflect the OLD sandbox behavior. Rebuild to be safe:" >&2 + echo " bash /tools/build-sandboxes.sh" >&2 + echo " (Heuristic — an unrelated CLI reinstall also trips this. Continuing.)" >&2 + echo >&2 + fi fi # ---- run benchlocal-cli ------------------------------------------------------ diff --git a/scripts/report.sh b/scripts/report.sh index 1f3de4a8..f4359c02 100755 --- a/scripts/report.sh +++ b/scripts/report.sh @@ -555,6 +555,58 @@ if have python3 && [[ -f tools/kv-calc.py ]]; then fi fi +# --------------------------------------------------------------------------- +# Quality tooling (benchlocal-cli + sandboxes) +# --------------------------------------------------------------------------- +# Triage for "my quality run skipped packs / scored weird": is benchlocal-cli +# installed, how fresh, are the sandbox images built, and do they PREDATE the +# CLI (the rebuilt-CLI-stale-sandboxes incident class)? All best-effort — a +# rig without any of this still produces a report. + +section "Quality tooling (benchlocal-cli + sandboxes)" +{ + bl_bin=$(command -v benchlocal-cli 2>/dev/null || true) + if [[ -z "$bl_bin" ]]; then + echo "- **benchlocal-cli:** not installed (quality-test.sh needs it — \`pip install git+https://github.com/noonghunna/benchlocal-cli.git\`)" + else + bl_mtime=$(stat -c %Y "$bl_bin" 2>/dev/null || echo 0) + bl_when=$([[ "$bl_mtime" -gt 0 ]] && date -d "@${bl_mtime}" +%F 2>/dev/null || echo "unknown") + # Version via the CLI's own interpreter (works for pip-from-git AND + # editable-checkout installs; console-script shebang points at the env). + bl_py=$(head -1 "$bl_bin" 2>/dev/null | sed 's/^#!//') + bl_ver=$([[ -x "$bl_py" ]] && "$bl_py" -c 'import importlib.metadata as m; print(m.version("benchlocal-cli"))' 2>/dev/null || true) + echo "- **benchlocal-cli:** \`${bl_bin}\` (version: \`${bl_ver:-unknown}\`, installed/updated: ${bl_when})" + if have docker && docker info >/dev/null 2>&1; then + stale_any=0 + echo "- **Sandbox images** (needed by the --full sandboxed packs):" + for img in benchlocal-sandbox-bugfind benchlocal-sandbox-cli benchlocal-sandbox-hermes benchlocal-sandbox-aider-polyglot; do + created=$(docker image inspect "${img}:latest" --format '{{.Created}}' 2>/dev/null || true) + if [[ -z "$created" ]]; then + echo " - \`${img}\`: ✗ not built" + continue + fi + cdate=$(date -d "$created" +%F 2>/dev/null || echo "$created") + cepoch=$(date -d "$created" +%s 2>/dev/null || echo 0) + mark="" + if [[ "$cepoch" -gt 0 && "$bl_mtime" -gt 0 && "$cepoch" -lt "$bl_mtime" ]]; then + mark=" ⚠ OLDER than the installed CLI — rebuild if the update touched sandbox sources" + stale_any=1 + fi + echo " - \`${img}\`: built ${cdate}${mark}" + done + [[ "$stale_any" == "1" ]] && echo "- **Rebuild:** \`bash /tools/build-sandboxes.sh\` (heuristic — an unrelated reinstall also trips it)" + else + echo "- **Sandbox images:** docker unavailable — cannot inspect (sandboxed packs need Docker)" + fi + fi + latest_q=$(ls -t results/quality/quality-*.json 2>/dev/null | head -1) + if [[ -n "$latest_q" ]]; then + echo "- **Latest quality result:** \`${latest_q}\` ($(date -d "@$(stat -c %Y "$latest_q")" +%F 2>/dev/null || echo '?'))" + else + echo "- **Latest quality result:** none found under results/quality/" + fi +} | redact + # --------------------------------------------------------------------------- # Active container # ---------------------------------------------------------------------------