club-3090/scripts/rerun-failed-packs.sh

#!/usr/bin/env bash
#
# rerun-failed-packs.sh — re-test ONLY the scenarios that failed in a prior
# quality run, and report which failures reproduced vs flipped (flakes).
#
# Since benchlocal-cli #84 (scenario-level selection) this runs the failed
# scenarios as ONE selection run instead of re-running each affected pack —
# e.g. 6 failures spread over 5 packs = 6 scenarios, not ~100. (The original
# pack-level version of this script predated #84; the name is kept for
# muscle-memory/back-compat.)
#
# What it does:
#   1. Parses a saved RunResult JSON: failed scenarios -> PACK_ID/SCENARIO_ID
#      selections (written to a temp scenarios-file).
#   2. Re-runs them in a single quality-test.sh invocation (keeping its
#      hermes-env + timeout guards), with --previous-result so benchlocal
#      emits its per-scenario delta gated on exactly the selected keys, and
#      --incremental so an interrupted re-run is resumable. Thinking mode is
#      taken from the ORIGINAL run's JSON (thinking_enabled) — nothing to get
#      wrong.
#   3. Prints a consolidated verdict per original failure:
#      REPRODUCED (real) / FIXED (flake or environment).
#
# Usage:
#   bash scripts/rerun-failed-packs.sh <result.json> [--repeat N] [extra quality-test.sh args...]
#
#   URL= / MODEL= env override the endpoint (default: autodetect via
#   quality-test.sh, same as any other run). --repeat N re-runs each scenario
#   N times (benchlocal aggregates at >=50% pass) — use N>=3 for flakiness
#   stats; at scenario granularity that's cheap.
#
#   RERUN_DRY=1  print the plan (selection + mode + command) without running.
#
# Output JSON lands next to the original as <original>.rerun.json (a PARTIAL
# selection result — never a canonical pack total).
set -euo pipefail

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(cd -- "${SCRIPT_DIR}/.." && pwd)"

RESULT_JSON="${1:-}"
if [[ -z "$RESULT_JSON" || "$RESULT_JSON" == "-h" || "$RESULT_JSON" == "--help" ]]; then
  sed -n '2,35p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
  exit 2
fi
if [[ ! -f "$RESULT_JSON" ]]; then
  echo "✗ result JSON not found: $RESULT_JSON" >&2
  echo "  Fix: pass a saved RunResult (e.g. results/quality/quality-<ts>.json)" >&2
  exit 2
fi
shift

# --- 1. plan: failed scenario selections + thinking mode from the original ----
PLAN="$(python3 - "$RESULT_JSON" <<'PY'
import json, sys
d = json.load(open(sys.argv[1], encoding="utf-8"))
selections = []
failures = []
for p in d.get("packs", []):
    pid = p.get("pack_id", "?")
    for s in p.get("scenarios", []):
        if s.get("passed") is not True:
            selections.append(f"{pid}/{s.get('id','?')}")
            failures.append(f"{pid}/{s.get('id','?')}:{s.get('failure_mode','fail')}")
mode = "--enable-thinking" if d.get("thinking_enabled") else "--no-thinking"
print(mode)
print(" ".join(selections))
print(" ".join(failures))
PY
)"
MODE_FLAG="$(sed -n '1p' <<<"$PLAN")"
SELECTIONS="$(sed -n '2p' <<<"$PLAN")"
ORIG_FAILURES="$(sed -n '3p' <<<"$PLAN")"

if [[ -z "$SELECTIONS" ]]; then
  echo "✓ no failed scenarios in $RESULT_JSON — nothing to re-run."
  exit 0
fi
N_SEL="$(wc -w <<<"$SELECTIONS" | tr -d ' ')"

echo "[rerun] original run: $RESULT_JSON  (mode: ${MODE_FLAG#--})"
echo "[rerun] failed scenarios (${N_SEL}): $SELECTIONS"

SEL_FILE="$(mktemp --suffix=.rerun-scenarios.txt)"
trap 'rm -f "$SEL_FILE"' EXIT
{
  echo "# auto-generated by rerun-failed-packs.sh from $RESULT_JSON"
  tr ' ' '\n' <<<"$SELECTIONS"
} > "$SEL_FILE"

RERUN_JSON="${RESULT_JSON}.rerun.json"

if [[ "${RERUN_DRY:-0}" == "1" ]]; then
  echo "[dry] selection file (${SEL_FILE}):"
  sed 's/^/[dry]   /' "$SEL_FILE"
  echo "[dry] quality-test.sh --scenarios-file $SEL_FILE $MODE_FLAG --previous-result $RESULT_JSON --incremental --save-json $RERUN_JSON $*"
  exit 0
fi

# --- 2. one selection re-run via the wrapper -----------------------------------
bash "${SCRIPT_DIR}/quality-test.sh" --scenarios-file "$SEL_FILE" "$MODE_FLAG" \
  --previous-result "$RESULT_JSON" \
  --incremental \
  --save-json "$RERUN_JSON" \
  "$@"

# --- 3. consolidated verdict ----------------------------------------------------
python3 - "$RESULT_JSON" "$RERUN_JSON" <<'PY'
import json, sys
orig_path, rerun_path = sys.argv[1], sys.argv[2]
orig = json.load(open(orig_path, encoding="utf-8"))
orig_state = {}
for p in orig.get("packs", []):
    for s in p.get("scenarios", []):
        orig_state[(p["pack_id"], s["id"])] = s.get("passed") is True

try:
    rr = json.load(open(rerun_path, encoding="utf-8"))
except FileNotFoundError:
    print(f"✗ no rerun JSON at {rerun_path} — the selection run did not complete", file=sys.stderr)
    sys.exit(1)

repro, fixed = [], []
for p in rr.get("packs", []):
    for s in p.get("scenarios", []):
        key = (p["pack_id"], s["id"])
        tag = f"{key[0]}/{key[1]}"
        if s.get("passed") is True:
            fixed.append(tag)
        else:
            repro.append(f"{tag} ({s.get('failure_mode','fail')})")

print("\n════ rerun verdict ════")
print(f"REPRODUCED ({len(repro)}) — likely real:")
for t in repro: print(f"  ✗ {t}")
print(f"FIXED on re-run ({len(fixed)}) — flake / environment:")
for t in fixed: print(f"  ↺ {t}")
print("\nNote: single re-run separates 'stable' from 'flaky', not 'model' from")
print("'harness'. For flakiness RATES, add --repeat 3 (cheap at scenario")
print("granularity) and read benchlocal's >=50% aggregation in the delta output.")
print("The rerun JSON is a PARTIAL selection result — not a pack total.")
PY