f6a344cf4f
The metadata version is frozen at install time and benchlocal fixes are pushed without bumping it, so "0.9.4" cannot identify the running code (live proof on the dev rig: metadata 0.9.4, checkout actually at v0.9.7-6-gc541550). pip records the truth in direct_url.json: - report.sh: the Quality-tooling line now appends `source:` — `git@<sha9>` for pip-from-git installs (vcs_info.commit_id), or `git describe --tags --always --dirty` for editable checkouts (the checkout PATH is withheld from the public report). - quality-test.sh staleness reference: editable installs update via `git pull` WITHOUT rewriting the console script, so the mtime under-reports "CLI last updated" — now max(script mtime, editable checkout last-commit time). Caught a real case on the dev rig: the cli/hermes sandboxes were built ~1 h before the same day's last benchlocal commit. Stale-warning timestamps now carry %H:%M so same-day boundaries read sensibly. test-quality-baseline / test-quality-thinking / test-report-calib green; report section verified live + leak-clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EfF565T9eSLaqGzidyJ1Pm
1056 lines
48 KiB
Bash
Executable File
1056 lines
48 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# scripts/report.sh — paste-ready triage report for club-3090
|
||
#
|
||
# Run when filing a bug report, sharing cross-rig benchmark data, or replying
|
||
# to a triage thread. Captures hardware, OS, GPU, container runtime, stack
|
||
# version, and active container state in markdown ready to paste into a GitHub
|
||
# issue or discussion.
|
||
#
|
||
# Usage:
|
||
# bash scripts/report.sh # default: hardware + stack + boot log highlights (~2 sec)
|
||
# bash scripts/report.sh --verify # adds verify-full.sh output (~1-2 min)
|
||
# bash scripts/report.sh --stress # adds verify-stress.sh 7/7 output (~5-10 min)
|
||
# bash scripts/report.sh --soak # adds SOAK_MODE=continuous summary (~25 min) — catches Cliff 2b
|
||
# bash scripts/report.sh --bench # adds bench.sh output (~3 min)
|
||
# bash scripts/report.sh --agentic # adds bench-agentic.sh curve-shape output (~8 min estimate)
|
||
# bash scripts/report.sh --full # ALL five: verify + stress + soak + bench + agentic (~43 min estimate, the canonical "everything" pass for cross-rig contributions)
|
||
# bash scripts/report.sh --studio # adds AI Studio container log tails (ComfyUI + director + …) — for image/video/audio generation bugs (~2 sec)
|
||
# bash scripts/report.sh --no-redact # disable path/host/user redaction
|
||
# bash scripts/report.sh --container NAME # override container auto-detection
|
||
# bash scripts/report.sh --full-calibration # kv-calc matrix for ALL models (default: only the running model; skipped on llama.cpp/ik_llama)
|
||
# bash scripts/report.sh > my-rig.md # capture for paste
|
||
#
|
||
# Why --soak is its own flag:
|
||
# verify-full + verify-stress + bench all PASS on configs that FAIL the
|
||
# multi-turn continuous soak (Cliff 2b at ~25K accumulated tokens). Until
|
||
# the upstream fix lands, soak is the only test that catches the agentic-
|
||
# workload failure mode. See docs/CLIFFS.md.
|
||
#
|
||
# By default, paths under user homes, hostnames, usernames, and HF tokens are
|
||
# redacted. Use --no-redact for internal sharing only.
|
||
|
||
set -uo pipefail
|
||
|
||
DO_VERIFY=0
|
||
DO_STRESS=0
|
||
DO_SOAK=0
|
||
DO_BENCH=0
|
||
DO_AGENTIC=0
|
||
DO_STUDIO=0
|
||
REDACT=1
|
||
CONTAINER=""
|
||
# KV-calc calibration is scoped to the running model by default (#168). Set to 1
|
||
# (flag or REPORT_FULL_CALIBRATION=1) to emit the full catalog-wide matrix.
|
||
FULL_CALIBRATION="${REPORT_FULL_CALIBRATION:-0}"
|
||
|
||
print_help() {
|
||
sed -n '2,/^set/p' "$0" | sed 's/^# \?//' | head -n -1
|
||
}
|
||
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
--verify) DO_VERIFY=1; shift ;;
|
||
--stress) DO_STRESS=1; shift ;;
|
||
--soak) DO_SOAK=1; shift ;;
|
||
--bench) DO_BENCH=1; shift ;;
|
||
--agentic) DO_AGENTIC=1; shift ;;
|
||
--studio) DO_STUDIO=1; shift ;;
|
||
--full) DO_VERIFY=1; DO_STRESS=1; DO_SOAK=1; DO_BENCH=1; DO_AGENTIC=1; shift ;;
|
||
--no-redact) REDACT=0; shift ;;
|
||
--container) CONTAINER="${2:-}"; shift 2 ;;
|
||
--full-calibration) FULL_CALIBRATION=1; shift ;;
|
||
-h|--help) print_help; exit 0 ;;
|
||
*) echo "Unknown arg: $1" >&2; echo "Try: bash scripts/report.sh --help" >&2; exit 1 ;;
|
||
esac
|
||
done
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||
cd "$REPO_ROOT"
|
||
|
||
# KV-calc calibration helpers (engine/model detection + per-model filter, #168).
|
||
source "$REPO_ROOT/scripts/lib/report_calib.sh"
|
||
# shellcheck source=lib/p2p-state.sh
|
||
source "$REPO_ROOT/scripts/lib/p2p-state.sh"
|
||
|
||
# Pick up a saved MODEL_DIR (and other config) from the repo .env — same as
|
||
# launch.sh / switch.sh, and what setup.sh writes there. An explicit exported
|
||
# MODEL_DIR still wins. This makes the Disk section report the user's real
|
||
# models path instead of falling back to the hardcoded mount below.
|
||
if [[ -z "${MODEL_DIR:-}" && -f "${REPO_ROOT}/.env" ]]; then
|
||
# shellcheck disable=SC1091
|
||
source "${REPO_ROOT}/.env"
|
||
fi
|
||
|
||
HOST_SHORT="$(hostname -s 2>/dev/null || echo unknown)"
|
||
USER_NAME="${USER:-$(whoami 2>/dev/null || echo unknown)}"
|
||
|
||
redact() {
|
||
if [[ $REDACT -eq 1 ]]; then
|
||
# Mask the literal MODEL_DIR value first (if exported) so an arbitrary models
|
||
# path — /data/..., /srv/... — is caught before the prefix rules below.
|
||
{ if [[ -n "${MODEL_DIR:-}" ]]; then sed -e "s|${MODEL_DIR}|<MODEL_DIR>|g"; else cat; fi; } | sed \
|
||
-e "s|/home/${USER_NAME}|~|g" \
|
||
-e "s|/root|~|g" \
|
||
-e "s|${HOST_SHORT}|<HOST>|g" \
|
||
-e "s|${USER_NAME}|<USER>|g" \
|
||
-e 's|HF_TOKEN=[^ "]*|HF_TOKEN=<REDACTED>|g' \
|
||
-e 's|HUGGING_FACE_HUB_TOKEN=[^ "]*|HUGGING_FACE_HUB_TOKEN=<REDACTED>|g' \
|
||
-e 's|api_key=[^ "]*|api_key=<REDACTED>|gi' \
|
||
-e 's|hf_[A-Za-z0-9]\{30,\}|hf_<REDACTED>|g' \
|
||
-e 's|/opt/ai|<STACK_ROOT>|g' \
|
||
-e 's|/mnt/[a-z]/Users/[^ /]*|/mnt/<DRIVE>/Users/<REDACTED>|g' \
|
||
-e 's|/mnt/models|<MODELS>|g'
|
||
else
|
||
cat
|
||
fi
|
||
}
|
||
|
||
section() { printf '\n## %s\n\n' "$1"; }
|
||
subsection() { printf '\n### %s\n\n' "$1"; }
|
||
|
||
details() {
|
||
local summary="$1"
|
||
printf '<details><summary>%s</summary>\n\n```\n' "$summary"
|
||
cat
|
||
printf '```\n\n</details>\n'
|
||
}
|
||
|
||
have() { command -v "$1" >/dev/null 2>&1; }
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Header
|
||
# ---------------------------------------------------------------------------
|
||
|
||
cat <<EOF
|
||
# club-3090 rig report
|
||
|
||
Generated: $(date -u +'%Y-%m-%d %H:%M:%S UTC')
|
||
EOF
|
||
|
||
if [[ $REDACT -eq 1 ]]; then
|
||
printf '\n_Redacted output (paths, host, user, tokens). Re-run with `--no-redact` for full data._\n'
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# System
|
||
# ---------------------------------------------------------------------------
|
||
|
||
section "System"
|
||
{
|
||
os_name="unknown"
|
||
if [[ -r /etc/os-release ]]; then
|
||
# shellcheck disable=SC1091
|
||
. /etc/os-release
|
||
os_name="${PRETTY_NAME:-${NAME:-unknown}}"
|
||
fi
|
||
echo "- **OS:** $os_name"
|
||
echo "- **Kernel:** $(uname -r)"
|
||
|
||
# Environment detection
|
||
env_kind="bare metal"
|
||
if grep -qiE 'microsoft|wsl' /proc/version 2>/dev/null; then
|
||
env_kind="WSL2"
|
||
if grep -qE 'WSL2' /proc/version 2>/dev/null; then
|
||
env_kind="WSL2 (kernel reports WSL2)"
|
||
fi
|
||
elif have systemd-detect-virt && [[ "$(systemd-detect-virt 2>/dev/null)" != "none" ]]; then
|
||
env_kind="$(systemd-detect-virt 2>/dev/null) (virtualized)"
|
||
elif [[ -r /.dockerenv ]]; then
|
||
env_kind="inside-container (unusual for this script)"
|
||
fi
|
||
echo "- **Environment:** $env_kind"
|
||
|
||
echo "- **Locale:** ${LANG:-unset}"
|
||
echo "- **Timezone:** $(date +%Z)"
|
||
echo "- **Uptime:** $(uptime -p 2>/dev/null || echo unknown)"
|
||
} | redact
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# CPU + RAM
|
||
# ---------------------------------------------------------------------------
|
||
|
||
section "CPU + RAM"
|
||
{
|
||
if have lscpu; then
|
||
cpu_model=$(lscpu 2>/dev/null | awk -F: '/Model name/ {sub(/^ */, "", $2); print $2; exit}')
|
||
cpu_cores=$(lscpu 2>/dev/null | awk -F: '/^CPU\(s\):/ {gsub(/ /, "", $2); print $2; exit}')
|
||
echo "- **CPU:** ${cpu_model:-unknown} (${cpu_cores:-?} threads)"
|
||
else
|
||
echo "- **CPU:** lscpu not available"
|
||
fi
|
||
|
||
if have free; then
|
||
ram_total=$(free -h 2>/dev/null | awk '/^Mem:/ {print $2}')
|
||
ram_avail=$(free -h 2>/dev/null | awk '/^Mem:/ {print $7}')
|
||
echo "- **RAM:** ${ram_total} total, ${ram_avail} available"
|
||
swap_total=$(free -h 2>/dev/null | awk '/^Swap:/ {print $2}')
|
||
[[ "$swap_total" != "0B" && -n "$swap_total" ]] && echo "- **Swap:** $swap_total"
|
||
fi
|
||
} | redact
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Disk
|
||
# ---------------------------------------------------------------------------
|
||
|
||
section "Disk"
|
||
{
|
||
declare -a checked_paths=()
|
||
add_disk_row() {
|
||
local p="$1"
|
||
[[ -z "$p" || ! -d "$p" ]] && return
|
||
for seen in "${checked_paths[@]:-}"; do
|
||
[[ "$seen" == "$p" ]] && return
|
||
done
|
||
checked_paths+=("$p")
|
||
local fs avail
|
||
fs=$(df -T "$p" 2>/dev/null | awk 'NR==2 {print $2}')
|
||
avail=$(df -h "$p" 2>/dev/null | awk 'NR==2 {print $4}')
|
||
echo "- **$p:** ${avail:-?} available, ${fs:-?} filesystem"
|
||
}
|
||
|
||
add_disk_row "${MODEL_DIR:-}"
|
||
add_disk_row "$REPO_ROOT/models-cache"
|
||
add_disk_row "/mnt/models/huggingface"
|
||
|
||
if have docker && docker info >/dev/null 2>&1; then
|
||
docker_root=$(docker info --format '{{.DockerRootDir}}' 2>/dev/null)
|
||
[[ -n "$docker_root" ]] && add_disk_row "$docker_root"
|
||
fi
|
||
} | redact
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# GPU hardware
|
||
# ---------------------------------------------------------------------------
|
||
|
||
section "GPU hardware"
|
||
if ! have nvidia-smi; then
|
||
echo "_nvidia-smi not available — no NVIDIA GPU detected or driver not installed_"
|
||
else
|
||
{
|
||
nvidia-smi --query-gpu=index,name,memory.total,driver_version,vbios_version,persistence_mode,power.limit,power.default_limit,power.max_limit,power.draw,pci.bus_id,pcie.link.gen.current,pcie.link.gen.max,pcie.link.width.current,pcie.link.width.max \
|
||
--format=csv,noheader 2>/dev/null \
|
||
| while IFS=, read -r idx name memtotal driver vbios persistence pwr_limit pwr_default pwr_max pwr_draw bus_id pcie_gen_cur pcie_gen_max pcie_width_cur pcie_width_max; do
|
||
# trim leading spaces from CSV fields
|
||
idx="${idx# }"; name="${name# }"; memtotal="${memtotal# }"
|
||
driver="${driver# }"; vbios="${vbios# }"; persistence="${persistence# }"
|
||
pwr_limit="${pwr_limit# }"; pwr_default="${pwr_default# }"
|
||
pwr_max="${pwr_max# }"; pwr_draw="${pwr_draw# }"
|
||
bus_id="${bus_id# }"; pcie_gen_cur="${pcie_gen_cur# }"; pcie_gen_max="${pcie_gen_max# }"
|
||
pcie_width_cur="${pcie_width_cur# }"; pcie_width_max="${pcie_width_max# }"
|
||
|
||
# Flag if user has capped below default
|
||
power_note=""
|
||
pwr_limit_w="${pwr_limit% W}"; pwr_limit_w="${pwr_limit_w%.*}"
|
||
pwr_default_w="${pwr_default% W}"; pwr_default_w="${pwr_default_w%.*}"
|
||
if [[ "$pwr_limit_w" =~ ^[0-9]+$ ]] && [[ "$pwr_default_w" =~ ^[0-9]+$ ]]; then
|
||
if [[ "$pwr_limit_w" -lt "$pwr_default_w" ]]; then
|
||
power_note=" ⚠ user-capped below default"
|
||
elif [[ "$pwr_limit_w" -gt "$pwr_default_w" ]]; then
|
||
power_note=" (overclocked above default)"
|
||
fi
|
||
fi
|
||
|
||
# Flag if PCIe lane width is below max — that's hardware-level (slot
|
||
# has fewer lanes wired, riser cables, BIOS bifurcation, etc.) and
|
||
# affects model load speed + per-card all-reduce bandwidth.
|
||
# NOTE: pcie.link.gen.current drops to Gen 1 at idle for power
|
||
# saving — that's normal, not a degradation. Re-check under load if
|
||
# you want the actual negotiated gen. Width is hardware-fixed.
|
||
pcie_note=""
|
||
if [[ -n "$pcie_width_cur" && -n "$pcie_width_max" && "$pcie_width_cur" != "$pcie_width_max" ]]; then
|
||
pcie_note=" ⚠ slot is narrower than GPU capability — affects load + all-reduce bandwidth"
|
||
fi
|
||
|
||
echo "- **GPU $idx:** $name | $memtotal | driver $driver | VBIOS $vbios | persistence=$persistence"
|
||
echo " - **Power:** limit=${pwr_limit} (default=${pwr_default}, max=${pwr_max}) | current_draw=${pwr_draw}${power_note}"
|
||
echo " - **PCIe:** x${pcie_width_cur} lanes negotiated (GPU max x${pcie_width_max}, Gen up to ${pcie_gen_max}) | bus ${bus_id}${pcie_note}"
|
||
done
|
||
|
||
cuda_ver=$(nvidia-smi 2>/dev/null | grep -oE 'CUDA Version: [0-9.]+' | head -1 | awk '{print $3}')
|
||
[[ -n "$cuda_ver" ]] && echo "- **CUDA Runtime (per driver):** $cuda_ver"
|
||
|
||
# Persistence mode + ECC summary
|
||
ecc_status=$(nvidia-smi --query-gpu=ecc.mode.current --format=csv,noheader 2>/dev/null | head -1 | tr -d ' ')
|
||
[[ -n "$ecc_status" ]] && echo "- **ECC mode:** $ecc_status (3090s don't have ECC; expect N/A)"
|
||
} | redact
|
||
|
||
subsection "NVLink"
|
||
if nvidia-smi nvlink --status -i 0 2>/dev/null | grep -qE 'Link [0-9]+:'; then
|
||
nvidia-smi nvlink --status 2>&1 | redact | details "NVLink link status"
|
||
else
|
||
echo "_No NVLink detected (PCIe-only)_"
|
||
fi
|
||
|
||
subsection "Topology"
|
||
nvidia-smi topo -m 2>&1 | redact | details "PCIe / GPU topology matrix"
|
||
|
||
# lspci-based PCIe/P2P detail. nvidia-smi reports negotiated gen/width but
|
||
# cannot show trained link state vs capability side-by-side, ACS state on the
|
||
# upstream bridge, or the real PCIe topology tree — the three things that
|
||
# actually decide whether GPU↔GPU P2P engages (see issues #137, #351).
|
||
subsection "PCIe / P2P detail (lspci)"
|
||
if ! have lspci; then
|
||
# Fallback: nvidia-smi topo -p2p doesn't need pciutils and shows P2P capability
|
||
if have nvidia-smi && nvidia-smi topo -p2p rw >/dev/null 2>&1; then
|
||
echo "_lspci not available (pciutils not installed) — showing P2P capability matrix instead._"
|
||
echo
|
||
nvidia-smi topo -p2p rw | redact
|
||
else
|
||
echo "_lspci not available (pciutils not installed) — skipping PCIe/P2P detail._"
|
||
fi
|
||
else
|
||
# sudo lspci -vvv is needed for full capability blocks (ACS lives in the
|
||
# extended config space, root-only). Degrade gracefully if sudo is
|
||
# unavailable / non-interactive — non-sudo lspci still shows LnkSta.
|
||
LSPCI_CMD=(lspci)
|
||
SUDO_NOTE=""
|
||
if [[ $EUID -ne 0 ]]; then
|
||
if have sudo && sudo -n true 2>/dev/null; then
|
||
LSPCI_CMD=(sudo lspci)
|
||
else
|
||
SUDO_NOTE="_Note: sudo unavailable/non-interactive — running lspci without root; ACS capability lines may be incomplete (LnkSta still accurate)._"
|
||
fi
|
||
fi
|
||
|
||
{
|
||
[[ -n "$SUDO_NOTE" ]] && { echo "$SUDO_NOTE"; echo; }
|
||
|
||
echo "# lspci -t (PCIe topology tree)"
|
||
lspci -t 2>&1
|
||
echo
|
||
|
||
# Per NVIDIA VGA / 3D-controller function: trained link state vs
|
||
# capability + ACS state. Filter to the four load-bearing lines only —
|
||
# never dump the full -vvv block (keeps the report compact + redaction-safe).
|
||
# ACS (ACSCap/ACSCtl) lives on the UPSTREAM PCIe port, not the GPU
|
||
# endpoint — and ACS-redirect on that bridge is exactly what blocks P2P
|
||
# (issues #137, #351) — so for each GPU we also dump its upstream bridge.
|
||
dump_func() {
|
||
local slot="$1" label="$2"
|
||
echo "# lspci -vvv -s ${slot} (${label}: LnkCap/LnkSta/ACSCap/ACSCtl)"
|
||
"${LSPCI_CMD[@]}" -vvv -s "$slot" 2>/dev/null \
|
||
| grep -E '^[[:space:]]*(LnkCap|LnkSta|ACSCap|ACSCtl):' \
|
||
|| echo " (no matching LnkCap/LnkSta/ACSCap/ACSCtl lines)"
|
||
echo
|
||
}
|
||
while read -r slot _; do
|
||
[[ -z "$slot" ]] && continue
|
||
dump_func "$slot" "GPU function"
|
||
# Resolve the upstream bridge via sysfs (../.. of the device node).
|
||
bridge=""
|
||
if [[ -e "/sys/bus/pci/devices/${slot}" ]]; then
|
||
bridge="$(basename "$(readlink -f "/sys/bus/pci/devices/${slot}/../" 2>/dev/null)" 2>/dev/null)"
|
||
fi
|
||
if [[ "$bridge" =~ ^[0-9a-fA-F]{4}: ]]; then
|
||
dump_func "$bridge" "upstream bridge of ${slot}"
|
||
else
|
||
echo " (could not resolve upstream bridge for ${slot} — ACS state for P2P may be elsewhere in the tree)"
|
||
echo
|
||
fi
|
||
done < <(lspci -D 2>/dev/null | grep -iE 'VGA compatible controller.*NVIDIA|3D controller.*NVIDIA')
|
||
|
||
echo "# lspci -nnk | grep -A3 -i nvidia (driver binding + device IDs)"
|
||
lspci -nnk 2>/dev/null | grep -A3 -i nvidia 2>/dev/null \
|
||
|| echo " (no NVIDIA functions found)"
|
||
} 2>&1 | redact | details "lspci PCIe/P2P detail (LnkSta / ACS / topology)"
|
||
fi
|
||
|
||
subsection "Full nvidia-smi"
|
||
nvidia-smi 2>&1 | redact | details "Full nvidia-smi output"
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Display / desktop state
|
||
# ---------------------------------------------------------------------------
|
||
|
||
section "Display / desktop state"
|
||
{
|
||
if [[ -n "${DISPLAY:-}" ]]; then
|
||
echo "- **\$DISPLAY:** ${DISPLAY} (X11 / Wayland session present)"
|
||
else
|
||
echo "- **\$DISPLAY:** unset (headless)"
|
||
fi
|
||
[[ -n "${WAYLAND_DISPLAY:-}" ]] && echo "- **\$WAYLAND_DISPLAY:** ${WAYLAND_DISPLAY}"
|
||
|
||
compositor=""
|
||
for proc in Xorg Xwayland weston gnome-shell kwin sway hyprland mutter; do
|
||
if pgrep -x "$proc" >/dev/null 2>&1; then
|
||
compositor="$compositor $proc"
|
||
fi
|
||
done
|
||
if [[ -n "$compositor" ]]; then
|
||
echo "- **Display processes running:**$compositor"
|
||
else
|
||
echo "- **Display processes running:** none detected"
|
||
fi
|
||
|
||
if have nvidia-smi; then
|
||
# Check if a club-3090 container is running (lightweight — full detection is later)
|
||
# NB: top-level (not in a function) — plain assignment, not `local`.
|
||
our_container=""
|
||
if have docker && docker info >/dev/null 2>&1; then
|
||
our_container=$(docker ps --format '{{.Names}}' --filter 'name=vllm-' --filter 'name=llama-cpp-' --filter 'name=beellama-' --filter 'name=club3090-' --filter 'name=ik-llama-' 2>/dev/null | head -1)
|
||
fi
|
||
nvidia-smi --query-gpu=index,memory.used --format=csv,noheader,nounits 2>/dev/null \
|
||
| while IFS=, read -r idx used; do
|
||
idx="${idx# }"; used="${used# }"
|
||
if [[ "$used" =~ ^[0-9]+$ ]] && [[ "$used" -gt 100 ]]; then
|
||
if [[ -n "$our_container" ]]; then
|
||
echo "- **GPU $idx idle VRAM:** ${used} MiB (held by running \`${our_container}\`)"
|
||
else
|
||
echo "- **GPU $idx idle VRAM:** ${used} MiB ⚠ something is using this GPU (display, browser, container)"
|
||
fi
|
||
else
|
||
echo "- **GPU $idx idle VRAM:** ${used} MiB ✓"
|
||
fi
|
||
done
|
||
fi
|
||
} | redact
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Container runtime
|
||
# ---------------------------------------------------------------------------
|
||
|
||
section "Container runtime"
|
||
{
|
||
if have docker; then
|
||
if docker info >/dev/null 2>&1; then
|
||
docker_ver=$(docker version --format '{{.Server.Version}}' 2>/dev/null)
|
||
echo "- **Docker:** ${docker_ver:-unknown}"
|
||
|
||
if docker compose version >/dev/null 2>&1; then
|
||
compose_ver=$(docker compose version --short 2>/dev/null)
|
||
echo "- **docker compose (v2):** ${compose_ver:-unknown}"
|
||
elif have docker-compose; then
|
||
compose_ver=$(docker-compose version --short 2>/dev/null)
|
||
echo "- **docker-compose (v1):** ${compose_ver:-unknown}"
|
||
fi
|
||
|
||
if have nvidia-ctk; then
|
||
nvct_ver=$(nvidia-ctk --version 2>&1 | head -1 | awk '{print $NF}')
|
||
echo "- **NVIDIA Container Toolkit:** ${nvct_ver:-unknown}"
|
||
elif have nvidia-container-toolkit; then
|
||
nvct_ver=$(nvidia-container-toolkit --version 2>&1 | head -1 | awk '{print $NF}')
|
||
echo "- **NVIDIA Container Toolkit:** ${nvct_ver:-unknown}"
|
||
fi
|
||
else
|
||
echo "- **Docker:** installed but daemon not accessible"
|
||
fi
|
||
else
|
||
echo "- **Docker:** not installed"
|
||
fi
|
||
} | redact
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Stack version
|
||
# ---------------------------------------------------------------------------
|
||
|
||
section "Stack version"
|
||
{
|
||
if [[ -d .git ]]; then
|
||
# Prefer `git describe` for a human-readable version (e.g. v0.6.2-3-ge299e70,
|
||
# "3 commits past v0.6.2 at SHA e299e70"). Falls back to raw SHA if no tags
|
||
# are reachable (shallow clone, fresh repo).
|
||
version=$(git describe --tags --always --dirty 2>/dev/null)
|
||
commit=$(git rev-parse --short HEAD 2>/dev/null)
|
||
branch=$(git branch --show-current 2>/dev/null)
|
||
echo "- **club-3090:** \`${version:-${commit:-unknown}}\` (branch: \`${branch:-detached}\`, SHA \`${commit:-unknown}\`)"
|
||
if ! git diff --quiet 2>/dev/null || ! git diff --cached --quiet 2>/dev/null; then
|
||
echo "- **Working tree:** ⚠ has uncommitted changes (run \`git status\` to inspect)"
|
||
fi
|
||
else
|
||
echo "- **club-3090:** not a git repo"
|
||
fi
|
||
|
||
if [[ -f scripts/setup.sh ]]; then
|
||
# Parse `GENESIS_PIN="${GENESIS_PIN:-<default>}"` — extract just the default value
|
||
genesis_pin=$(grep -E '^GENESIS_PIN=' scripts/setup.sh 2>/dev/null | head -1 \
|
||
| sed -E 's/.*:-([^}]+)\}.*/\1/; t; s/.*=//' \
|
||
| tr -d '"' | tr -d "'")
|
||
[[ -n "$genesis_pin" ]] && echo "- **GENESIS_PIN default:** \`$genesis_pin\` (per scripts/setup.sh)"
|
||
# Override from env if set
|
||
[[ -n "${GENESIS_PIN:-}" ]] && echo "- **GENESIS_PIN env override:** \`$GENESIS_PIN\`"
|
||
fi
|
||
|
||
if have docker && docker info >/dev/null 2>&1; then
|
||
cached=$(docker images vllm/vllm-openai --format '{{.Tag}} {{.Digest}} {{.CreatedSince}}' 2>/dev/null | head -3)
|
||
if [[ -n "$cached" ]]; then
|
||
echo "- **Cached vLLM images:**"
|
||
echo "$cached" | while read -r tag digest age rest; do
|
||
echo " - tag \`$tag\` digest \`$digest\` ($age $rest)"
|
||
done
|
||
fi
|
||
fi
|
||
} | redact
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Profile state
|
||
# ---------------------------------------------------------------------------
|
||
|
||
if [[ -x scripts/lib/profiles/estate_cli.py || -f scripts/lib/profiles/estate_cli.py ]]; then
|
||
python3 scripts/lib/profiles/estate_cli.py report-state 2>&1 | redact || true
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# KV math calibration
|
||
# ---------------------------------------------------------------------------
|
||
# When a user files a VRAM-OOM or context-ceiling bug, the maintainer's first
|
||
# question is "does kv-calc still agree with measured reality?" — a calibration
|
||
# failure means the projection model has drifted from the actual VRAM cost of a
|
||
# compose, so any "predicted PASS" verdict can't be trusted. Surface the
|
||
# verdict line + any FAIL rows here so a triage reply can immediately see
|
||
# whether to trust kv-calc projections for this user's config.
|
||
|
||
# Engine + model detection for kv-calc scoping (#168). Resolve the active
|
||
# container (explicit --container wins; else first running club-3090 container),
|
||
# then map it to a kv-calc engine family + model id via scripts/lib/report_calib.sh.
|
||
_calib_container="${CONTAINER:-}"
|
||
if [[ -z "$_calib_container" ]] && have docker && docker info >/dev/null 2>&1; then
|
||
_calib_container=$(docker ps --format '{{.Names}}' --filter 'name=vllm-' --filter 'name=llama-cpp-' --filter 'name=beellama-' --filter 'name=club3090-' --filter 'name=ik-llama-' 2>/dev/null | head -1)
|
||
fi
|
||
CALIB_ENGINE_KIND="${ENGINE_KIND:-$(calib_engine_for_container "$_calib_container")}"
|
||
CALIB_MODEL_ID="$(calib_model_for_container "$_calib_container")"
|
||
|
||
if have python3 && [[ -f tools/kv-calc.py ]]; then
|
||
section "KV math calibration"
|
||
# kv-calc is vLLM-memory-model-coupled; skip on the ggml engines (llama.cpp + ik_llama).
|
||
if [[ "$CALIB_ENGINE_KIND" == "llamacpp" ]]; then
|
||
echo "- _kv-calc calibration is vLLM-specific — skipped on the llama.cpp / ik_llama engine (ggml uses a different allocator)._"
|
||
elif ! python3 -c 'import yaml' 2>/dev/null; then
|
||
# Item 1: graceful-degrade when PyYAML is missing
|
||
echo "- _kv-calc calibration skipped — PyYAML not installed (\`pip install pyyaml\`)._"
|
||
else
|
||
# #168: scope to the running model by default; --full-calibration (or
|
||
# REPORT_FULL_CALIBRATION=1) restores the catalog-wide matrix. Falls back to
|
||
# the full matrix when the model can't be resolved.
|
||
calib_scope=""
|
||
if [[ "$FULL_CALIBRATION" != "1" && -n "$CALIB_MODEL_ID" ]]; then
|
||
calib_scope="$CALIB_MODEL_ID"
|
||
echo "- _Scoped to the running model \`${CALIB_MODEL_ID}\` — pass \`--full-calibration\` for all calibrated models._"
|
||
fi
|
||
calib_output=$(python3 tools/kv-calc.py --calibration 2>&1 | calib_filter_model_section "$calib_scope" || true)
|
||
overall=$(echo "$calib_output" | grep -E '^Overall:' | head -1)
|
||
fail_rows=$(echo "$calib_output" | grep -E '\bFAIL\b' || true)
|
||
{
|
||
if [[ -n "$overall" ]]; then
|
||
echo "- ${overall}"
|
||
else
|
||
echo "- _kv-calc --calibration produced no Overall line; see output below._"
|
||
fi
|
||
if [[ -n "$fail_rows" ]]; then
|
||
echo "- ⚠ Failing rows:"
|
||
echo '```'
|
||
echo "$fail_rows"
|
||
echo '```'
|
||
echo "- Math model is mis-calibrated against measured reality for the rows above. Any kv-calc projection on this checkout should be treated as suspect until the calibration anchors / formulas are reconciled."
|
||
else
|
||
echo "- No FAIL rows. kv-calc projections should agree with measured VRAM within the ±1.5 GB error band."
|
||
fi
|
||
} | redact
|
||
echo "$calib_output" | redact | details "Full kv-calc --calibration output"
|
||
fi
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Quality tooling (benchlocal-cli + sandboxes)
|
||
# ---------------------------------------------------------------------------
|
||
# Triage for "my quality run skipped packs / scored weird": is benchlocal-cli
|
||
# installed, how fresh, are the sandbox images built, and do they PREDATE the
|
||
# CLI (the rebuilt-CLI-stale-sandboxes incident class)? All best-effort — a
|
||
# rig without any of this still produces a report.
|
||
|
||
section "Quality tooling (benchlocal-cli + sandboxes)"
|
||
{
|
||
bl_bin=$(command -v benchlocal-cli 2>/dev/null || true)
|
||
if [[ -z "$bl_bin" ]]; then
|
||
echo "- **benchlocal-cli:** not installed (quality-test.sh needs it — \`pip install git+https://github.com/noonghunna/benchlocal-cli.git\`)"
|
||
else
|
||
bl_mtime=$(stat -c %Y "$bl_bin" 2>/dev/null || echo 0)
|
||
bl_when=$([[ "$bl_mtime" -gt 0 ]] && date -d "@${bl_mtime}" +%F 2>/dev/null || echo "unknown")
|
||
# Version via the CLI's own interpreter (works for pip-from-git AND
|
||
# editable-checkout installs; console-script shebang points at the env).
|
||
bl_py=$(head -1 "$bl_bin" 2>/dev/null | sed 's/^#!//')
|
||
bl_ver=$([[ -x "$bl_py" ]] && "$bl_py" -c 'import importlib.metadata as m; print(m.version("benchlocal-cli"))' 2>/dev/null || true)
|
||
# PRECISE source — the metadata version is frozen at install time and
|
||
# fixes are pushed without bumping it, so it alone can't identify the
|
||
# code. pip records the truth in direct_url.json: a git install carries
|
||
# the exact commit; an editable install carries the checkout dir → git
|
||
# describe (path itself withheld from the public report).
|
||
bl_src=$([[ -x "$bl_py" ]] && "$bl_py" - <<'PYEOF' 2>/dev/null
|
||
import importlib.metadata as m, json, subprocess
|
||
try:
|
||
raw = m.distribution("benchlocal-cli").read_text("direct_url.json") or ""
|
||
d = json.loads(raw)
|
||
except Exception:
|
||
d = {}
|
||
vcs = (d.get("vcs_info") or {}).get("commit_id")
|
||
if vcs:
|
||
print(f"git@{vcs[:9]}")
|
||
elif (d.get("dir_info") or {}).get("editable") and str(d.get("url", "")).startswith("file://"):
|
||
path = d["url"][7:]
|
||
try:
|
||
desc = subprocess.run(["git", "-C", path, "describe", "--tags", "--always", "--dirty"],
|
||
capture_output=True, text=True, timeout=5).stdout.strip()
|
||
print(f"{desc} (editable checkout)" if desc else "editable checkout")
|
||
except Exception:
|
||
print("editable checkout")
|
||
PYEOF
|
||
)
|
||
echo "- **benchlocal-cli:** \`${bl_bin}\` (version: \`${bl_ver:-unknown}\`${bl_src:+, source: \`${bl_src}\`}, installed/updated: ${bl_when})"
|
||
if have docker && docker info >/dev/null 2>&1; then
|
||
stale_any=0
|
||
echo "- **Sandbox images** (needed by the --full sandboxed packs):"
|
||
for img in benchlocal-sandbox-bugfind benchlocal-sandbox-cli benchlocal-sandbox-hermes benchlocal-sandbox-aider-polyglot; do
|
||
created=$(docker image inspect "${img}:latest" --format '{{.Created}}' 2>/dev/null || true)
|
||
if [[ -z "$created" ]]; then
|
||
echo " - \`${img}\`: ✗ not built"
|
||
continue
|
||
fi
|
||
cdate=$(date -d "$created" +%F 2>/dev/null || echo "$created")
|
||
cepoch=$(date -d "$created" +%s 2>/dev/null || echo 0)
|
||
mark=""
|
||
if [[ "$cepoch" -gt 0 && "$bl_mtime" -gt 0 && "$cepoch" -lt "$bl_mtime" ]]; then
|
||
mark=" ⚠ OLDER than the installed CLI — rebuild if the update touched sandbox sources"
|
||
stale_any=1
|
||
fi
|
||
echo " - \`${img}\`: built ${cdate}${mark}"
|
||
done
|
||
[[ "$stale_any" == "1" ]] && echo "- **Rebuild:** \`bash <benchlocal-cli-checkout>/tools/build-sandboxes.sh\` (heuristic — an unrelated reinstall also trips it)"
|
||
else
|
||
echo "- **Sandbox images:** docker unavailable — cannot inspect (sandboxed packs need Docker)"
|
||
fi
|
||
fi
|
||
latest_q=$(ls -t results/quality/quality-*.json 2>/dev/null | head -1)
|
||
if [[ -n "$latest_q" ]]; then
|
||
echo "- **Latest quality result:** \`${latest_q}\` ($(date -d "@$(stat -c %Y "$latest_q")" +%F 2>/dev/null || echo '?'))"
|
||
else
|
||
echo "- **Latest quality result:** none found under results/quality/"
|
||
fi
|
||
} | redact
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Active container
|
||
# ---------------------------------------------------------------------------
|
||
|
||
section "Active container"
|
||
# Engine-agnostic auto-detection: try vllm-* first (most common on this stack),
|
||
# fall back to llama-cpp-* (the alternate engine we ship). User can override
|
||
# with CONTAINER=... env var for non-standard naming (microk8s deployments,
|
||
# host engine builds via CONTAINER=none, etc.).
|
||
if [[ -z "$CONTAINER" ]] && have docker && docker info >/dev/null 2>&1; then
|
||
CONTAINER=$(docker ps --format '{{.Names}}' --filter 'name=vllm-qwen36' 2>/dev/null | head -1)
|
||
[[ -z "$CONTAINER" ]] && CONTAINER=$(docker ps --format '{{.Names}}' --filter 'name=vllm-' 2>/dev/null | head -1)
|
||
[[ -z "$CONTAINER" ]] && CONTAINER=$(docker ps --format '{{.Names}}' --filter 'name=llama-cpp-' 2>/dev/null | head -1)
|
||
[[ -z "$CONTAINER" ]] && CONTAINER=$(docker ps --format '{{.Names}}' --filter 'name=beellama-' 2>/dev/null | head -1)
|
||
[[ -z "$CONTAINER" ]] && CONTAINER=$(docker ps --format '{{.Names}}' --filter 'name=club3090-' 2>/dev/null | head -1)
|
||
fi
|
||
|
||
# Engine class — drives which probes run inside the container body. Inferred
|
||
# from container name; user can override with ENGINE_KIND=vllm|llamacpp env var.
|
||
case "${ENGINE_KIND:-}" in
|
||
vllm|llamacpp|unknown) ;; # respect user override
|
||
*)
|
||
case "$CONTAINER" in
|
||
vllm-*) ENGINE_KIND="vllm" ;;
|
||
llama-cpp-*) ENGINE_KIND="llamacpp" ;;
|
||
club3090-*)
|
||
container_image=$(docker ps --filter "name=$CONTAINER" --format '{{.Image}}' 2>/dev/null | head -1)
|
||
case "$container_image" in
|
||
*llama.cpp*|*llama-cpp*) ENGINE_KIND="llamacpp" ;;
|
||
*vllm*) ENGINE_KIND="vllm" ;;
|
||
*) ENGINE_KIND="unknown" ;;
|
||
esac ;;
|
||
*) ENGINE_KIND="unknown" ;;
|
||
esac ;;
|
||
esac
|
||
|
||
if [[ -z "$CONTAINER" ]]; then
|
||
echo "_No vLLM, llama.cpp, or estate container running. Start one with \`bash scripts/launch.sh\` and re-run for the full report._"
|
||
else
|
||
{
|
||
status=$(docker ps --filter "name=$CONTAINER" --format '{{.Status}}' 2>/dev/null | head -1)
|
||
ports=$(docker ps --filter "name=$CONTAINER" --format '{{.Ports}}' 2>/dev/null | head -1)
|
||
image=$(docker ps --filter "name=$CONTAINER" --format '{{.Image}}' 2>/dev/null | head -1)
|
||
# Digest + OCI labels (load-bearing when tag is rolling, e.g. llama.cpp
|
||
# `:server-cuda`). Without these, a bug report can't reproduce the bytes.
|
||
image_digest=$(docker inspect "$CONTAINER" --format '{{.Image}}' 2>/dev/null | head -1)
|
||
image_revision=$(docker inspect "$CONTAINER" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' 2>/dev/null)
|
||
image_version=$(docker inspect "$CONTAINER" --format '{{ index .Config.Labels "org.opencontainers.image.version" }}' 2>/dev/null)
|
||
image_source=$(docker inspect "$CONTAINER" --format '{{ index .Config.Labels "org.opencontainers.image.source" }}' 2>/dev/null)
|
||
echo "- **Name:** \`$CONTAINER\`"
|
||
echo "- **Engine:** \`${ENGINE_KIND}\`"
|
||
echo "- **Status:** ${status:-unknown}"
|
||
echo "- **Ports:** ${ports:-unknown}"
|
||
echo "- **Image:** \`${image:-unknown}\`"
|
||
[[ -n "$image_digest" ]] && echo "- **Image digest:** \`${image_digest}\`"
|
||
[[ -n "$image_version" && "$image_version" != "<no value>" ]] && echo "- **Build tag (OCI version):** \`${image_version}\`"
|
||
[[ -n "$image_revision" && "$image_revision" != "<no value>" ]] && echo "- **Upstream commit (OCI revision):** \`${image_revision}\`"
|
||
[[ -n "$image_source" && "$image_source" != "<no value>" ]] && echo "- **Upstream source:** ${image_source}"
|
||
} | redact
|
||
|
||
# Engine-specific probes from this point. vLLM container has Python +
|
||
# PyTorch + Genesis markers; llama.cpp container ships a stripped C++
|
||
# binary with no Python — different probe set.
|
||
|
||
# Engine-specific subsections. vLLM container has Python + PyTorch + Genesis
|
||
# markers; llama.cpp container ships a stripped C++ binary with no Python
|
||
# exec available — different probe set.
|
||
|
||
if [[ "$ENGINE_KIND" == "llamacpp" ]]; then
|
||
# ---- llama.cpp probe set ----
|
||
subsection "Container engine state (llama.cpp)"
|
||
{
|
||
# llama-server prints its version + build flags on startup. Grep the
|
||
# boot log for the version banner instead of trying to docker exec
|
||
# (the llama-cpp image doesn't ship interactive shell utilities).
|
||
llama_version=$(docker logs "$CONTAINER" 2>&1 | grep -E '^build_info:|^version:|^system_info:' | head -3)
|
||
if [[ -n "$llama_version" ]]; then
|
||
echo "**llama-server version + build:**"
|
||
echo '```'
|
||
echo "$llama_version"
|
||
echo '```'
|
||
echo
|
||
fi
|
||
|
||
# Loaded model + ctx + KV type — surfaces model identity from boot log.
|
||
model_loaded=$(docker logs "$CONTAINER" 2>&1 | grep -E 'load_model:|llama_model_load_from_file_impl:|llama_kv_cache_init:|llama_init_from_model:' | head -8)
|
||
if [[ -n "$model_loaded" ]]; then
|
||
echo "**Model load + KV cache init:**"
|
||
echo '```'
|
||
echo "$model_loaded"
|
||
echo '```'
|
||
echo
|
||
fi
|
||
|
||
# llama.cpp doesn't have Genesis / vLLM SpecDecoding metrics. Skip
|
||
# those grep patterns. Capture warnings/errors only.
|
||
boot_errors=$(docker logs "$CONTAINER" 2>&1 | grep -iE '^(warn|error|fatal|abort)|panic|core dumped' | tail -5)
|
||
if [[ -n "$boot_errors" ]]; then
|
||
echo "**Recent warnings/errors (last 5):**"
|
||
echo '```'
|
||
echo "$boot_errors"
|
||
echo '```'
|
||
fi
|
||
} | redact
|
||
|
||
subsection "Full boot log (first 200 lines)"
|
||
docker logs "$CONTAINER" 2>&1 | head -200 | redact | details "First 200 lines of docker logs"
|
||
|
||
else
|
||
# ---- vLLM probe set (default for engine=vllm or unknown) ----
|
||
subsection "Container Python / CUDA versions"
|
||
{
|
||
# vLLM version + Torch CUDA build vs host driver mismatch is one of the
|
||
# rare failure modes that image SHA pinning doesn't catch. Quick docker
|
||
# exec to surface what the container actually sees.
|
||
py_versions=$(docker exec "$CONTAINER" python3 -c \
|
||
'import torch, sys; print(f"torch={torch.__version__} torch_cuda_build={torch.version.cuda} cudnn={torch.backends.cudnn.version()}")' \
|
||
2>&1)
|
||
if [[ -n "$py_versions" ]] && [[ "$py_versions" != *"Error"* ]] && [[ "$py_versions" != *"error"* ]]; then
|
||
echo "- **PyTorch:** \`${py_versions}\`"
|
||
else
|
||
echo "- **PyTorch:** (could not query — \`docker exec\` failed or torch not importable)"
|
||
fi
|
||
|
||
vllm_ver=$(docker exec "$CONTAINER" python3 -c 'import vllm; print(vllm.__version__)' 2>&1)
|
||
if [[ -n "$vllm_ver" ]] && [[ "$vllm_ver" != *"Error"* ]] && [[ "$vllm_ver" != *"error"* ]]; then
|
||
echo "- **vLLM:** \`${vllm_ver}\`"
|
||
else
|
||
echo "- **vLLM:** (could not query)"
|
||
fi
|
||
|
||
# Container's view of the GPUs — should match host driver, but if NVIDIA
|
||
# Container Toolkit is mis-configured this surfaces the mismatch.
|
||
cuda_in_container=$(docker exec "$CONTAINER" nvidia-smi --query-gpu=index,name,driver_version --format=csv,noheader 2>&1 | head -4)
|
||
if [[ -n "$cuda_in_container" ]] && [[ "$cuda_in_container" != *"Error"* ]] && [[ "$cuda_in_container" != *"command not found"* ]]; then
|
||
echo "- **nvidia-smi inside container:**"
|
||
echo ' ```'
|
||
echo "$cuda_in_container" | sed 's/^/ /'
|
||
echo ' ```'
|
||
fi
|
||
} | redact
|
||
|
||
subsection "Boot log highlights"
|
||
{
|
||
# Interconnect / P2P ENGAGEMENT — the runtime truth. The GPU "Topology" /
|
||
# "PCIe / P2P detail" sections above report P2P *capability* (can it?); this
|
||
# reports whether P2P is actually ON for the running serving container.
|
||
# detect_nvlink.sh emits an [nvlink] decision trail at boot stating the
|
||
# resolved NCCL_P2P_LEVEL + custom-all-reduce state. Grep the WHOLE log (not
|
||
# head -200) so a late line on a 3-4 GPU boot isn't missed, and fall back to
|
||
# the live container env. ALWAYS prints something so a reviewer never has to
|
||
# guess whether P2P was engaged (the gap that forced asks on #446 / #488).
|
||
nvlink_boot=$(docker logs "$CONTAINER" 2>&1 | grep -E '\[nvlink\]' | head -8)
|
||
p2p_env=$(docker exec "$CONTAINER" env 2>/dev/null | grep -E '^(NCCL_P2P|NVLINK_MODE|NCCL_CUMEM)=' | sort)
|
||
echo "**Interconnect / P2P engagement:**"
|
||
if [[ -n "$nvlink_boot" || -n "$p2p_env" ]]; then
|
||
echo '```'
|
||
[[ -n "$nvlink_boot" ]] && echo "$nvlink_boot"
|
||
[[ -n "$p2p_env" ]] && { echo "# resolved container env:"; echo "$p2p_env"; }
|
||
echo '```'
|
||
else
|
||
echo "_No \`[nvlink]\` boot line or NCCL_P2P/NVLINK_MODE env found — P2P engagement undetermined (single-GPU, a non-NCCL engine like llama.cpp, or an entrypoint predating detect_nvlink.sh)._"
|
||
fi
|
||
# Cross-referenced VERDICT (capability x engagement — the #488/#158 matrix).
|
||
# Silent on single-GPU / no-capability rigs so the OK/WARN/INFO line is
|
||
# always signal, never boilerplate.
|
||
_p2p_verdict_line="$(p2p_verdict "$(p2p_gpu_count)" "$(p2p_host_capability)" \
|
||
"$(printf '%s\n%s' "$nvlink_boot" "$p2p_env" | p2p_classify_engagement)")"
|
||
[[ -n "$_p2p_verdict_line" ]] && { echo; echo "**Interconnect verdict:** ${_p2p_verdict_line}"; }
|
||
echo
|
||
|
||
genesis_results=$(docker logs "$CONTAINER" 2>&1 | grep -E '\[INFO:genesis\.apply_all\] (Genesis|✅) Results' | tail -1)
|
||
if [[ -n "$genesis_results" ]]; then
|
||
echo "**Genesis patches applied:**"
|
||
echo '```'
|
||
echo "$genesis_results" | sed 's/.*Genesis Results: /Genesis Results: /'
|
||
echo '```'
|
||
echo
|
||
fi
|
||
|
||
sidecar_status=$(docker logs "$CONTAINER" 2>&1 | grep -E '^\[(tolist_cudagraph_fix|inputs_embeds_optional|workspace_lock_disable|pn25_genesis_register_fix|pn30_dst_shaped_temp_fix|fa_max_seqlen_clamp|pn12_ffn_pool_anchor|pn12_compile_safe_custom_op)\]' | head -10)
|
||
if [[ -n "$sidecar_status" ]]; then
|
||
echo "**Local sidecar application:**"
|
||
echo '```'
|
||
echo "$sidecar_status"
|
||
echo '```'
|
||
echo
|
||
fi
|
||
|
||
kv_pool=$(docker logs "$CONTAINER" 2>&1 | grep -E 'Available KV cache memory|GPU KV cache size:|Maximum concurrency for' | tail -3)
|
||
if [[ -n "$kv_pool" ]]; then
|
||
echo "**KV pool sizing:**"
|
||
echo '```'
|
||
echo "$kv_pool"
|
||
echo '```'
|
||
echo
|
||
fi
|
||
|
||
# Engine config — the line containing "non-default args" or "Initializing a V1 LLM engine"
|
||
# captures every important CLI flag (max_model_len, mem_util, kv dtype, spec config, etc.)
|
||
engine_config=$(docker logs "$CONTAINER" 2>&1 | grep -E 'non-default args:|Initializing a V1 LLM engine' | head -2)
|
||
if [[ -n "$engine_config" ]]; then
|
||
echo "**Engine config (CLI flags + engine init):**"
|
||
echo '```'
|
||
echo "$engine_config"
|
||
echo '```'
|
||
echo
|
||
fi
|
||
|
||
boot_errors=$(docker logs "$CONTAINER" 2>&1 | grep -E '^(WARNING|ERROR|CRITICAL)' | tail -5)
|
||
if [[ -n "$boot_errors" ]]; then
|
||
echo "**Recent warnings/errors (last 5):**"
|
||
echo '```'
|
||
echo "$boot_errors"
|
||
echo '```'
|
||
fi
|
||
} | redact
|
||
|
||
subsection "Full boot log (first 200 lines)"
|
||
docker logs "$CONTAINER" 2>&1 | head -200 | redact | details "First 200 lines of docker logs"
|
||
fi # end of vLLM/llamacpp engine branch
|
||
fi # end of "if no container running"
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Recent failed boot attempts
|
||
# ---------------------------------------------------------------------------
|
||
# Capture exited vLLM/llama.cpp containers from the last 24h. Most valuable
|
||
# diagnostic data for boot-failure scenarios — without this, contributors hit
|
||
# "no container running" and have to manually paste docker logs ad-hoc.
|
||
# Engine-agnostic: matches both vllm-* and llama-cpp-* container patterns.
|
||
|
||
section "Recent failed boot attempts"
|
||
if ! have docker; then
|
||
echo "_docker not available — skipping._"
|
||
elif ! docker info >/dev/null 2>&1; then
|
||
echo "_docker daemon unreachable — skipping._"
|
||
else
|
||
# Get exited containers matching club-3090 engine patterns. `docker ps -a`
|
||
# without a time filter; we'll filter to last 24h via the FinishedAt field.
|
||
exited_lines=$(docker ps -a \
|
||
--format '{{.Names}}\t{{.Image}}\t{{.Status}}\t{{.ID}}' \
|
||
--filter 'status=exited' 2>/dev/null \
|
||
| grep -E '^(vllm-|llama-cpp-)' || true)
|
||
|
||
if [[ -z "$exited_lines" ]]; then
|
||
echo "_No recently-exited vLLM or llama.cpp containers found._"
|
||
else
|
||
found_recent=0
|
||
while IFS=$'\t' read -r ex_name ex_image ex_status ex_id; do
|
||
[[ -z "$ex_name" ]] && continue
|
||
# Cutoff: last 24h. docker inspect gives ISO-8601 FinishedAt.
|
||
finished_at=$(docker inspect "$ex_id" --format '{{.State.FinishedAt}}' 2>/dev/null || echo "")
|
||
exit_code=$(docker inspect "$ex_id" --format '{{.State.ExitCode}}' 2>/dev/null || echo "?")
|
||
[[ -z "$finished_at" ]] && continue
|
||
|
||
# Skip containers that exited >24h ago (epoch comparison)
|
||
finished_epoch=$(date -d "$finished_at" +%s 2>/dev/null || echo 0)
|
||
cutoff_epoch=$(date -d '24 hours ago' +%s 2>/dev/null || echo 0)
|
||
[[ "$finished_epoch" -lt "$cutoff_epoch" ]] && continue
|
||
|
||
found_recent=1
|
||
relative_when=$(date -d "$finished_at" '+%Y-%m-%dT%H:%M:%SZ (%s seconds ago)' 2>/dev/null || echo "$finished_at")
|
||
# Format relative_when nicely: how many minutes ago?
|
||
mins_ago=$(( ($(date +%s) - finished_epoch) / 60 ))
|
||
if [[ $mins_ago -lt 60 ]]; then
|
||
relative_label="${mins_ago} min ago"
|
||
else
|
||
hrs_ago=$(( mins_ago / 60 ))
|
||
rem_mins=$(( mins_ago % 60 ))
|
||
relative_label="${hrs_ago}h ${rem_mins}min ago"
|
||
fi
|
||
|
||
subsection "\`$ex_name\` — exited $relative_label (code $exit_code)"
|
||
{
|
||
echo "- **Name:** \`$ex_name\`"
|
||
echo "- **Image:** \`$ex_image\`"
|
||
echo "- **Status:** $ex_status"
|
||
echo "- **Exit code:** $exit_code"
|
||
echo "- **Finished at:** $finished_at"
|
||
} | redact
|
||
|
||
docker logs --tail 80 "$ex_id" 2>&1 | redact | details "Last 80 log lines from \`$ex_name\`"
|
||
done <<< "$exited_lines"
|
||
|
||
if [[ "$found_recent" == "0" ]]; then
|
||
echo "_Exited vLLM/llama.cpp containers exist but all >24h old — likely not relevant to current investigation._"
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Optional: verify-full
|
||
# ---------------------------------------------------------------------------
|
||
|
||
if [[ $DO_VERIFY -eq 1 ]]; then
|
||
section "verify-full.sh output"
|
||
if [[ -f scripts/verify-full.sh ]]; then
|
||
bash scripts/verify-full.sh 2>&1 | redact | details "verify-full output"
|
||
else
|
||
echo "_scripts/verify-full.sh not found_"
|
||
fi
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Optional: verify-stress
|
||
# ---------------------------------------------------------------------------
|
||
|
||
if [[ $DO_STRESS -eq 1 ]]; then
|
||
section "verify-stress.sh output"
|
||
if [[ -f scripts/verify-stress.sh ]]; then
|
||
bash scripts/verify-stress.sh 2>&1 | redact | details "verify-stress output (7 boundary checks incl. Cliff 2 needle recall)"
|
||
else
|
||
echo "_scripts/verify-stress.sh not found_"
|
||
fi
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Optional: soak-continuous (catches Cliff 2b — the only test that does)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
if [[ $DO_SOAK -eq 1 ]]; then
|
||
section "soak-test.sh (SOAK_MODE=continuous) output"
|
||
if [[ -f scripts/soak-test.sh ]]; then
|
||
soak_run_dir="results/report-soak-$(date +%Y%m%d-%H%M%S)"
|
||
SOAK_MODE=continuous SOAK_SESSIONS=5 SOAK_TURNS=5 SOAK_OUTPUT="$soak_run_dir" \
|
||
SOAK_TIMEOUT_S="${SOAK_TIMEOUT_S:-1800}" \
|
||
bash scripts/soak-test.sh 2>&1 | redact | details "soak-test stdout (5-session × 5-turn ramping conversation, ~25 min)"
|
||
if [[ -f "$soak_run_dir/summary.md" ]]; then
|
||
echo
|
||
echo "**Soak summary** (\`$soak_run_dir/summary.md\`):"
|
||
echo
|
||
redact < "$soak_run_dir/summary.md"
|
||
else
|
||
echo "_soak summary.md not produced — check stdout above_"
|
||
fi
|
||
else
|
||
echo "_scripts/soak-test.sh not found_"
|
||
fi
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Optional: bench
|
||
# ---------------------------------------------------------------------------
|
||
|
||
if [[ $DO_BENCH -eq 1 ]]; then
|
||
section "bench.sh output"
|
||
if [[ -f scripts/bench.sh ]]; then
|
||
bash scripts/bench.sh 2>&1 | redact | details "bench output (3 warmups + 5 measured per prompt)"
|
||
else
|
||
echo "_scripts/bench.sh not found_"
|
||
fi
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Soak-not-run reminder — fired when --bench (or partial) was used without
|
||
# --soak/--full. Cross-rig bench rows want the soak verdict; without it we
|
||
# can't say if Cliff 2b is open on this rig class.
|
||
# ---------------------------------------------------------------------------
|
||
|
||
if [[ $DO_BENCH -eq 1 && $DO_SOAK -eq 0 ]]; then
|
||
section "Soak status"
|
||
cat <<'EOF'
|
||
> ⚠️ **Soak: not included in this report.**
|
||
>
|
||
> This run used `--bench` (or `--verify`/`--stress` only) — the soak-continuous
|
||
> test was skipped. Cross-rig bench contributions on club-3090 want the soak
|
||
> verdict so we can tell whether Cliff 2b is open on your rig class.
|
||
>
|
||
> Run soak separately and paste its output as a follow-up:
|
||
>
|
||
> ```bash
|
||
> bash scripts/soak-test.sh --continuous # auto-detects endpoint + container
|
||
> ```
|
||
>
|
||
> Takes ~25 min. The `[soak]` summary block (verdict, max VRAM growth, silent-empty %, TPS retention) is what ends up in the bench-template's "Soak verdict" dropdown. See [docs/CLIFFS.md](https://github.com/noonghunna/club-3090/blob/master/docs/CLIFFS.md) for context.
|
||
EOF
|
||
fi
|
||
|
||
if [[ $DO_AGENTIC -eq 1 ]]; then
|
||
section "bench-agentic.sh output"
|
||
if [[ -f scripts/bench-agentic.sh ]]; then
|
||
SESSIONS=1 bash scripts/bench-agentic.sh 2>&1 | redact | details "bench-agentic output (1 session x 12 default turns, curve-shape estimate; ~8 min estimate)"
|
||
else
|
||
echo "_scripts/bench-agentic.sh not found_"
|
||
fi
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# AI Studio container logs (--studio) — opt-in: the ComfyUI / director / orchestrator
|
||
# log tails that diagnose an image/video/audio GENERATION failure (e.g. a "ComfyUI
|
||
# generation error" in a lane). Off by default — verbose + only relevant for studio bugs.
|
||
# ---------------------------------------------------------------------------
|
||
if [[ $DO_STUDIO -eq 1 ]]; then
|
||
section "AI Studio logs (--studio)"
|
||
echo "_Container log tails for image/video/audio generation bugs. ComfyUI carries the workflow"
|
||
echo "execution trace (the actual generation error). Redacted; pass \`--no-redact\` for full paths._"
|
||
_studio_found=0
|
||
# ComfyUI first (the generation engine — longest tail), then the studio sidecars.
|
||
for c in comfyui studio-director studio-orchestrator studio-image-shim studio-tts studio-step-voice studio-gallery; do
|
||
if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -qx "$c"; then
|
||
_studio_found=1
|
||
_tail=200; [[ "$c" == comfyui ]] && _tail=400
|
||
_running=$(docker ps --filter "name=^${c}$" --format '{{.Status}}' 2>/dev/null | head -1)
|
||
# strip ANSI colour codes (ComfyUI logs are coloured) so the pasted block reads cleanly
|
||
docker logs --tail "$_tail" "$c" 2>&1 | sed -E 's/\x1b\[[0-9;]*[mK]//g' | redact \
|
||
| details "$c — ${_running:-not running} (last $_tail lines)"
|
||
fi
|
||
done
|
||
[[ $_studio_found -eq 0 ]] && echo "_No AI Studio containers found. Bring the studio up (\`gpu-mode ai-studio\` or \`bash scripts/setup-ai-studio.sh\`), reproduce the failure, then re-run with \`--studio\`._"
|
||
fi
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Footer
|
||
# ---------------------------------------------------------------------------
|
||
|
||
cat <<'EOF'
|
||
|
||
---
|
||
|
||
_Generated by `bash scripts/report.sh`. Flags: `--verify` (verify-full), `--stress` (verify-stress 7/7 incl. Cliff 2 needles), `--soak` (SOAK_MODE=continuous, catches Cliff 2b), `--bench` (canonical TPS), `--agentic` (multi-turn TTFT/decode curve-shape, ~8 min estimate), `--studio` (AI Studio / ComfyUI container log tails — for generation bugs), `--full` (all five, ~43 min estimate). Use `--no-redact` to disable redaction (internal sharing only)._
|
||
EOF
|