From 496f82d8999034d1953cd7a7249fa4e7686830f4 Mon Sep 17 00:00:00 2001 From: noonghunna Date: Thu, 18 Jun 2026 07:31:02 +0500 Subject: [PATCH] =?UTF-8?q?Add=20c3t=20=E2=80=94=20the=20club-3090=20test-?= =?UTF-8?q?console=20TUI=20(built=20by=20Qwen=20Max=20from=20the=20spec)?= =?UTF-8?q?=20(#428)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A lazydocker-style terminal UI for the stack's test scripts (verify / bench / verify-stress-NIAH / quality / soak / rebench-full): auto-detects the running model + endpoint, injects MODEL=/URL=, streams live progress, and browses the results/ history. Built by Qwen Max from docs/club3090-test-tui-prompt.md. - scripts/c3t — launcher that derives the repo root from its own location and runs the tool from tools/test-console's own venv (no global installs). - tools/test-console/ — Python + Textual package, pinned pyproject + uv.lock, and an 83-test offline pytest suite (parsers vs fixtures, mocked docker / /v1/models detection, BENCH_MOCK bench parse) — all green, no GPU required. Fixed before merge: __main__.py and a test fixture hardcoded the rig path /opt/ai/github/club-3090 — now derived from __file__ (parents[3]; override via C3T_REPO_ROOT) so it works from any clone, not just this rig. .gitignore excludes .venv / caches. Co-authored-by: noonghunna <10742901+noonghunna@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 --- scripts/c3t | 33 + tools/test-console/.gitignore | 6 + tools/test-console/README.md | 107 +++ .../club3090_test_console/__init__.py | 3 + .../club3090_test_console/__main__.py | 54 ++ .../test-console/club3090_test_console/app.py | 765 ++++++++++++++++++ .../club3090_test_console/app.tcss | 34 + .../club3090_test_console/detect.py | 279 +++++++ .../club3090_test_console/parsers.py | 609 ++++++++++++++ .../club3090_test_console/runner.py | 404 +++++++++ .../club3090_test_console/widgets/__init__.py | 1 + .../widgets/history_view.py | 344 ++++++++ .../widgets/live_pane.py | 309 +++++++ .../widgets/manual_target.py | 125 +++ .../widgets/target_pane.py | 131 +++ .../widgets/test_menu.py | 171 ++++ tools/test-console/pyproject.toml | 34 + tools/test-console/test_summary.md | 120 +++ tools/test-console/tests/__init__.py | 0 tools/test-console/tests/test_detect.py | 214 +++++ tools/test-console/tests/test_parsers.py | 480 +++++++++++ tools/test-console/tests/test_runner.py | 215 +++++ tools/test-console/uv.lock | 281 +++++++ 23 files changed, 4719 insertions(+) create mode 100755 scripts/c3t create mode 100644 tools/test-console/.gitignore create mode 100644 tools/test-console/README.md create mode 100644 tools/test-console/club3090_test_console/__init__.py create mode 100644 tools/test-console/club3090_test_console/__main__.py create mode 100644 tools/test-console/club3090_test_console/app.py create mode 100644 tools/test-console/club3090_test_console/app.tcss create mode 100644 tools/test-console/club3090_test_console/detect.py create mode 100644 tools/test-console/club3090_test_console/parsers.py create mode 100644 tools/test-console/club3090_test_console/runner.py create mode 100644 tools/test-console/club3090_test_console/widgets/__init__.py create mode 100644 tools/test-console/club3090_test_console/widgets/history_view.py create mode 100644 tools/test-console/club3090_test_console/widgets/live_pane.py create mode 100644 tools/test-console/club3090_test_console/widgets/manual_target.py create mode 100644 tools/test-console/club3090_test_console/widgets/target_pane.py create mode 100644 tools/test-console/club3090_test_console/widgets/test_menu.py create mode 100644 tools/test-console/pyproject.toml create mode 100644 tools/test-console/test_summary.md create mode 100644 tools/test-console/tests/__init__.py create mode 100644 tools/test-console/tests/test_detect.py create mode 100644 tools/test-console/tests/test_parsers.py create mode 100644 tools/test-console/tests/test_runner.py create mode 100644 tools/test-console/uv.lock diff --git a/scripts/c3t b/scripts/c3t new file mode 100755 index 00000000..c7600b13 --- /dev/null +++ b/scripts/c3t @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# c3t — launcher for the club3090 test console TUI +# +# Runs the tool from its own isolated env (uv run / .venv), never the system Python. +# +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +TOOL_DIR="${SCRIPT_DIR}/../tools/test-console" + +# Ensure we're in the repo root for cwd resolution +REPO_ROOT="${SCRIPT_DIR}/.." + +if [[ ! -d "${TOOL_DIR}" ]]; then + echo "Error: test-console tool dir not found at ${TOOL_DIR}" >&2 + exit 1 +fi + +cd "${REPO_ROOT}" + +# Prefer uv if available, fall back to .venv, fall back to pipx +if command -v uv &>/dev/null; then + exec uv run --project "${TOOL_DIR}" -- python -m club3090_test_console "$@" +elif [[ -d "${TOOL_DIR}/.venv" ]]; then + exec "${TOOL_DIR}/.venv/bin/python" -m club3090_test_console "$@" +elif command -v pipx &>/dev/null; then + exec pipx run --spec "${TOOL_DIR}" c3t "$@" +else + echo "Error: No Python runner found. Install uv, create a .venv, or install pipx." >&2 + echo " Quick setup: cd ${TOOL_DIR} && python3 -m venv .venv && .venv/bin/pip install -e ." >&2 + exit 1 +fi diff --git a/tools/test-console/.gitignore b/tools/test-console/.gitignore new file mode 100644 index 00000000..e1320ce5 --- /dev/null +++ b/tools/test-console/.gitignore @@ -0,0 +1,6 @@ +.venv/ +__pycache__/ +*.py[cod] +.pytest_cache/ +.ruff_cache/ +*.egg-info/ diff --git a/tools/test-console/README.md b/tools/test-console/README.md new file mode 100644 index 00000000..4b5f6540 --- /dev/null +++ b/tools/test-console/README.md @@ -0,0 +1,107 @@ +# club3090-test-console (c3t) + +A lazydocker-style terminal UI for the club-3090 AI inference stack test suite. + +## What it does + +The TUI wraps the stack's existing test scripts (`bench.sh`, `verify-full.sh`, `quality-test.sh`, etc.) and: + +1. **Auto-detects** the serving model + endpoint (no more wrong-port/wrong-MODEL headaches) +2. **Runs tests** with one keystroke — full pipeline or individual tests +3. **Streams live progress** with structured parsing (bench TPS bars, NIAH ladder, quality counters, soak gauges) +4. **Shows GPU stats** (VRAM, utilization, power draw/cap, temperature) in real-time + +## Quick start + +```bash +# From the repo root: +bash scripts/c3t + +# Or install with uv: +cd tools/test-console +uv sync +uv run c3t +``` + +## Setup + +```bash +cd tools/test-console + +# Option A: uv (recommended) +uv sync + +# Option B: pip + venv +python3 -m venv .venv +source .venv/bin/activate +pip install -e . +``` + +## Keybindings + +| Key | Action | +|-----|--------| +| `↑/↓` or `j/k` | Move in test menu | +| `Enter` | Run selected test (default config) | +| `c` | Configure selected test | +| `x` | Stop current run | +| `r` | Re-detect serving target | +| `m` | Manual target override | +| `f` | Toggle log follow/scroll-lock | +| `Tab` | Cycle pane focus | +| `?` | Help | +| `q` | Quit | + +## Test catalog + +| Test | Script | Duration | +|------|--------|----------| +| Smoke | `verify.sh` | ~15s | +| Functional | `verify-full.sh` | ~2min | +| Speed bench | `bench.sh` | ~5min | +| Stress / NIAH | `verify-stress.sh` | ~15min | +| Quality packs | `quality-test.sh` | 5-90min | +| Soak / stability | `soak-test.sh` | ~20min | +| ★ FULL rebench | `rebench-full.sh` | ~45min-4h | + +## Architecture + +``` +tools/test-console/ +├── pyproject.toml +├── README.md +├── club3090_test_console/ +│ ├── __init__.py +│ ├── __main__.py # Entry point +│ ├── app.py # Main Textual App +│ ├── app.tcss # Textual CSS +│ ├── detect.py # Endpoint/model auto-detection +│ ├── parsers.py # Output parsers per test +│ ├── runner.py # Subprocess management +│ └── widgets/ +│ ├── target_pane.py # Target status display +│ ├── test_menu.py # Test selection menu +│ └── live_pane.py # Structured progress + log +└── tests/ + ├── test_parsers.py # Parser unit tests (offline) + └── test_detect.py # Detection unit tests (offline) +``` + +## Development + +```bash +# Run tests (no GPU needed) +cd tools/test-console +uv run pytest -v + +# Run the TUI +uv run c3t +``` + +## Design decisions + +- **Python + Textual** over Go+BubbleTea: reuses the stack's Python ecosystem and can import `compose_registry` directly +- **No script modifications**: the TUI is a pure wrapper — it spawns existing scripts and parses their output +- **No global installs**: uses `uv` / project-local `.venv` for isolation +- **XDG state**: run history persists under `~/.local/state/club3090-test-console/`, never in the repo tree +- **Headless-testable**: all parsers and detection logic are unit-testable without a GPU diff --git a/tools/test-console/club3090_test_console/__init__.py b/tools/test-console/club3090_test_console/__init__.py new file mode 100644 index 00000000..38cec8a9 --- /dev/null +++ b/tools/test-console/club3090_test_console/__init__.py @@ -0,0 +1,3 @@ +"""club3090-test-console: A lazydocker-style TUI for the AI inference stack test suite.""" + +__version__ = "0.1.0" diff --git a/tools/test-console/club3090_test_console/__main__.py b/tools/test-console/club3090_test_console/__main__.py new file mode 100644 index 00000000..c3d283aa --- /dev/null +++ b/tools/test-console/club3090_test_console/__main__.py @@ -0,0 +1,54 @@ +"""Entry point for c3t command.""" + +import os +import sys +from pathlib import Path + + +def _suppress_event_loop_cleanup_error(): + """Suppress harmless 'Event loop is closed' errors from asyncio subprocess cleanup.""" + # Python 3.8+ has sys.unraisablehook for exceptions in __del__ methods + if hasattr(sys, 'unraisablehook'): + original_hook = sys.unraisablehook + + def filtered_hook(unraisable): + # Suppress 'Event loop is closed' from asyncio subprocess cleanup + if (unraisable.exc_type is RuntimeError and + unraisable.exc_value and + 'Event loop is closed' in str(unraisable.exc_value)): + return + # Also suppress 'Cannot run the event loop while another loop is running' + if (unraisable.exc_type is RuntimeError and + unraisable.exc_value and + 'loop is' in str(unraisable.exc_value).lower()): + return + original_hook(unraisable) + + sys.unraisablehook = filtered_hook + + +def main(): + """Launch the test console TUI.""" + _suppress_event_loop_cleanup_error() + + # Resolve the repo root from this file's location (…//tools/test-console/ + # club3090_test_console/__main__.py → parents[3] == repo root), so the tool works + # from any clone. Override with C3T_REPO_ROOT if the package is installed elsewhere. + env_root = os.environ.get("C3T_REPO_ROOT") + repo_root = Path(env_root) if env_root else Path(__file__).resolve().parents[3] + if not (repo_root / "scripts").is_dir(): + print( + f"Error: club-3090 repo root not found at {repo_root} " + f"(no scripts/ dir). Run via scripts/c3t, or set C3T_REPO_ROOT.", + file=sys.stderr, + ) + sys.exit(1) + + from .app import TestConsoleApp + + app = TestConsoleApp(repo_root=repo_root) + app.run() + + +if __name__ == "__main__": + main() diff --git a/tools/test-console/club3090_test_console/app.py b/tools/test-console/club3090_test_console/app.py new file mode 100644 index 00000000..f8e0313b --- /dev/null +++ b/tools/test-console/club3090_test_console/app.py @@ -0,0 +1,765 @@ +"""Main Textual application for the club3090 test console.""" + +from __future__ import annotations + +import asyncio +import json +import time +from pathlib import Path +from typing import Optional + +from textual.app import App, ComposeResult +from textual.binding import Binding +from textual.containers import Container, Horizontal, Vertical +from textual.screen import ModalScreen +from textual.widgets import Footer, Header, Input, Label, Static, Button, RichLog, Select + +from .detect import ServingTarget, detect_endpoint, detect_from_registry, match_target_to_registry, get_gpu_info +from .parsers import ParseEvent, TestType, Status +from .runner import TestConfig, TestRunner, RunState +from .widgets.target_pane import TargetPane +from .widgets.test_menu import TestMenuPane, TestEntry +from .widgets.live_pane import LivePane +from .widgets.history_view import HistoryScreen +from .widgets.manual_target import ManualTargetScreen + + +class ConfigScreen(ModalScreen[Optional[TestConfig]]): + """Modal screen for configuring a test run.""" + + DEFAULT_CSS = """ + ConfigScreen { + align: center middle; + } + ConfigScreen > Vertical { + width: 70; + height: auto; + max-height: 90%; + border: thick $primary; + background: $surface; + padding: 1 2; + } + ConfigScreen #config-fields { + height: auto; + max-height: 1fr; + overflow-y: auto; + } + ConfigScreen .config-title { + text-style: bold; + color: $accent; + text-align: center; + margin-bottom: 1; + } + ConfigScreen .config-hint { + color: $text-muted; + height: auto; + } + ConfigScreen Label { + margin-top: 1; + } + ConfigScreen Select { + margin-bottom: 1; + } + ConfigScreen Input { + margin-bottom: 1; + } + ConfigScreen #button-bar { + height: 3; + } + ConfigScreen Button { + width: 1fr; + } + """ + + BINDINGS = [ + Binding("escape", "cancel", "Cancel"), + ] + + def __init__(self, test_entry: TestEntry, current_config: Optional[TestConfig] = None): + super().__init__() + self.test_entry = test_entry + self._config = current_config or TestConfig(test_type=test_entry.test_type) + + def compose(self) -> ComposeResult: + with Vertical(): + yield Label(f"Configure: {self.test_entry.display_name}", classes="config-title") + yield Label(self._get_help_text(), classes="config-hint") + with Vertical(id="config-fields"): + yield from self._compose_config_fields() + with Horizontal(id="button-bar"): + yield Button("Run", variant="primary", id="btn-run") + yield Button("Cancel", variant="default", id="btn-cancel") + + def _get_help_text(self) -> str: + match self.test_entry.test_type: + case TestType.BENCH: + return "Tune RUNS, WARMUPS, prompt selection, and thinking mode." + case TestType.VERIFY_FULL: + return "Optionally skip tool-call check or add a bench run." + case TestType.VERIFY_STRESS: + return "Control which stress probes to run (longctx, tool-prefill, ceiling)." + case TestType.QUALITY: + return "Choose tier, pack, thinking mode, and repeat count." + case TestType.SOAK: + return "Set session/turn counts and VRAM growth limits." + case TestType.REBENCH: + return "Configure 8-pack quality toggle, skips, and soak sizing." + case _: + return "Configure test parameters." + + def _compose_config_fields(self) -> ComposeResult: + """Yield config fields based on test type.""" + tt = self.test_entry.test_type + c = self._config + + # Helper for boolean select + bool_options = [("No", False), ("Yes", True)] + + if tt == TestType.BENCH: + yield Label("RUNS (measured runs per prompt):") + yield Input(str(c.run_count), id="cfg-runs", type="integer") + yield Label("WARMUPS:") + yield Input(str(c.warmups), id="cfg-warmups", type="integer") + yield Label("ONLY (prompt selection):") + yield Select( + [("Both (narrative + code)", "both"), ("Narrative only", "narr"), ("Code only", "code")], + value=c.only, + id="cfg-only" + ) + yield Label("Enable thinking:") + yield Select(bool_options, value=c.enable_thinking, id="cfg-thinking") + + elif tt == TestType.VERIFY_FULL: + yield Label("Skip tools check:") + yield Select(bool_options, value=c.skip_tools, id="cfg-skip-tools") + yield Label("Run bench after:") + yield Select(bool_options, value=c.run_bench, id="cfg-run-bench") + + elif tt == TestType.VERIFY_STRESS: + yield Label("Skip long-context:") + yield Select(bool_options, value=c.skip_longctx, id="cfg-skip-longctx") + yield Label("Skip tool prefill:") + yield Select(bool_options, value=c.skip_tool_prefill, id="cfg-skip-prefill") + yield Label("Skip ceiling ladder:") + yield Select(bool_options, value=c.skip_ceiling, id="cfg-skip-ceiling") + + elif tt == TestType.QUALITY: + yield Label("Tier:") + yield Select( + [("Quick (2 packs)", "quick"), ("Medium (5 packs)", "medium"), + ("Full (8 packs)", "full"), ("Reasoning", "reasoning")], + value=c.quality_tier, + id="cfg-tier" + ) + yield Label("Pack ID (empty = all for tier):") + yield Input(c.quality_pack, id="cfg-pack", placeholder="e.g., toolcall-15") + yield Label("Enable thinking:") + yield Select(bool_options, value=c.enable_thinking, id="cfg-thinking") + yield Label("Repeat count:") + yield Input(str(c.quality_repeat), id="cfg-repeat", type="integer") + yield Label("Max tokens (0 = default):") + yield Input(str(c.max_tokens) if c.max_tokens > 0 else "", id="cfg-max-tokens", + type="integer", placeholder="auto") + yield Label("Thinking max tokens (0 = default):") + yield Input(str(c.thinking_max_tokens) if c.thinking_max_tokens > 0 else "", + id="cfg-thinking-max-tokens", type="integer", placeholder="auto") + + elif tt == TestType.SOAK: + yield Label("Mode:") + yield Select( + [("Fresh (cold start)", "fresh"), ("Continuous (warm)", "continuous"), ("Quick (short)", "quick")], + value=c.soak_mode, + id="cfg-mode" + ) + yield Label("Sessions:") + yield Input(str(c.soak_sessions), id="cfg-sessions", type="integer") + yield Label("Turns per session:") + yield Input(str(c.soak_turns), id="cfg-turns", type="integer") + yield Label("Max VRAM growth (MiB):") + yield Input(str(c.soak_max_growth), id="cfg-growth", type="integer") + + elif tt == TestType.REBENCH: + yield Label("8-pack thinking:") + yield Select( + [("None (fast gates only)", ""), ("Off", "off"), ("On", "on"), ("Both (promotion gate)", "both")], + value=c.rebench_8pack, + id="cfg-8pack" + ) + yield Label("Skip steps (CSV: verify-full,bench,...):") + yield Input(",".join(c.rebench_skip), id="cfg-skip", placeholder="e.g., bench,soak") + yield Label("Resume:") + yield Select(bool_options, value=c.rebench_resume, id="cfg-resume") + yield Label("Tag (empty = auto):") + yield Input(c.rebench_tag, id="cfg-tag", placeholder="auto-generated") + yield Label("SOAK_SESSIONS:") + yield Input(str(c.soak_sessions), id="cfg-sessions", type="integer") + yield Label("Max tokens (0 = default):") + yield Input(str(c.max_tokens) if c.max_tokens > 0 else "", id="cfg-max-tokens", + type="integer", placeholder="auto") + yield Label("Thinking max tokens (0 = default):") + yield Input(str(c.thinking_max_tokens) if c.thinking_max_tokens > 0 else "", + id="cfg-thinking-max-tokens", type="integer", placeholder="auto") + + def _read_config(self) -> TestConfig: + """Read values from widgets into config.""" + c = TestConfig(test_type=self.test_entry.test_type) + tt = self.test_entry.test_type + + def get_select_val(id: str, default): + try: + widget = self.query_one(f"#{id}", Select) + val = widget.value + return val if val != Select.BLANK else default + except Exception: + return default + + def get_input_val(id: str, default: str = "") -> str: + try: + return self.query_one(f"#{id}", Input).value + except Exception: + return default + + def get_int(id: str, default: int = 0) -> int: + try: + val = get_input_val(id, str(default)) + return int(val) if val else default + except ValueError: + return default + + if tt == TestType.BENCH: + c.run_count = get_int("cfg-runs", 5) + c.warmups = get_int("cfg-warmups", 3) + c.only = get_select_val("cfg-only", "both") + c.enable_thinking = get_select_val("cfg-thinking", False) + + elif tt == TestType.VERIFY_FULL: + c.skip_tools = get_select_val("cfg-skip-tools", False) + c.run_bench = get_select_val("cfg-run-bench", False) + + elif tt == TestType.VERIFY_STRESS: + c.skip_longctx = get_select_val("cfg-skip-longctx", False) + c.skip_tool_prefill = get_select_val("cfg-skip-prefill", False) + c.skip_ceiling = get_select_val("cfg-skip-ceiling", False) + + elif tt == TestType.QUALITY: + c.quality_tier = get_select_val("cfg-tier", "medium") + c.quality_pack = get_input_val("cfg-pack") + c.enable_thinking = get_select_val("cfg-thinking", False) + c.quality_repeat = get_int("cfg-repeat", 1) + c.max_tokens = get_int("cfg-max-tokens", 0) + c.thinking_max_tokens = get_int("cfg-thinking-max-tokens", 0) + + elif tt == TestType.SOAK: + c.soak_mode = get_select_val("cfg-mode", "fresh") + c.soak_sessions = get_int("cfg-sessions", 10) + c.soak_turns = get_int("cfg-turns", 5) + c.soak_max_growth = get_int("cfg-growth", 200) + + elif tt == TestType.REBENCH: + c.rebench_8pack = get_select_val("cfg-8pack", "") + skip_csv = get_input_val("cfg-skip") + c.rebench_skip = [s.strip() for s in skip_csv.split(",") if s.strip()] + c.rebench_resume = get_select_val("cfg-resume", False) + c.rebench_tag = get_input_val("cfg-tag") + c.soak_sessions = get_int("cfg-sessions", 10) + c.max_tokens = get_int("cfg-max-tokens", 0) + c.thinking_max_tokens = get_int("cfg-thinking-max-tokens", 0) + + return c + + async def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == "btn-run": + config = self._read_config() + self.dismiss(config) + elif event.button.id == "btn-cancel": + self.dismiss(None) + + def action_cancel(self) -> None: + self.dismiss(None) + + +class HelpScreen(ModalScreen): + """Help overlay showing keybindings.""" + + DEFAULT_CSS = """ + HelpScreen { + align: center middle; + } + HelpScreen > Vertical { + width: 70; + height: auto; + border: thick $accent; + background: $surface; + padding: 1 2; + } + HelpScreen .help-title { + text-style: bold; + color: $accent; + text-align: center; + margin-bottom: 1; + } + """ + + BINDINGS = [ + Binding("escape", "dismiss", "Close"), + Binding("question_mark", "dismiss", "Close"), + ] + + HELP_TEXT = """ +[bold]Keybindings[/bold] + + [cyan]↑/↓ or j/k[/cyan] Move selection in test menu + [cyan]Enter[/cyan] Run selected test (default config) + [cyan]c[/cyan] Configure selected test + [cyan]x[/cyan] Stop current run + [cyan]r[/cyan] Re-detect serving target + [cyan]m[/cyan] Manual target override + [cyan]f[/cyan] Toggle log follow/scroll-lock + [cyan]Tab[/cyan] Cycle pane focus + [cyan]1/2/3[/cyan] Jump to pane (Target/Tests/Live) + [cyan]?[/cyan] Show this help + [cyan]q[/cyan] Quit + +[bold]Status glyphs[/bold] + + [green]✓[/green] passed [red]✗[/red] failed [yellow]△[/yellow] partial/recall-miss + [yellow]⊘[/yellow] skipped [cyan]▶[/cyan] running ◔ queued +""" + + def compose(self) -> ComposeResult: + with Vertical(): + yield Label("club3090 test console — Help", classes="help-title") + yield Static(self.HELP_TEXT) + + def action_dismiss(self) -> None: + self.app.pop_screen() + + +class QuitConfirmScreen(ModalScreen[bool]): + """Confirm quit when a test is running. Offers stop & quit or cancel.""" + + DEFAULT_CSS = """ + QuitConfirmScreen { + align: center middle; + } + QuitConfirmScreen > Vertical { + width: 60; + height: auto; + border: thick $warning; + background: $surface; + padding: 1 2; + } + QuitConfirmScreen .quit-title { + text-style: bold; + color: $warning; + text-align: center; + margin-bottom: 1; + } + QuitConfirmScreen .quit-info { + color: $text-muted; + text-align: center; + margin-bottom: 1; + } + QuitConfirmScreen Button { + margin: 1 1; + width: 1fr; + } + """ + + BINDINGS = [ + Binding("escape", "cancel", "Cancel"), + ] + + def __init__(self, test_name: str): + super().__init__() + self.test_name = test_name + + def compose(self) -> ComposeResult: + with Vertical(): + yield Label("⚠ Test Running", classes="quit-title") + yield Label(f"{self.test_name} is still running.", classes="quit-info") + yield Label("Choose an action:", classes="quit-info") + with Horizontal(): + yield Button("Stop & Quit", variant="error", id="btn-stop-quit") + yield Button("Cancel", variant="default", id="btn-cancel") + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == "btn-stop-quit": + self.dismiss(True) # stop and quit + elif event.button.id == "btn-cancel": + self.dismiss(False) # cancel + + def action_cancel(self) -> None: + self.dismiss(False) + + +class TestConsoleApp(App): + """The main club3090 test console application.""" + + TITLE = "club3090 test console" + CSS_PATH = "app.tcss" + + BINDINGS = [ + Binding("q", "safe_quit", "Quit", show=True), + Binding("question_mark", "help", "Help", show=True), + Binding("r", "redetect", "Re-detect", show=True), + Binding("c", "config", "Configure", show=True), + Binding("x", "stop", "Stop", show=True), + Binding("h", "history", "History", show=True), + Binding("f", "toggle_follow", "Follow", show=False), + Binding("m", "manual_target", "Target", show=True), + Binding("enter", "run_test", "Run", show=True), + ] + + def __init__(self, repo_root: Path, **kwargs): + super().__init__(**kwargs) + self.repo_root = repo_root + self.target: Optional[ServingTarget] = None + self.registry_variants: list[dict] = [] + self.runner = TestRunner(repo_root) + self._gpu_refresh_task: Optional[asyncio.Task] = None + self._state_dir = Path.home() / ".local" / "state" / "club3090-test-console" + + def compose(self) -> ComposeResult: + yield Header(show_clock=True) + with Horizontal(id="main-layout"): + with Vertical(id="left-rail"): + yield TargetPane(id="target-pane") + yield TestMenuPane(id="test-menu") + yield LivePane(id="live-pane") + yield Footer() + + CSS = """ + #main-layout { + height: 1fr; + } + #left-rail { + width: 38; + height: 1fr; + } + #target-pane { + height: auto; + min-height: 8; + } + #test-menu { + height: 1fr; + } + #live-pane { + width: 1fr; + height: 1fr; + } + """ + + async def on_mount(self) -> None: + """Initialize: detect target, load registry, set up runner.""" + self.runner.set_callbacks( + on_event=self._on_run_event, + on_line=self._on_run_line, + on_complete=self._on_run_complete, + ) + # Ensure state dir exists + self._state_dir.mkdir(parents=True, exist_ok=True) + + # Start detection in background + asyncio.create_task(self._initial_detect()) + + # Set up elapsed timer update + self.set_interval(1.0, self._update_elapsed_timer) + + async def on_unmount(self) -> None: + """Clean up background tasks on app shutdown.""" + if self._gpu_refresh_task and not self._gpu_refresh_task.done(): + self._gpu_refresh_task.cancel() + # Don't await - just cancel to avoid event loop cleanup issues + + async def _initial_detect(self) -> None: + """Detect serving target and load registry.""" + # Detect endpoint + self.target = await detect_endpoint() + + # Load registry for enrichment + self.registry_variants = await detect_from_registry(str(self.repo_root)) + if self.target and self.registry_variants: + self.target = match_target_to_registry(self.target, self.registry_variants) + + # Update the target pane + self._update_target_pane() + + # Cancel old GPU refresh loop if exists, then start new one + if self._gpu_refresh_task and not self._gpu_refresh_task.done(): + self._gpu_refresh_task.cancel() + try: + await self._gpu_refresh_task + except asyncio.CancelledError: + pass + self._gpu_refresh_task = asyncio.create_task(self._gpu_refresh_loop()) + + async def _gpu_refresh_loop(self) -> None: + """Periodically refresh GPU stats.""" + try: + while True: + await asyncio.sleep(2) + if self.target and self.target.is_active: + self.target.gpus = await get_gpu_info() + self._update_target_pane() + except asyncio.CancelledError: + pass + + def _update_elapsed_timer(self) -> None: + """Update the elapsed timer in the live pane.""" + try: + live = self.query_one("#live-pane", LivePane) + live.update_elapsed_timer() + except Exception: + pass + + def _update_target_pane(self) -> None: + """Update the target pane with current target info.""" + try: + pane = self.query_one("#target-pane", TargetPane) + pane.target = self.target + except Exception: + pass + + def _on_run_event(self, event: ParseEvent) -> None: + """Called when the runner parses a structured event.""" + # Capture test_type at scheduling time before current_run may be cleared + test_type = self.runner.current_run.test_type if self.runner.current_run else None + self.call_later(self._handle_run_event, event, test_type) + + def _handle_run_event(self, event: ParseEvent, test_type: TestType | None = None) -> None: + """Handle a run event on the UI thread.""" + try: + live = self.query_one("#live-pane", LivePane) + if test_type: + live.process_event(event, test_type) + except Exception: + pass + + def _on_run_line(self, line: str) -> None: + """Called for every stdout line from the runner.""" + self.call_later(self._handle_run_line, line) + + def _handle_run_line(self, line: str) -> None: + """Handle a raw log line on the UI thread.""" + try: + live = self.query_one("#live-pane", LivePane) + live.append_line(line) + except Exception: + pass + + def _on_run_complete(self, state: RunState) -> None: + """Called when a test run completes.""" + self.call_later(self._handle_run_complete, state) + + def _handle_run_complete(self, state: RunState) -> None: + """Handle run completion on the UI thread.""" + # Update menu status + try: + menu = self.query_one("#test-menu", TestMenuPane) + status = "passed" if state.verdict == "passed" else "failed" + menu.set_status(state.test_type, status) + except Exception: + pass + + # Save run record + self._save_run_record(state) + + # Show completion in log + try: + live = self.query_one("#live-pane", LivePane) + elapsed = state.elapsed_s + if state.verdict == "passed": + live.append_line(f"\n[bold green]✓ {state.test_type.value} passed in {elapsed:.0f}s[/bold green]") + else: + live.append_line(f"\n[bold red]✗ {state.test_type.value} failed (rc={state.exit_code}) in {elapsed:.0f}s[/bold red]") + if state.report_path: + live.append_line(f" report: {state.report_path}") + if state.artifact_dir: + live.append_line(f" artifacts: {state.artifact_dir}") + except Exception: + pass + + def _save_run_record(self, state: RunState) -> None: + """Persist a run record to disk.""" + runs_dir = self._state_dir / "runs" + runs_dir.mkdir(parents=True, exist_ok=True) + + record = { + "test": state.test_type.value, + "model": state.target.model if state.target else "", + "url": state.target.url if state.target else "", + "slug": state.target.slug if state.target else "", + "started": state.started, + "finished": state.finished, + "exit_code": state.exit_code, + "verdict": state.verdict, + "elapsed_s": state.elapsed_s, + "artifact_dir": state.artifact_dir, + "report_path": state.report_path, + "power_cap_w": ( + state.target.gpus[0].power_limit_w + if state.target and state.target.gpus + else None + ), + } + + ts = time.strftime("%Y%m%d-%H%M%S", time.localtime(state.started)) + path = runs_dir / f"{ts}-{state.test_type.value}.json" + try: + path.write_text(json.dumps(record, indent=2)) + except Exception: + pass + + # ── Actions ────────────────────────────────────────────────────────── + + async def action_run_test(self) -> None: + """Run the selected test.""" + menu = self.query_one("#test-menu", TestMenuPane) + entry = menu.get_selected_entry() + if not entry: + return + + # Check if a run is active + if self.runner.current_run: + self.notify("A test is already running. Press x to stop first.", severity="warning") + return + + # Check target + if not self.target or not self.target.is_active: + self.notify("No model serving. Start one with `gpu-mode `.", severity="error") + return + + config = TestConfig(test_type=entry.test_type) + await self._start_run(config) + + async def action_config(self) -> None: + """Open config for the selected test.""" + menu = self.query_one("#test-menu", TestMenuPane) + entry = menu.get_selected_entry() + if not entry: + return + + def on_config_dismiss(config: Optional[TestConfig]) -> None: + if config: + self.run_worker(self._start_run(config)) + + self.push_screen(ConfigScreen(entry), callback=on_config_dismiss) + + async def _start_run(self, config: TestConfig) -> None: + """Start a test run with the given config.""" + # Guard against starting a second run while one is already active + if self.runner.current_run: + self.notify("A test is already running. Press x to stop first.", severity="warning") + return + + if not self.target: + self.notify("No serving target detected.", severity="error") + return + + # Update menu status + try: + menu = self.query_one("#test-menu", TestMenuPane) + menu.set_status(config.test_type, "running") + except Exception: + pass + + # Set up live pane + try: + live = self.query_one("#live-pane", LivePane) + live.set_run_header(config.test_type, self.target.model) + except Exception: + pass + + # Start the run + await self.runner.start(config, self.target) + + async def action_stop(self) -> None: + """Stop the current run.""" + if self.runner.current_run: + test_type = self.runner.current_run.test_type + orphans = await self.runner.cancel() + + if orphans: + orphan_names = ' '.join(orphans) + self.notify( + f"Test cancelled. ⚠ Orphaned benchlocal containers: {', '.join(orphans)}. " + f"Run: docker rm -f {orphan_names}", + severity="warning", + timeout=10, + ) + else: + self.notify("Test cancelled.", severity="warning") + + try: + menu = self.query_one("#test-menu", TestMenuPane) + menu.set_status(test_type, "idle") + except Exception: + pass + else: + self.notify("No active run.", severity="information") + + async def action_redetect(self) -> None: + """Re-detect the serving target.""" + self.notify("Re-detecting...", severity="information") + await self._initial_detect() + if self.target and self.target.is_active: + self.notify(f"Found: {self.target.model} on :{self.target.host_port}", severity="information") + else: + self.notify("No model serving.", severity="warning") + + def action_help(self) -> None: + """Show the help screen.""" + self.push_screen(HelpScreen()) + + def action_toggle_follow(self) -> None: + """Toggle log follow mode.""" + try: + live = self.query_one("#live-pane", LivePane) + live.toggle_follow() + follow_state = "on" if live._follow else "off" + self.notify(f"Log follow: {follow_state}", severity="information") + except Exception: + pass + + async def action_manual_target(self) -> None: + """Open manual target override.""" + variants = self.registry_variants + if not variants: + variants = await detect_from_registry(str(self.repo_root)) + + def on_target_dismiss(result: Optional[ServingTarget]) -> None: + if result: + self.target = result + self._update_target_pane() + self.notify(f"Target set: {result.model} @ {result.url}", severity="information") + + self.push_screen(ManualTargetScreen(str(self.repo_root), variants), callback=on_target_dismiss) + + def action_history(self) -> None: + """Open run history view.""" + self.push_screen(HistoryScreen(self._state_dir, self.repo_root)) + + async def action_safe_quit(self) -> None: + """Quit with confirmation if a test is running.""" + if not self.runner.current_run: + # No active run, just quit + self.exit() + return + + # Show confirmation dialog + test_name = self.runner.current_run.test_type.value + + def on_quit_choice(stop_and_quit: bool) -> None: + if stop_and_quit: + # Stop & quit + self.run_worker(self._stop_and_quit()) + # else: cancel (do nothing) + + self.push_screen(QuitConfirmScreen(test_name), callback=on_quit_choice) + + async def _stop_and_quit(self) -> None: + """Stop the current run and quit.""" + if self.runner.current_run: + await self.runner.cancel() + self.exit() diff --git a/tools/test-console/club3090_test_console/app.tcss b/tools/test-console/club3090_test_console/app.tcss new file mode 100644 index 00000000..97b60079 --- /dev/null +++ b/tools/test-console/club3090_test_console/app.tcss @@ -0,0 +1,34 @@ +/* club3090 test console — Textual CSS */ + +/* Dark theme by default */ +Screen { + background: $surface; +} + +#main-layout { + height: 1fr; +} + +#left-rail { + width: 38; + height: 1fr; + min-width: 32; + max-width: 42; +} + +#live-pane { + width: 1fr; + height: 1fr; +} + +/* Pane borders — accent on focus */ +TargetPane:focus-within, +TestMenuPane:focus-within, +LivePane:focus-within { + border: solid $accent; +} + +/* Footer customization */ +Footer { + dock: bottom; +} diff --git a/tools/test-console/club3090_test_console/detect.py b/tools/test-console/club3090_test_console/detect.py new file mode 100644 index 00000000..b46db550 --- /dev/null +++ b/tools/test-console/club3090_test_console/detect.py @@ -0,0 +1,279 @@ +"""Auto-detection of serving model + endpoint. + +Replicates the logic from scripts/preflight.sh::preflight_autodetect_endpoint +and preflight_autodetect_model. +""" + +from __future__ import annotations + +import asyncio +import re +from dataclasses import dataclass, field +from typing import Optional + +import httpx + +# Engine-internal ports: 8000=vLLM, 8080=llama.cpp, 30000=SGLang +ENGINE_INTERNAL_PORTS = {"8000", "8080", "30000"} + +# Recognized engine-family container prefixes +ENGINE_PREFIXES = re.compile(r"^(vllm-|llama-cpp-|ik-llama-|sglang-|beellama-)") + +# Port mapping regex: matches 0.0.0.0:8011->8000/tcp, [::]:8011->8000/tcp, 127.0.0.1:8011->8000/tcp +PORT_MAP_RE = re.compile( + r"(?:[0-9]{1,3}(?:\.[0-9]{1,3}){3}|\[::\]):(\d+)->(8000|8080|30000)/tcp" +) + +# All known engine-port patterns (also match without IP prefix) +PORT_MAP_BROAD_RE = re.compile( + r":(\d+)->(8000|8080|30000)/tcp" +) + + +@dataclass +class GpuInfo: + """Information about a single GPU.""" + index: int + utilization: int = 0 # % + mem_used_mib: int = 0 + mem_total_mib: int = 0 + power_draw_w: float = 0.0 + power_limit_w: float = 0.0 + temp_c: int = 0 + + +@dataclass +class ServingTarget: + """Resolved serving target — model + endpoint + metadata.""" + url: str = "" + model: str = "" + container: str = "" + engine: str = "" # vllm | llamacpp | ik-llama | sglang | beellama | unknown + host_port: int = 0 + internal_port: int = 0 + slug: str = "" # registry slug if matched + kv_format: str = "" + max_ctx: int = 0 + tp: int = 0 + status: str = "" # registry status + status_note: str = "" + health: str = "unknown" # serving | unreachable | multiple + gpus: list[GpuInfo] = field(default_factory=list) + + @property + def is_localhost(self) -> bool: + return "localhost" in self.url or "127." in self.url or "[::1]" in self.url + + @property + def is_active(self) -> bool: + return bool(self.url and self.model and self.health == "serving") + + +def _classify_engine(internal_port: str) -> str: + """Map internal port to engine family.""" + return {"8000": "vllm", "8080": "llamacpp", "30000": "sglang"}.get(internal_port, "unknown") + + +def _classify_engine_from_container(name: str) -> str: + """Refine engine from container name prefix.""" + if name.startswith("vllm-"): + return "vllm" + if name.startswith("llama-cpp-") or name.startswith("ik-llama-"): + return "llamacpp" + if name.startswith("sglang-"): + return "sglang" + if name.startswith("beellama-"): + return "beellama" + return "unknown" + + +async def detect_endpoint(container_name: Optional[str] = None) -> ServingTarget: + """Detect the currently-serving model and endpoint. + + Args: + container_name: If set, detect only from this container. + + Returns: + A ServingTarget with whatever was resolved. + """ + target = ServingTarget() + + # Step 1: docker ps + try: + proc = await asyncio.create_subprocess_exec( + "docker", "ps", "--format", "{{.Names}}|{{.Ports}}", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=10) + lines = stdout.decode().strip().split("\n") + except (asyncio.TimeoutError, FileNotFoundError, Exception): + target.health = "unreachable" + return target + + # Step 2: find inference containers + # Filter to recognized engine prefixes FIRST to exclude Open WebUI and other non-inference containers + candidates: list[tuple[str, int, int, str]] = [] # (name, host_port, internal_port, engine) + seen: set[tuple[str, int]] = set() # dedupe by (container_name, host_port) for dual-stack + + for line in lines: + if "|" not in line: + continue + name, ports_str = line.split("|", 1) + + # Only consider recognized engine containers + if not ENGINE_PREFIXES.match(name): + continue + + for match in PORT_MAP_BROAD_RE.finditer(ports_str): + host_port = int(match.group(1)) + internal_port = int(match.group(2)) + + # Dedupe dual-stack mappings (same container, same host port) + key = (name, host_port) + if key in seen: + continue + seen.add(key) + + engine = _classify_engine_from_container(name) + if engine == "unknown": + engine = _classify_engine(str(internal_port)) + candidates.append((name, host_port, internal_port, engine)) + + if not candidates: + target.health = "unreachable" + return target + + # If caller pinned a container, filter to it + if container_name: + candidates = [c for c in candidates if c[0] == container_name] + if not candidates: + target.health = "unreachable" + return target + + # Step 3: prefer recognized engine prefix; else first match + preferred = [c for c in candidates if ENGINE_PREFIXES.match(c[0])] + chosen = preferred[0] if preferred else candidates[0] + + name, host_port, internal_port, engine = chosen + target.container = name + target.host_port = host_port + target.internal_port = internal_port + target.engine = engine + target.url = f"http://localhost:{host_port}" + + # Check for truly different containers (not just multiple ports on same container) + unique_containers = set(c[0] for c in candidates) + if len(unique_containers) > 1: + target.health = "multiple" + else: + target.health = "serving" + + # Step 4: probe /v1/models + try: + async with httpx.AsyncClient(timeout=5.0) as client: + resp = await client.get(f"{target.url}/v1/models") + if resp.status_code == 200: + data = resp.json().get("data", []) + if data: + target.model = data[0].get("id", "") + if target.health == "multiple": + pass # keep multiple + else: + target.health = "serving" + else: + target.health = "unreachable" + except Exception: + target.health = "unreachable" + + # Step 5: GPU info + target.gpus = await get_gpu_info() + + return target + + +async def get_gpu_info() -> list[GpuInfo]: + """Query nvidia-smi for GPU stats.""" + try: + proc = await asyncio.create_subprocess_exec( + "nvidia-smi", + "--query-gpu=index,utilization.gpu,memory.used,memory.total,power.draw,power.limit,temperature.gpu", + "--format=csv,noheader,nounits", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=10) + gpus = [] + for line in stdout.decode().strip().split("\n"): + parts = [p.strip() for p in line.split(",")] + if len(parts) >= 7: + gpus.append(GpuInfo( + index=int(parts[0]), + utilization=int(float(parts[1])), + mem_used_mib=int(float(parts[2])), + mem_total_mib=int(float(parts[3])), + power_draw_w=float(parts[4]), + power_limit_w=float(parts[5]), + temp_c=int(float(parts[6])), + )) + return gpus + except Exception: + return [] + + +async def detect_from_registry(repo_root: str) -> list[dict]: + """Enumerate registry variants via registry-emit.sh. + + Returns list of dicts with keys: slug, engine, port, model, status, etc. + """ + try: + cmd = f'source "{repo_root}/scripts/lib/registry-emit.sh" && registry_variant_rows "{repo_root}"' + proc = await asyncio.create_subprocess_exec( + "bash", "-c", cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=repo_root, + ) + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=15) + variants = [] + for line in stdout.decode().strip().split("\n"): + parts = line.split("\t") + if len(parts) >= 13 and parts[0] == "VARIANT": + variants.append({ + "slug": parts[1], + "switch_engine": parts[2], + "launch_engine": parts[3], + "compose_dir": parts[4], + "file": parts[5], + "port": int(parts[6]) if parts[6].isdigit() else 0, + "model": parts[7], + "engine": parts[8], + "kvcalc_key": parts[9], + "container": parts[10], + "compose_path": parts[11], + "status": parts[12], + "ctx_label": parts[13] if len(parts) > 13 else "", + "status_note": parts[14] if len(parts) > 14 else "", + }) + return variants + except Exception: + return [] + + +def match_target_to_registry(target: ServingTarget, variants: list[dict]) -> ServingTarget: + """Enrich a ServingTarget with registry metadata by matching port/container.""" + for v in variants: + # Match by container name first, then port + if target.container and v.get("container", "").replace("_", "-") in target.container: + target.slug = v["slug"] + target.kv_format = v.get("kvcalc_key", "") + target.status = v.get("status", "") + target.status_note = v.get("status_note", "") + return target + if v["port"] == target.host_port: + target.slug = v["slug"] + target.kv_format = v.get("kvcalc_key", "") + target.status = v.get("status", "") + target.status_note = v.get("status_note", "") + return target + return target diff --git a/tools/test-console/club3090_test_console/parsers.py b/tools/test-console/club3090_test_console/parsers.py new file mode 100644 index 00000000..c266ffd7 --- /dev/null +++ b/tools/test-console/club3090_test_console/parsers.py @@ -0,0 +1,609 @@ +"""Output parsers for each test script. + +Each parser is a stateful class that processes lines and emits structured events. +Regex patterns are derived from the actual script output formats (Section 10 of spec). +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Optional + + +class TestType(str, Enum): + VERIFY = "verify" + VERIFY_FULL = "verify-full" + BENCH = "bench" + VERIFY_STRESS = "verify-stress" + QUALITY = "quality" + SOAK = "soak" + REBENCH = "rebench-full" + + +class Status(str, Enum): + QUEUED = "queued" + RUNNING = "running" + PASSED = "passed" + FAILED = "failed" + PARTIAL = "partial" + SKIPPED = "skipped" + + +def strip_ansi(text: str) -> str: + """Strip ANSI/SGR escape codes.""" + return re.sub(r"\x1b\[[0-9;]*m", "", text) + + +@dataclass +class ParseEvent: + """A structured event parsed from a test output line.""" + event_type: str # e.g., "bench_run", "summary_metric", "niah_rung", "verdict" + data: dict[str, Any] = field(default_factory=dict) + raw_line: str = "" + + +# ============================================================================ +# bench.sh parser +# ============================================================================ + +class BenchParser: + """Parser for bench.sh output.""" + + # Section: ========== NARRATIVE (prompt=65 chars, max_tokens=1000) ========== + SECTION_RE = re.compile(r"^========== (\w+) \(prompt=") + + # Warmup/measured: run-1 wall= 11.53s ttft= 118ms toks=1000 wall_TPS= 86.68 decode_TPS= 87.58 + RUN_RE = re.compile( + r"^\s+(warm|run)-(\d+)\s+wall=\s*([\d.]+)s\s+ttft=\s*(\d+)ms\s+" + r"toks=\s*(\d+)\s+wall_TPS=\s*([\d.]+)\s+decode_TPS=\s*([\d.]+)" + ) + + # Summary: wall_TPS mean= 84.14 std= 2.41 CV= 2.9% + SUMMARY_RE = re.compile( + r"^\s+(wall_TPS|decode_TPS)\s+mean=\s*([\d.]+).*CV=\s*([\d.]+)%" + ) + + # TTFT summary + TTFT_RE = re.compile(r"^\s+TTFT\s+mean=\s*(\d+)ms") + + # Spec decoding metrics + SPEC_DEC_RE = re.compile(r"^=== Last \d+ SpecDecoding metrics ===") + + def __init__(self): + self.current_section: Optional[str] = None + self.runs: dict[str, list[dict]] = {"narrative": [], "code": []} + self.summary: dict[str, dict] = {} + self.total_runs: int = 0 + self.warmups: int = 0 + self.measured_runs: int = 0 + + def parse_line(self, line: str) -> Optional[ParseEvent]: + """Parse a single line and return an event if matched.""" + clean = strip_ansi(line) + + # Section header + m = self.SECTION_RE.match(clean) + if m: + self.current_section = m.group(1).lower() + return ParseEvent("bench_section", {"section": self.current_section}, line) + + # Run line (warmup or measured) + m = self.RUN_RE.match(clean) + if m: + run_type, run_num = m.group(1), int(m.group(2)) + data = { + "type": run_type, + "run": run_num, + "wall_s": float(m.group(3)), + "ttft_ms": int(m.group(4)), + "tokens": int(m.group(5)), + "wall_tps": float(m.group(6)), + "decode_tps": float(m.group(7)), + "section": self.current_section, + } + if self.current_section and run_type == "run": + self.runs[self.current_section].append(data) + return ParseEvent("bench_run", data, line) + + # Summary metric + m = self.SUMMARY_RE.match(clean) + if m: + metric, mean, cv = m.group(1), float(m.group(2)), float(m.group(3)) + data = {"metric": metric, "mean": mean, "cv": cv, "section": self.current_section} + if self.current_section: + self.summary.setdefault(self.current_section, {})[metric] = data + return ParseEvent("summary_metric", data, line) + + # TTFT summary + m = self.TTFT_RE.match(clean) + if m: + data = {"ttft_mean_ms": int(m.group(1)), "section": self.current_section} + if self.current_section: + self.summary.setdefault(self.current_section, {})["ttft"] = data + return ParseEvent("summary_ttft", data, line) + + # Spec decoding header + if self.SPEC_DEC_RE.match(clean): + return ParseEvent("spec_dec_header", {}, line) + + return None + + +# ============================================================================ +# verify.sh / verify-full.sh parser +# ============================================================================ + +class VerifyParser: + """Parser for verify.sh and verify-full.sh output.""" + + # [3/9] Basic completion — capital of France ... + STEP_RE = re.compile(r"^\[(\d+)/(\d+)\] (.+?) \.\.\.") + + # ✓ reply contains 'Paris' + # ✗ tool-call request failed + # ⊘ Genesis patches applied (skipped) + CHECK_RE = re.compile(r"^\s+([✓✗⊘]) (.+)") + + # → Check docker logs vllm-qwen36-27b + HINT_RE = re.compile(r"^\s+→ (.+)") + + # All checks passed. + PASS_RE = re.compile(r"^All checks passed\.") + + # 3 check(s) failed. + FAIL_RE = re.compile(r"^(\d+) check\(s\) failed\.") + + def __init__(self): + self.steps: list[dict] = [] + self.current_step: Optional[dict] = None + self.total_steps: int = 0 + self.passed: int = 0 + self.failed: int = 0 + self.skipped: int = 0 + + def parse_line(self, line: str) -> Optional[ParseEvent]: + clean = strip_ansi(line) + + # Step header + m = self.STEP_RE.match(clean) + if m: + step_num, total, name = int(m.group(1)), int(m.group(2)), m.group(3) + self.total_steps = total + self.current_step = {"num": step_num, "total": total, "name": name, "checks": []} + self.steps.append(self.current_step) + return ParseEvent("verify_step", {"step": step_num, "total": total, "name": name}, line) + + # Check result + m = self.CHECK_RE.match(clean) + if m: + glyph, msg = m.group(1), m.group(2) + status = {"✓": "passed", "✗": "failed", "⊘": "skipped"}[glyph] + if status == "passed": + self.passed += 1 + elif status == "failed": + self.failed += 1 + else: + self.skipped += 1 + + check = {"glyph": glyph, "message": msg, "status": status, "hint": ""} + if self.current_step: + self.current_step["checks"].append(check) + return ParseEvent("verify_check", check, line) + + # Hint + m = self.HINT_RE.match(clean) + if m and self.current_step and self.current_step["checks"]: + hint = m.group(1) + self.current_step["checks"][-1]["hint"] = hint + return ParseEvent("verify_hint", {"hint": hint}, line) + + # Final verdict + if self.PASS_RE.match(clean): + return ParseEvent("verdict", {"status": Status.PASSED, "message": "All checks passed"}, line) + + m = self.FAIL_RE.match(clean) + if m: + count = int(m.group(1)) + return ParseEvent("verdict", {"status": Status.FAILED, "failed": count, "message": f"{count} check(s) failed"}, line) + + return None + + +# ============================================================================ +# verify-stress.sh parser (incl. NIAH) +# ============================================================================ + +class StressParser: + """Parser for verify-stress.sh output.""" + + # [1/8] Long-context needle small rungs (10K / 30K) ... + PROBE_RE = re.compile(r"^\[(\d+)/8\] (.+?) \.\.\.") + + # ✓ 10000 tokens: recalled '…' (got: …) + # △ 30000 tokens: recall MISS (…) — system OK, quality ceiling reached + TOKEN_RE = re.compile(r"^\s+([✓△✗⊘])\s+(\d+) tokens:") + + # ✓ rung 1/6: target=95K actual=95K tok (36%) recalled '…' prefill=… t/s (…s) VRAM_free=…MB + RUNG_RE = re.compile( + r"^\s+([✓△✗⊘]) rung (\d+)/(\d+): target=(\d+)K" + ) + + # n_ctx=262000 ladder: 95000 → 125000 → ... + LADDER_RE = re.compile(r"^\s+n_ctx=(\d+)\s+ladder:") + + # VRAM free (ladder start): 12345 MB + VRAM_FREE_RE = re.compile(r"^\s+VRAM free \(ladder start\): (\d+) MB") + + # All stress / boundary checks passed. + PASS_RE = re.compile(r"^All stress") + + # 3 stress check(s) failed. + FAIL_RE = re.compile(r"^(\d+) stress check") + + def __init__(self): + self.probes: list[dict] = [] + self.current_probe: Optional[dict] = None + self.niah_results: list[dict] = [] # rung/token results + self.ladder_info: dict = {} + + def parse_line(self, line: str) -> Optional[ParseEvent]: + clean = strip_ansi(line) + + # Probe header + m = self.PROBE_RE.match(clean) + if m: + probe_num, name = int(m.group(1)), m.group(2) + self.current_probe = {"num": probe_num, "name": name, "results": []} + self.probes.append(self.current_probe) + return ParseEvent("stress_probe", {"probe": probe_num, "name": name}, line) + + # Token-level result (probes 1, 7) + m = self.TOKEN_RE.match(clean) + if m: + glyph, tokens = m.group(1), int(m.group(2)) + status = {"✓": "passed", "△": "partial", "✗": "failed", "⊘": "skipped"}[glyph] + result = {"tokens": tokens, "status": status, "glyph": glyph} + self.niah_results.append(result) + if self.current_probe: + self.current_probe["results"].append(result) + return ParseEvent("niah_token", result, line) + + # Rung-level result (probe 8 ceiling ladder) + m = self.RUNG_RE.match(clean) + if m: + glyph, rung, total, target_k = m.group(1), int(m.group(2)), int(m.group(3)), int(m.group(4)) + status = {"✓": "passed", "△": "partial", "✗": "failed", "⊘": "skipped"}[glyph] + result = {"rung": rung, "total": total, "target_k": target_k, "status": status, "glyph": glyph} + self.niah_results.append(result) + if self.current_probe: + self.current_probe["results"].append(result) + return ParseEvent("niah_rung", result, line) + + # Ladder info + m = self.LADDER_RE.match(clean) + if m: + self.ladder_info["n_ctx"] = int(m.group(1)) + return ParseEvent("niah_ladder", {"n_ctx": int(m.group(1))}, line) + + # VRAM free + m = self.VRAM_FREE_RE.match(clean) + if m: + self.ladder_info["vram_free_mb"] = int(m.group(1)) + return ParseEvent("niah_vram", {"vram_free_mb": int(m.group(1))}, line) + + # Verdicts + if self.PASS_RE.match(clean): + return ParseEvent("verdict", {"status": Status.PASSED, "message": "All stress checks passed"}, line) + + m = self.FAIL_RE.match(clean) + if m: + count = int(m.group(1)) + return ParseEvent("verdict", {"status": Status.FAILED, "failed": count, "message": f"{count} stress check(s) failed"}, line) + + return None + + +# ============================================================================ +# quality-test.sh parser +# ============================================================================ + +class QualityParser: + """Parser for quality-test.sh output.""" + + # [1/15] TC-01 ✓ passed (2.3s) + # [7/15] TC-07 ✗ verifier_fail (3.1s) + SCENARIO_RE = re.compile( + r"^\s+\[(\d+)/(\d+)\] (\S+) ([✓✗]) (\w+) \(([\d.]+)s\)" + ) + + # Pack ID prefix mapping + PACK_PREFIXES = { + "TC": "toolcall-15", + "IF": "instructfollow-15", + "SO": "structoutput-15", + "DE": "dataextract-15", + "RM": "reasonmath-15", + "BF": "bugfind-15", + "CL": "cli-40", + "HA": "hermesagent-20", + "HE": "humaneval-plus-30", + "LC": "lcb-v6-30", + "GS": "gsm-symbolic-30", + "GP": "gpqa-diamond", + } + + # TOTAL line + TOTAL_RE = re.compile(r"TOTAL.*?(\d+)/(\d+)") + + def __init__(self): + self.scenarios: list[dict] = [] + self.packs: dict[str, dict] = {} # pack_id -> {passed, total, scenarios} + self.total_passed: int = 0 + self.total_count: int = 0 + + def _pack_from_prefix(self, scenario_id: str) -> str: + prefix = scenario_id[:2].upper() + return self.PACK_PREFIXES.get(prefix, "unknown") + + def parse_line(self, line: str) -> Optional[ParseEvent]: + clean = strip_ansi(line) + + # Scenario result + m = self.SCENARIO_RE.match(clean) + if m: + num, total, scenario_id, glyph, failure_mode, elapsed = ( + int(m.group(1)), int(m.group(2)), m.group(3), + m.group(4), m.group(5), float(m.group(6)) + ) + passed = glyph == "✓" + pack_id = self._pack_from_prefix(scenario_id) + + result = { + "num": num, "total": total, + "scenario_id": scenario_id, + "passed": passed, + "failure_mode": failure_mode if not passed else "", + "elapsed_s": elapsed, + "pack_id": pack_id, + } + self.scenarios.append(result) + + # Update pack totals + pack = self.packs.setdefault(pack_id, {"passed": 0, "total": 0, "scenarios": []}) + pack["total"] += 1 + pack["scenarios"].append(result) + if passed: + pack["passed"] += 1 + self.total_passed += 1 + self.total_count += 1 + + return ParseEvent("quality_scenario", result, line) + + # TOTAL line + m = self.TOTAL_RE.match(clean) + if m: + self.total_passed = int(m.group(1)) + self.total_count = int(m.group(2)) + return ParseEvent("quality_total", { + "passed": self.total_passed, + "total": self.total_count, + }, line) + + return None + + +# ============================================================================ +# soak-test.sh parser +# ============================================================================ + +class SoakParser: + """Parser for soak-test.sh output.""" + + # [soak] mode=fresh sessions=20 turns=5 max_growth=200MiB timeout=1800s + MODE_RE = re.compile(r"^\[soak\] mode=(\w+) sessions=(\d+) turns=(\d+)") + + # [soak] session 1/20 + SESSION_RE = re.compile(r"^\[soak\] session (\d+)/(\d+)") + + # [soak] turn 1/5: status=200 wall=5159ms ttft=481ms decode_tps=42.113 vram=43104MiB + TURN_RE = re.compile( + r"^\[soak\]\s+turn (\d+)/(\d+): status=(\d+) wall=(\d+)ms " + r"ttft=(\d+)ms decode_tps=([\d.]+) vram=(\d+)MiB" + ) + + # [soak] verdict PASS + VERDICT_RE = re.compile(r"^\[soak\]\s+verdict\s+(PASS|FAIL)") + + # [soak] silent_empty 0 / 100 (0.0%) + METRIC_RE = re.compile(r"^\[soak\]\s+(silent_empty|tps_retention|p50_decode_tps|max_growth_mib)\s+(.+)") + + # [soak] warm baseline after session 1: 43104 MiB + BASELINE_RE = re.compile(r"^\[soak\] warm baseline after session (\d+): (\d+) MiB") + + def __init__(self): + self.mode: str = "" + self.total_sessions: int = 0 + self.total_turns: int = 0 + self.current_session: int = 0 + self.turns: list[dict] = [] + self.baseline_vram: int = 0 + self.verdict: Optional[str] = None + self.metrics: dict = {} + + def parse_line(self, line: str) -> Optional[ParseEvent]: + clean = strip_ansi(line) + + # Mode/config line + m = self.MODE_RE.match(clean) + if m: + self.mode = m.group(1) + self.total_sessions = int(m.group(2)) + self.total_turns = int(m.group(3)) + return ParseEvent("soak_config", { + "mode": self.mode, + "sessions": self.total_sessions, + "turns": self.total_turns, + }, line) + + # Session start + m = self.SESSION_RE.match(clean) + if m: + self.current_session = int(m.group(1)) + return ParseEvent("soak_session", { + "session": self.current_session, + "total": int(m.group(2)), + }, line) + + # Turn result + m = self.TURN_RE.match(clean) + if m: + turn_data = { + "turn": int(m.group(1)), + "total": int(m.group(2)), + "status": int(m.group(3)), + "wall_ms": int(m.group(4)), + "ttft_ms": int(m.group(5)), + "decode_tps": float(m.group(6)), + "vram_mib": int(m.group(7)), + "session": self.current_session, + } + self.turns.append(turn_data) + return ParseEvent("soak_turn", turn_data, line) + + # Baseline + m = self.BASELINE_RE.match(clean) + if m: + self.baseline_vram = int(m.group(2)) + return ParseEvent("soak_baseline", {"vram_mib": self.baseline_vram}, line) + + # Verdict + m = self.VERDICT_RE.match(clean) + if m: + self.verdict = m.group(1) + status = Status.PASSED if self.verdict == "PASS" else Status.FAILED + return ParseEvent("verdict", { + "status": status, + "verdict": self.verdict, + }, line) + + # Metrics + m = self.METRIC_RE.match(clean) + if m: + key, value = m.group(1), m.group(2) + self.metrics[key] = value + return ParseEvent("soak_metric", {"key": key, "value": value}, line) + + return None + + +# ============================================================================ +# rebench-full.sh parser (orchestrator) +# ============================================================================ + +class RebenchParser: + """Parser for rebench-full.sh orchestrator output.""" + + # [verify-full] running… + STEP_RUNNING_RE = re.compile(r"^\[([\w-]+)\] running…") + + # [verify-full] ✓ 96s — log: results/rebench//verify-full.log + STEP_PASS_RE = re.compile(r"^\[([\w-]+)\] ✓ (\d+)s") + + # [bench] ✗ 14s — failed (rc=1) — log: … + STEP_FAIL_RE = re.compile(r"^\[([\w-]+)\] ✗ (\d+)s — failed \(rc=(\d+)\)") + + # [quality-full] skipped — 8-pack is opt-in + STEP_SKIP_RE = re.compile(r"^\[([\w-]+)\] skipped") + + # report: results/rebench//REPORT.md + REPORT_RE = re.compile(r"^\s+report:\s+(.+REPORT\.md)") + + # artifacts: results/rebench/ + ARTIFACTS_RE = re.compile(r"^\s+artifacts:\s+(.+)") + + # rebench complete + COMPLETE_RE = re.compile(r"^\s*rebench complete") + + STEP_ORDER = ["verify-full", "bench", "verify-stress", "quality-full", "quality-thinking", "soak"] + + def __init__(self): + self.steps: dict[str, dict] = {} + self.current_step: Optional[str] = None + self.report_path: str = "" + self.artifacts_dir: str = "" + self.complete: bool = False + + def parse_line(self, line: str) -> Optional[ParseEvent]: + clean = strip_ansi(line) + + # Step running + m = self.STEP_RUNNING_RE.match(clean) + if m: + step = m.group(1) + self.current_step = step + self.steps[step] = {"status": Status.RUNNING, "elapsed_s": 0} + return ParseEvent("rebench_step_start", {"step": step}, line) + + # Step passed + m = self.STEP_PASS_RE.match(clean) + if m: + step, elapsed = m.group(1), int(m.group(2)) + self.steps[step] = {"status": Status.PASSED, "elapsed_s": elapsed} + self.current_step = None + return ParseEvent("rebench_step_done", { + "step": step, "status": Status.PASSED, "elapsed_s": elapsed, + }, line) + + # Step failed + m = self.STEP_FAIL_RE.match(clean) + if m: + step, elapsed, rc = m.group(1), int(m.group(2)), int(m.group(3)) + self.steps[step] = {"status": Status.FAILED, "elapsed_s": elapsed, "rc": rc} + self.current_step = None + return ParseEvent("rebench_step_done", { + "step": step, "status": Status.FAILED, "elapsed_s": elapsed, "rc": rc, + }, line) + + # Step skipped + m = self.STEP_SKIP_RE.match(clean) + if m: + step = m.group(1) + self.steps[step] = {"status": Status.SKIPPED, "elapsed_s": 0} + return ParseEvent("rebench_step_done", { + "step": step, "status": Status.SKIPPED, "elapsed_s": 0, + }, line) + + # Report path + m = self.REPORT_RE.match(clean) + if m: + self.report_path = m.group(1) + return ParseEvent("rebench_report", {"path": self.report_path}, line) + + # Artifacts dir + m = self.ARTIFACTS_RE.match(clean) + if m: + self.artifacts_dir = m.group(1) + return ParseEvent("rebench_artifacts", {"dir": self.artifacts_dir}, line) + + # Complete + if self.COMPLETE_RE.match(clean): + self.complete = True + return ParseEvent("rebench_complete", {}, line) + + return None + + +def get_parser(test_type: TestType): + """Factory to get the right parser for a test type.""" + parsers = { + TestType.VERIFY: VerifyParser, + TestType.VERIFY_FULL: VerifyParser, + TestType.BENCH: BenchParser, + TestType.VERIFY_STRESS: StressParser, + TestType.QUALITY: QualityParser, + TestType.SOAK: SoakParser, + TestType.REBENCH: RebenchParser, + } + return parsers[test_type]() diff --git a/tools/test-console/club3090_test_console/runner.py b/tools/test-console/club3090_test_console/runner.py new file mode 100644 index 00000000..41df314c --- /dev/null +++ b/tools/test-console/club3090_test_console/runner.py @@ -0,0 +1,404 @@ +"""Test runner — spawns and manages test subprocesses.""" + +from __future__ import annotations + +import asyncio +import os +import signal +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Optional + +from .detect import ServingTarget +from .parsers import ParseEvent, TestType, get_parser + + +@dataclass +class TestConfig: + """Configuration for a test run.""" + test_type: TestType + # Test-specific tunables + run_count: int = 5 # bench RUNS + warmups: int = 3 # bench WARMUPS + only: str = "both" # bench ONLY (both/narr/code) + enable_thinking: bool = False # bench/quality ENABLE_THINKING + pp: bool = False # bench PP + force_tokens: int = 0 # bench FORCE_TOKENS + skip_tools: bool = False # verify SKIP_TOOLS + run_bench: bool = False # verify-full --bench + skip_longctx: bool = False # verify-stress SKIP_LONGCTX + skip_tool_prefill: bool = False # verify-stress SKIP_TOOL_PREFILL + skip_ceiling: bool = False # verify-stress SKIP_CEILING + quality_tier: str = "medium" # quality --quick/--medium/--full/--reasoning + quality_pack: str = "" # quality --pack + quality_no_sandboxed: bool = False + quality_sandboxed_only: bool = False + quality_sampling_server: bool = False + quality_repeat: int = 1 + max_tokens: int = 0 # quality MAX_TOKENS (0 = default) + thinking_max_tokens: int = 0 # quality THINKING_MAX_TOKENS (0 = default) + soak_mode: str = "fresh" # soak --fresh/--continuous/--quick + soak_sessions: int = 10 + soak_turns: int = 5 + soak_max_growth: int = 200 + soak_timeout: int = 1800 + rebench_8pack: str = "" # "" or "off" or "on" or "both" + rebench_skip: list[str] = field(default_factory=list) + rebench_resume: bool = False + rebench_tag: str = "" + # External endpoint + external_url: str = "" + external_model: str = "" + external_engine: str = "" + + +@dataclass +class RunState: + """State of an active test run.""" + test_type: TestType + config: TestConfig + target: ServingTarget + started: float = 0.0 + finished: float = 0.0 + exit_code: Optional[int] = None + verdict: str = "" # passed/failed/unknown + events: list[ParseEvent] = field(default_factory=list) + log_lines: list[str] = field(default_factory=list) + error: str = "" + artifact_dir: str = "" + report_path: str = "" + + @property + def elapsed_s(self) -> float: + end = self.finished or time.time() + return end - self.started if self.started else 0 + + @property + def is_running(self) -> bool: + return self.started > 0 and self.finished == 0 + + @property + def is_finished(self) -> bool: + return self.finished > 0 + + +class TestRunner: + """Manages spawning and tracking of test subprocesses.""" + + def __init__(self, repo_root: Path): + self.repo_root = repo_root + self.current_run: Optional[RunState] = None + self._process: Optional[asyncio.subprocess.Process] = None + self._cancel_event = asyncio.Event() + self._on_event: Optional[Callable[[ParseEvent], None]] = None + self._on_line: Optional[Callable[[str], None]] = None + self._on_complete: Optional[Callable[[RunState], None]] = None + self.history: list[RunState] = [] + + def set_callbacks( + self, + on_event: Optional[Callable[[ParseEvent], None]] = None, + on_line: Optional[Callable[[str], None]] = None, + on_complete: Optional[Callable[[RunState], None]] = None, + ): + """Set callback functions for events, lines, and completion.""" + self._on_event = on_event + self._on_line = on_line + self._on_complete = on_complete + + def _build_command(self, config: TestConfig) -> tuple[list[str], dict[str, str]]: + """Build the command and environment for a test run.""" + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + + args: list[str] = [] + + # Inject target + target = self.current_run.target if self.current_run else ServingTarget() + if target.url: + env["URL"] = target.url + if target.model: + env["MODEL"] = target.model + if target.container: + env["CONTAINER"] = target.container + + # BENCHLOCAL_HERMES_RESOLVE_LOCALHOST for localhost quality/rebench + if target.is_localhost and config.test_type in (TestType.QUALITY, TestType.REBENCH): + env["BENCHLOCAL_HERMES_RESOLVE_LOCALHOST"] = "1" + + # External endpoint mode (for rebench) + if config.external_url: + env["PREFLIGHT_NO_AUTODETECT"] = "1" + env["CONTAINER"] = "none" + + match config.test_type: + case TestType.VERIFY: + args = ["bash", "scripts/verify.sh"] + if config.skip_tools: + env["SKIP_TOOLS"] = "1" + + case TestType.VERIFY_FULL: + args = ["bash", "scripts/verify-full.sh"] + if config.skip_tools: + env["SKIP_TOOLS"] = "1" + if config.run_bench: + args.append("--bench") + + case TestType.BENCH: + args = ["bash", "scripts/bench.sh"] + env["RUNS"] = str(config.run_count) + env["WARMUPS"] = str(config.warmups) + env["ONLY"] = config.only + if config.enable_thinking: + env["ENABLE_THINKING"] = "1" + if config.pp: + env["PP"] = "1" + if config.force_tokens: + env["FORCE_TOKENS"] = str(config.force_tokens) + + case TestType.VERIFY_STRESS: + args = ["bash", "scripts/verify-stress.sh"] + if config.skip_longctx: + env["SKIP_LONGCTX"] = "1" + if config.skip_tool_prefill: + env["SKIP_TOOL_PREFILL"] = "1" + if config.skip_ceiling: + env["SKIP_CEILING"] = "1" + + case TestType.QUALITY: + args = ["bash", "scripts/quality-test.sh"] + # Tier flag + args.append(f"--{config.quality_tier}") + if config.quality_pack: + args.extend(["--pack", config.quality_pack]) + if config.quality_no_sandboxed: + args.append("--no-sandboxed") + if config.quality_sandboxed_only: + args.append("--sandboxed-only") + if config.enable_thinking: + args.append("--enable-thinking") + else: + args.append("--no-thinking") + if config.quality_sampling_server: + args.append("--sampling-from-server") + if config.quality_repeat > 1: + args.extend(["--repeat", str(config.quality_repeat)]) + if config.max_tokens > 0: + env["MAX_TOKENS"] = str(config.max_tokens) + if config.thinking_max_tokens > 0: + env["THINKING_MAX_TOKENS"] = str(config.thinking_max_tokens) + + case TestType.SOAK: + args = ["bash", "scripts/soak-test.sh"] + match config.soak_mode: + case "fresh": + args.append("--fresh") + case "continuous": + args.append("--continuous") + case "quick": + args.append("--quick") + env["SOAK_SESSIONS"] = str(config.soak_sessions) + env["SOAK_TURNS"] = str(config.soak_turns) + env["SOAK_MAX_GROWTH_MIB"] = str(config.soak_max_growth) + env["SOAK_TIMEOUT_S"] = str(config.soak_timeout) + + case TestType.REBENCH: + args = ["bash", "scripts/rebench-full.sh"] + if config.rebench_8pack: + args.append(f"--with-8pack-thinking={config.rebench_8pack}") + if config.rebench_skip: + args.append(f"--skip={','.join(config.rebench_skip)}") + if config.rebench_resume: + args.append("--resume") + if config.rebench_tag: + args.append(f"--tag={config.rebench_tag}") + if config.external_url: + args.extend([ + "--url", config.external_url, + "--model", config.external_model, + "--engine", config.external_engine or "other", + ]) + env["SOAK_SESSIONS"] = str(config.soak_sessions) + env["SOAK_TURNS"] = str(config.soak_turns) + if config.max_tokens > 0: + env["MAX_TOKENS"] = str(config.max_tokens) + if config.thinking_max_tokens > 0: + env["THINKING_MAX_TOKENS"] = str(config.thinking_max_tokens) + + # Use stdbuf for line-buffered output + full_cmd = ["stdbuf", "-oL", "-eL"] + args + return full_cmd, env + + async def start(self, config: TestConfig, target: ServingTarget) -> RunState: + """Start a test run.""" + state = RunState( + test_type=config.test_type, + config=config, + target=target, + started=time.time(), + ) + self.current_run = state + self._cancel_event.clear() + + cmd, env = self._build_command(config) + + try: + self._process = await asyncio.create_subprocess_exec( + *cmd, + cwd=str(self.repo_root), + env=env, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, # Merge stderr into stdout + start_new_session=True, # Own process group for signal delivery + ) + except Exception as e: + state.error = str(e) + state.finished = time.time() + state.exit_code = -1 + state.verdict = "failed" + if self._on_complete: + self._on_complete(state) + return state + + # Start the reader task + asyncio.create_task(self._read_output(state)) + return state + + async def _read_output(self, state: RunState): + """Read subprocess output and parse it.""" + parser = get_parser(state.test_type) + proc = self._process + + try: + while True: + if self._cancel_event.is_set(): + break + + try: + line_bytes = await asyncio.wait_for( + proc.stdout.readline(), + timeout=1.0, + ) + except asyncio.TimeoutError: + # Normal — retry + continue + + if not line_bytes: + # EOF + break + + line = line_bytes.decode("utf-8", errors="replace").rstrip("\n") + state.log_lines.append(line) + + # Notify line callback + if self._on_line: + self._on_line(line) + + # Parse for structured events + event = parser.parse_line(line) + if event: + state.events.append(event) + + # Extract artifacts/report from rebench + if event.event_type == "rebench_report": + state.report_path = event.data.get("path", "") + elif event.event_type == "rebench_artifacts": + state.artifact_dir = event.data.get("dir", "") + elif event.event_type == "verdict": + state.verdict = "passed" if event.data.get("status") == "passed" else "failed" + + if self._on_event: + self._on_event(event) + + except asyncio.CancelledError: + pass + except Exception as e: + state.error = str(e) + + # Wait for process exit + try: + await asyncio.wait_for(proc.wait(), timeout=10) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + + state.exit_code = proc.returncode + state.finished = time.time() + + # Determine verdict from exit code if not set by parser + if not state.verdict: + if state.exit_code == 0: + state.verdict = "passed" + else: + state.verdict = "failed" + + self.history.append(state) + self.current_run = None + self._process = None + + if self._on_complete: + self._on_complete(state) + + async def cancel(self) -> list[str]: + """Cancel the current run. Returns list of orphaned container names if any.""" + if not self._process: + return [] + + was_quality = (self.current_run and + self.current_run.test_type == TestType.QUALITY) + + self._cancel_event.set() + proc = self._process + + # SIGINT first (graceful) + try: + pgid = os.getpgid(proc.pid) + os.killpg(pgid, signal.SIGINT) + except (ProcessLookupError, OSError): + pass + + # Wait up to 5s + try: + await asyncio.wait_for(proc.wait(), timeout=5) + except asyncio.TimeoutError: + # SIGTERM + try: + pgid = os.getpgid(proc.pid) + os.killpg(pgid, signal.SIGTERM) + except (ProcessLookupError, OSError): + pass + + # Wait up to 5s more + try: + await asyncio.wait_for(proc.wait(), timeout=5) + except asyncio.TimeoutError: + # SIGKILL + try: + pgid = os.getpgid(proc.pid) + os.killpg(pgid, signal.SIGKILL) + except (ProcessLookupError, OSError): + pass + await proc.wait() + + # Check for orphaned benchlocal containers (known issue with quality tests) + orphans = [] + if was_quality: + orphans = await self._check_benchlocal_orphans() + + return orphans + + async def _check_benchlocal_orphans(self) -> list[str]: + """Check for orphaned benchlocal containers that may be squatting ports.""" + try: + proc = await asyncio.create_subprocess_exec( + "docker", "ps", "--format", "{{.Names}}", + "--filter", "name=benchlocal-", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=5) + containers = [name.strip() for name in stdout.decode().split("\n") if name.strip()] + return containers + except Exception: + return [] diff --git a/tools/test-console/club3090_test_console/widgets/__init__.py b/tools/test-console/club3090_test_console/widgets/__init__.py new file mode 100644 index 00000000..7815ec46 --- /dev/null +++ b/tools/test-console/club3090_test_console/widgets/__init__.py @@ -0,0 +1 @@ +"""Custom Textual widgets for the test console.""" diff --git a/tools/test-console/club3090_test_console/widgets/history_view.py b/tools/test-console/club3090_test_console/widgets/history_view.py new file mode 100644 index 00000000..5e204764 --- /dev/null +++ b/tools/test-console/club3090_test_console/widgets/history_view.py @@ -0,0 +1,344 @@ +"""History view — browse past runs and existing artifacts.""" + +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Optional + +from textual.app import ComposeResult +from textual.containers import Grid, Horizontal, Vertical +from textual.screen import ModalScreen +from textual.widgets import Button, Label, RichLog, Static, DataTable + +from ..parsers import TestType + + +class RunRecord: + """A single run record from history.""" + + def __init__(self, data: dict): + self.test = data.get("test", "unknown") + self.model = data.get("model", "") + self.url = data.get("url", "") + self.slug = data.get("slug", "") + self.started = data.get("started", 0) + self.finished = data.get("finished", 0) + self.exit_code = data.get("exit_code", -1) + self.verdict = data.get("verdict", "unknown") + self.elapsed_s = data.get("elapsed_s", 0) + self.artifact_dir = data.get("artifact_dir", "") + self.report_path = data.get("report_path", "") + self.power_cap_w = data.get("power_cap_w") + self.log_path = data.get("log_path", "") + + @property + def timestamp_str(self) -> str: + if self.started: + return time.strftime("%Y-%m-%d %H:%M", time.localtime(self.started)) + return "unknown" + + @property + def status_glyph(self) -> str: + if self.verdict == "passed": + return "[green]✓[/green]" + elif self.verdict == "failed": + return "[red]✗[/red]" + return "[dim]?[/dim]" + + +class RunDetailScreen(ModalScreen): + """Show details of a single run with report/log content.""" + + DEFAULT_CSS = """ + RunDetailScreen { + align: center middle; + } + RunDetailScreen > Vertical { + width: 90%; + height: 90%; + border: thick $primary; + background: $surface; + padding: 1 2; + } + RunDetailScreen .detail-title { + text-style: bold; + color: $accent; + text-align: center; + height: 1; + } + RunDetailScreen #detail-content { + height: 1fr; + } + RunDetailScreen #button-bar { + height: 3; + } + RunDetailScreen Button { + width: 1fr; + } + """ + + BINDINGS = [ + ("escape", "dismiss", "Close"), + ] + + def __init__(self, record: RunRecord, repo_root: Path): + super().__init__() + self.record = record + self.repo_root = repo_root + self._current_view = "summary" # summary, report, log + + def compose(self) -> ComposeResult: + with Vertical(): + yield Label(f"Run Details: {self.record.test}", classes="detail-title") + yield RichLog(id="detail-content") + with Horizontal(id="button-bar"): + yield Button("Summary", variant="primary", id="btn-summary") + if self.record.report_path: + yield Button("Report", variant="default", id="btn-report") + if self.record.log_path: + yield Button("Log", variant="default", id="btn-log") + yield Button("Close", variant="default", id="btn-close") + + def on_mount(self) -> None: + self._show_summary() + + def _show_summary(self) -> None: + """Display run summary.""" + self._current_view = "summary" + content = self.query_one("#detail-content", RichLog) + content.clear() + r = self.record + content.write(f"[bold]Test:[/bold] {r.test}") + content.write(f"[bold]Model:[/bold] {r.model}") + content.write(f"[bold]URL:[/bold] {r.url}") + content.write(f"[bold]Slug:[/bold] {r.slug or '(none)'}") + content.write(f"[bold]Started:[/bold] {r.timestamp_str}") + content.write(f"[bold]Elapsed:[/bold] {r.elapsed_s:.1f}s") + content.write(f"[bold]Exit code:[/bold] {r.exit_code}") + content.write(f"[bold]Verdict:[/bold] {r.status_glyph} {r.verdict}") + if r.power_cap_w: + content.write(f"[bold]Power cap:[/bold] {r.power_cap_w:.0f}W") + if r.artifact_dir: + content.write(f"\n[bold]Artifacts:[/bold] {r.artifact_dir}") + if r.report_path: + content.write(f"[bold]Report:[/bold] {r.report_path}") + if r.log_path: + content.write(f"[bold]Log:[/bold] {r.log_path}") + + def _show_report(self) -> None: + """Load and display the report file.""" + if not self.record.report_path: + return + self._current_view = "report" + content = self.query_one("#detail-content", RichLog) + content.clear() + + report_path = Path(self.record.report_path) + # Make relative to repo root if needed + if not report_path.is_absolute(): + report_path = self.repo_root / report_path + + if report_path.exists(): + try: + text = report_path.read_text() + # Render markdown-ish content + for line in text.split("\n"): + if line.startswith("# "): + content.write(f"[bold cyan]{line[2:]}[/bold cyan]") + elif line.startswith("## "): + content.write(f"[bold]{line[3:]}[/bold]") + elif line.startswith("### "): + content.write(f"[bold yellow]{line[4:]}[/bold yellow]") + elif line.startswith("- "): + content.write(f" • {line[2:]}") + elif line.startswith("```"): + content.write("[dim]" + line + "[/dim]") + else: + content.write(line) + except Exception as e: + content.write(f"[red]Error reading report: {e}[/red]") + else: + content.write(f"[red]Report file not found: {report_path}[/red]") + + def _show_log(self) -> None: + """Load and display the log file.""" + if not self.record.log_path: + return + self._current_view = "log" + content = self.query_one("#detail-content", RichLog) + content.clear() + + log_path = Path(self.record.log_path) + if not log_path.is_absolute(): + log_path = self.repo_root / log_path + + if log_path.exists(): + try: + text = log_path.read_text() + # Show last 500 lines + lines = text.split("\n") + if len(lines) > 500: + content.write(f"[dim]... showing last 500 of {len(lines)} lines ...[/dim]\n") + lines = lines[-500:] + for line in lines: + content.write(line) + except Exception as e: + content.write(f"[red]Error reading log: {e}[/red]") + else: + content.write(f"[red]Log file not found: {log_path}[/red]") + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == "btn-close": + self.app.pop_screen() + elif event.button.id == "btn-summary": + self._show_summary() + elif event.button.id == "btn-report": + self._show_report() + elif event.button.id == "btn-log": + self._show_log() + + def action_dismiss(self) -> None: + self.app.pop_screen() + + +class HistoryScreen(ModalScreen): + """Browse past runs from ~/.local/state and results/ artifacts.""" + + DEFAULT_CSS = """ + HistoryScreen { + align: center middle; + } + HistoryScreen > Vertical { + width: 90%; + height: 80%; + border: thick $primary; + background: $surface; + padding: 1 2; + } + HistoryScreen .history-title { + text-style: bold; + color: $accent; + text-align: center; + margin-bottom: 1; + } + HistoryScreen DataTable { + height: 1fr; + } + HistoryScreen .history-hint { + text-align: center; + color: $text-muted; + height: 1; + } + """ + + BINDINGS = [ + ("escape", "dismiss", "Close"), + ] + + def __init__(self, state_dir: Path, repo_root: Path): + super().__init__() + self.state_dir = state_dir + self.repo_root = repo_root + self.records: list[RunRecord] = [] + + def compose(self) -> ComposeResult: + with Vertical(): + yield Label("Run History", classes="history-title") + yield DataTable(id="history-table", cursor_type="row") + yield Label("Press Enter or click a row to view details", classes="history-hint") + yield Button("Close", variant="default", id="btn-close") + + def on_mount(self) -> None: + self._load_records() + table = self.query_one("#history-table", DataTable) + table.add_columns("Time", "Test", "Model", "Verdict", "Elapsed", "Power") + for i, record in enumerate(self.records): + table.add_row( + record.timestamp_str, + record.test, + record.model[:20], + record.status_glyph, + f"{record.elapsed_s:.0f}s", + f"{record.power_cap_w:.0f}W" if record.power_cap_w else "-", + key=str(i), + ) + # Focus the table + table.focus() + + def _load_records(self) -> None: + """Load run records from state dir and results/ artifacts.""" + # Load from state dir + runs_dir = self.state_dir / "runs" + if runs_dir.exists(): + for path in sorted(runs_dir.glob("*.json"), reverse=True): + try: + data = json.loads(path.read_text()) + # Infer log path from artifact dir + if "artifact_dir" in data and data["artifact_dir"]: + artifact_dir = Path(data["artifact_dir"]) + test_name = data.get("test", "") + log_path = artifact_dir / f"{test_name}.log" + if log_path.exists(): + data["log_path"] = str(log_path) + self.records.append(RunRecord(data)) + except Exception: + pass + + # Discover results/ artifacts + results_dir = self.repo_root / "results" + if results_dir.exists(): + # Rebench results + for tag_dir in sorted(results_dir.glob("rebench/*"), reverse=True): + if (tag_dir / "REPORT.md").exists(): + # Find log files in the directory + log_files = list(tag_dir.glob("*.log")) + log_path = str(log_files[0]) if log_files else "" + + self.records.append(RunRecord({ + "test": "rebench-full", + "model": tag_dir.name.split("-")[0] if "-" in tag_dir.name else tag_dir.name, + "started": tag_dir.stat().st_mtime, + "verdict": "passed", + "artifact_dir": str(tag_dir), + "report_path": str(tag_dir / "REPORT.md"), + "log_path": log_path, + })) + + # Quality results + for qdir in sorted(results_dir.glob("quality"), reverse=True): + for json_file in qdir.glob("quality-*.json"): + try: + data = json.loads(json_file.read_text()) + self.records.append(RunRecord({ + "test": "quality", + "model": data.get("model", "unknown"), + "started": json_file.stat().st_mtime, + "verdict": "passed" if data.get("totals", {}).get("score", 0) >= 0.8 else "failed", + "artifact_dir": str(json_file.parent), + "log_path": "", + })) + except Exception: + pass + + # Sort by time descending + self.records.sort(key=lambda r: r.started, reverse=True) + + def on_data_table_row_selected(self, event: DataTable.RowSelected) -> None: + """Handle row selection in the table.""" + try: + row_key = str(event.row_key.value) + idx = int(row_key) + if 0 <= idx < len(self.records): + record = self.records[idx] + self.app.push_screen(RunDetailScreen(record, self.repo_root)) + except (ValueError, IndexError): + pass + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == "btn-close": + self.app.pop_screen() + + def action_dismiss(self) -> None: + self.app.pop_screen() diff --git a/tools/test-console/club3090_test_console/widgets/live_pane.py b/tools/test-console/club3090_test_console/widgets/live_pane.py new file mode 100644 index 00000000..b03b4078 --- /dev/null +++ b/tools/test-console/club3090_test_console/widgets/live_pane.py @@ -0,0 +1,309 @@ +"""Live output pane — shows structured progress + raw log.""" + +from __future__ import annotations + +import time +from typing import Optional + +from textual.app import ComposeResult +from textual.containers import Vertical +from textual.reactive import reactive +from textual.widget import Widget +from textual.widgets import Label, RichLog, Static + +from ..parsers import ( + ParseEvent, + TestType, + BenchParser, + VerifyParser, + StressParser, + QualityParser, + SoakParser, + RebenchParser, +) + + +class StructuredHeader(Static): + """Structured progress display above the raw log.""" + + DEFAULT_CSS = """ + StructuredHeader { + width: 100%; + height: auto; + min-height: 3; + max-height: 12; + padding: 0 1; + background: $boost; + } + StructuredHeader .header-title { + text-style: bold; + color: $accent; + } + StructuredHeader .progress-bar { + height: 1; + } + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._lines: list[str] = [] + + def update_from_event(self, event: ParseEvent, test_type: TestType) -> None: + """Update the structured display from a parsed event.""" + # Build display based on test type + match test_type: + case TestType.BENCH: + self._update_bench(event) + case TestType.VERIFY | TestType.VERIFY_FULL: + self._update_verify(event) + case TestType.VERIFY_STRESS: + self._update_stress(event) + case TestType.QUALITY: + self._update_quality(event) + case TestType.SOAK: + self._update_soak(event) + case TestType.REBENCH: + self._update_rebench(event) + + def _update_bench(self, event: ParseEvent) -> None: + if event.event_type == "bench_section": + section = event.data.get("section", "").upper() + self._lines.append(f"[bold cyan]═══ {section} ═══[/bold cyan]") + elif event.event_type == "bench_run": + run_type = event.data.get("type", "") + run_num = event.data.get("run", 0) + decode_tps = event.data.get("decode_tps", 0) + ttft = event.data.get("ttft_ms", 0) + if run_type == "run": + self._lines.append(f" run-{run_num} decode [green]{decode_tps:.1f}[/green] TPS ttft {ttft}ms") + elif event.event_type == "summary_metric": + metric = event.data.get("metric", "") + mean = event.data.get("mean", 0) + cv = event.data.get("cv", 0) + self._lines.append(f" [bold]{metric}[/bold] mean=[cyan]{mean:.1f}[/cyan] CV={cv:.1f}%") + self._lines = self._lines[-20:] # Keep last 20 lines + self._refresh_display() + + def _update_verify(self, event: ParseEvent) -> None: + if event.event_type == "verify_step": + step = event.data.get("step", 0) + total = event.data.get("total", 0) + name = event.data.get("name", "") + self._lines.append(f"[cyan][{step}/{total}][/cyan] {name}") + elif event.event_type == "verify_check": + glyph = event.data.get("glyph", "") + msg = event.data.get("message", "") + color = {"✓": "green", "✗": "red", "⊘": "yellow"}.get(glyph, "") + self._lines.append(f" [{color}]{glyph}[/{color}] {msg}") + self._lines = self._lines[-20:] + elif event.event_type == "verdict": + status = event.data.get("status", "") + msg = event.data.get("message", "") + if status == "passed": + self._lines.append(f"[bold green]✓ {msg}[/bold green]") + else: + self._lines.append(f"[bold red]✗ {msg}[/bold red]") + self._refresh_display() + + def _update_stress(self, event: ParseEvent) -> None: + if event.event_type == "stress_probe": + probe = event.data.get("probe", 0) + name = event.data.get("name", "") + self._lines.append(f"[cyan][{probe}/8][/cyan] {name}") + elif event.event_type == "niah_rung": + rung = event.data.get("rung", 0) + total = event.data.get("total", 0) + target_k = event.data.get("target_k", 0) + glyph = event.data.get("glyph", "") + status = event.data.get("status", "") + color = {"passed": "green", "partial": "yellow", "failed": "red", "skipped": "dim"}.get(status, "") + self._lines.append(f" rung {rung}/{total} [{color}]{glyph} {target_k}K[/{color}]") + self._lines = self._lines[-20:] + elif event.event_type == "niah_token": + tokens = event.data.get("tokens", 0) + glyph = event.data.get("glyph", "") + status = event.data.get("status", "") + color = {"passed": "green", "partial": "yellow", "failed": "red", "skipped": "dim"}.get(status, "") + self._lines.append(f" [{color}]{glyph}[/] {tokens:,} tokens") + elif event.event_type == "verdict": + status = event.data.get("status", "") + msg = event.data.get("message", "") + color = "green" if status == "passed" else "red" + self._lines.append(f"[bold {color}]{msg}[/bold {color}]") + self._refresh_display() + + def _update_quality(self, event: ParseEvent) -> None: + if event.event_type == "quality_scenario": + num = event.data.get("num", 0) + total = event.data.get("total", 0) + scenario_id = event.data.get("scenario_id", "") + passed = event.data.get("passed", False) + elapsed = event.data.get("elapsed_s", 0) + glyph = "✓" if passed else "✗" + color = "green" if passed else "red" + self._lines.append(f" [{num}/{total}] {scenario_id} [{color}]{glyph}[/{color}] ({elapsed:.1f}s)") + self._lines = self._lines[-25:] + elif event.event_type == "quality_total": + passed = event.data.get("passed", 0) + total = event.data.get("total", 0) + pct = (passed / total * 100) if total > 0 else 0 + self._lines.append(f"[bold]TOTAL: {passed}/{total} ({pct:.0f}%)[/bold]") + self._refresh_display() + + def _update_soak(self, event: ParseEvent) -> None: + if event.event_type == "soak_session": + session = event.data.get("session", 0) + total = event.data.get("total", 0) + self._lines.append(f"[cyan]session {session}/{total}[/cyan]") + elif event.event_type == "soak_turn": + turn = event.data.get("turn", 0) + total = event.data.get("total", 0) + tps = event.data.get("decode_tps", 0) + vram = event.data.get("vram_mib", 0) + self._lines.append(f" turn {turn}/{total} [green]{tps:.1f}[/green] TPS {vram}MiB") + self._lines = self._lines[-20:] + elif event.event_type == "verdict": + verdict = event.data.get("verdict", "") + color = "green" if verdict == "PASS" else "red" + self._lines.append(f"[bold {color}]verdict: {verdict}[/bold {color}]") + elif event.event_type == "soak_metric": + key = event.data.get("key", "") + value = event.data.get("value", "") + self._lines.append(f" {key}: {value}") + self._refresh_display() + + def _update_rebench(self, event: ParseEvent) -> None: + if event.event_type == "rebench_step_start": + step = event.data.get("step", "") + self._lines.append(f"[cyan]▶ {step} running…[/cyan]") + elif event.event_type == "rebench_step_done": + step = event.data.get("step", "") + status = event.data.get("status", "") + elapsed = event.data.get("elapsed_s", 0) + if status == "passed": + self._lines.append(f"[green]✓ {step} {elapsed}s[/green]") + elif status == "failed": + rc = event.data.get("rc", 0) + self._lines.append(f"[red]✗ {step} {elapsed}s (rc={rc})[/red]") + elif status == "skipped": + self._lines.append(f"[yellow]⊘ {step} skipped[/yellow]") + elif event.event_type == "rebench_complete": + self._lines.append("[bold green]═══ rebench complete ═══[/bold green]") + elif event.event_type == "rebench_report": + path = event.data.get("path", "") + self._lines.append(f" report: {path}") + self._refresh_display() + + def _refresh_display(self) -> None: + """Refresh the widget display.""" + try: + content = "\n".join(self._lines[-25:]) if self._lines else "Waiting for output..." + self.update(content) + except Exception: + pass + + def clear(self) -> None: + """Clear the structured header.""" + self._lines = [] + self.update("") + + +class LivePane(Static): + """Right pane showing structured progress + raw log.""" + + DEFAULT_CSS = """ + LivePane { + width: 1fr; + height: 1fr; + border: solid $primary; + } + LivePane .live-title { + text-style: bold; + color: $accent; + dock: top; + padding: 0 1; + height: 1; + } + LivePane StructuredHeader { + dock: top; + height: auto; + max-height: 12; + } + LivePane RichLog { + height: 1fr; + overflow-y: auto; + } + """ + + current_test: reactive[Optional[TestType]] = reactive(None) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._follow = True + self._run_start_time = None + + def compose(self) -> ComposeResult: + yield Label("Live", classes="live-title") + yield StructuredHeader(id="structured-header") + yield RichLog(id="live-log", wrap=True, highlight=True, markup=True) + + def on_mount(self) -> None: + log = self.query_one("#live-log", RichLog) + log.write("[dim]Ready. Select a test and press Enter to run.[/dim]") + + def append_line(self, line: str) -> None: + """Append a raw log line.""" + try: + log = self.query_one("#live-log", RichLog) + log.write(line) + if self._follow: + log.scroll_end(animate=False) + except Exception: + pass + + def process_event(self, event: ParseEvent, test_type: TestType) -> None: + """Process a parsed event and update the structured header.""" + try: + header = self.query_one("#structured-header", StructuredHeader) + header.update_from_event(event, test_type) + except Exception: + pass + + def clear_log(self) -> None: + """Clear the log and structured header.""" + try: + log = self.query_one("#live-log", RichLog) + log.clear() + header = self.query_one("#structured-header", StructuredHeader) + header.clear() + except Exception: + pass + + def set_run_header(self, test_type: TestType, model: str, elapsed: str = "") -> None: + """Set the header for a new run.""" + self.current_test = test_type + self._run_start_time = time.time() + self.clear_log() + try: + log = self.query_one("#live-log", RichLog) + log.write(f"[bold cyan]▶ {test_type.value}[/bold cyan] model={model} {elapsed}") + log.write("─" * 60) + except Exception: + pass + + def update_elapsed_timer(self) -> None: + """Update the elapsed timer display.""" + if not self._run_start_time: + return + elapsed_s = time.time() - self._run_start_time + minutes = int(elapsed_s) // 60 + seconds = int(elapsed_s) % 60 + try: + title = self.query_one(".live-title", Label) + title.update(f"Live [dim]elapsed {minutes:02d}:{seconds:02d}[/dim]") + except Exception: + pass + + def toggle_follow(self) -> None: + """Toggle log follow/scroll-lock.""" + self._follow = not self._follow diff --git a/tools/test-console/club3090_test_console/widgets/manual_target.py b/tools/test-console/club3090_test_console/widgets/manual_target.py new file mode 100644 index 00000000..ea1f9f68 --- /dev/null +++ b/tools/test-console/club3090_test_console/widgets/manual_target.py @@ -0,0 +1,125 @@ +"""Manual target override screen.""" + +from __future__ import annotations + +from typing import Optional + +from textual.app import ComposeResult +from textual.containers import Vertical +from textual.screen import ModalScreen +from textual.widgets import Button, Input, Label, Static, DataTable + +from ..detect import ServingTarget, detect_from_registry + + +class ManualTargetScreen(ModalScreen[Optional[ServingTarget]]): + """Manual target override — pick registry slug or enter external URL.""" + + DEFAULT_CSS = """ + ManualTargetScreen { + align: center middle; + } + ManualTargetScreen > Vertical { + width: 80; + height: 40; + border: thick $primary; + background: $surface; + padding: 1 2; + } + ManualTargetScreen .manual-title { + text-style: bold; + color: $accent; + text-align: center; + margin-bottom: 1; + } + ManualTargetScreen .section-title { + text-style: bold; + margin-top: 1; + } + ManualTargetScreen DataTable { + height: 10; + } + ManualTargetScreen Input { + margin: 0 0 1 0; + } + """ + + BINDINGS = [ + ("escape", "cancel", "Cancel"), + ] + + def __init__(self, repo_root: str, variants: list[dict]): + super().__init__() + self.repo_root = repo_root + self.variants = variants + + def compose(self) -> ComposeResult: + with Vertical(): + yield Label("Manual Target Override", classes="manual-title") + + yield Label("Option 1: Select from registry", classes="section-title") + yield DataTable(id="registry-table") + + yield Label("Option 2: External endpoint", classes="section-title") + yield Label("URL:") + yield Input(placeholder="http://192.168.1.50:8887", id="input-url") + yield Label("Model name:") + yield Input(placeholder="qwen3.6-27b-autoround", id="input-model") + yield Label("Engine (vllm/llamacpp/sglang/other):") + yield Input(placeholder="vllm", id="input-engine") + + yield Button("Use selected", variant="primary", id="btn-registry") + yield Button("Use external", variant="primary", id="btn-external") + yield Button("Cancel", variant="default", id="btn-cancel") + + def on_mount(self) -> None: + table = self.query_one("#registry-table", DataTable) + table.add_columns("Slug", "Model", "Engine", "Port", "Status") + for v in self.variants[:50]: # Limit to 50 rows + table.add_row( + v.get("slug", ""), + v.get("model", "")[:20], + v.get("engine", ""), + str(v.get("port", "")), + v.get("status", ""), + ) + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == "btn-registry": + table = self.query_one("#registry-table", DataTable) + if table.cursor_row is not None and table.cursor_row < len(self.variants): + v = self.variants[table.cursor_row] + target = ServingTarget( + url=f"http://localhost:{v.get('port', 8020)}", + model=v.get("model", ""), + slug=v.get("slug", ""), + engine=v.get("engine", ""), + kv_format=v.get("kvcalc_key", ""), + status=v.get("status", ""), + host_port=v.get("port", 8020), + container=v.get("container", ""), # Set container identity + health="serving", + ) + self.dismiss(target) + + elif event.button.id == "btn-external": + url = self.query_one("#input-url", Input).value + model = self.query_one("#input-model", Input).value + engine = self.query_one("#input-engine", Input).value or "other" + if not url or not model: + self.notify("URL and model are required", severity="error") + return + target = ServingTarget( + url=url, + model=model, + engine=engine, + container="none", + health="serving", + ) + self.dismiss(target) + + elif event.button.id == "btn-cancel": + self.dismiss(None) + + def action_cancel(self) -> None: + self.dismiss(None) diff --git a/tools/test-console/club3090_test_console/widgets/target_pane.py b/tools/test-console/club3090_test_console/widgets/target_pane.py new file mode 100644 index 00000000..b8dcb86d --- /dev/null +++ b/tools/test-console/club3090_test_console/widgets/target_pane.py @@ -0,0 +1,131 @@ +"""Target status pane widget — shows detected model, endpoint, GPU stats.""" + +from __future__ import annotations + +from textual.app import ComposeResult +from textual.containers import Vertical +from textual.reactive import reactive +from textual.widget import Widget +from textual.widgets import Label, Static + +from ..detect import ServingTarget + +STATUS_GLYPHS = { + "serving": "● serving", + "unreachable": "○ unreachable", + "multiple": "⚠ multiple containers", + "unknown": "? unknown", +} + +STATUS_COLORS = { + "serving": "green", + "unreachable": "red", + "multiple": "yellow", + "unknown": "dim", +} + +REGISTRY_STATUS_GLYPHS = { + "production": "✅", + "caveats": "⚠️", + "experimental": "🧪", + "incubating": "🐣", + "preview": "👁️", + "upstream-gated": "⏸️", + "deprecated": "🗑️", +} + + +class TargetPane(Static): + """Left-top pane showing serving target info.""" + + target: reactive[ServingTarget | None] = reactive(None) + + DEFAULT_CSS = """ + TargetPane { + width: 100%; + height: auto; + min-height: 8; + border: solid $primary; + padding: 0 1; + } + TargetPane .pane-title { + text-style: bold; + color: $accent; + dock: top; + } + TargetPane .target-row { + height: 1; + } + TargetPane .target-label { + color: $text-muted; + } + TargetPane .no-target { + color: $warning; + text-style: italic; + } + """ + + def compose(self) -> ComposeResult: + yield Label("Target", classes="pane-title") + yield Label("Detecting...", classes="target-row", id="target-model") + yield Label("", classes="target-row", id="target-slug") + yield Label("", classes="target-row", id="target-engine") + yield Label("", classes="target-row", id="target-url") + yield Label("", classes="target-row", id="target-gpu0") + yield Label("", classes="target-row", id="target-gpu1") + + def watch_target(self, target: ServingTarget | None) -> None: + """Update display when target changes.""" + if target is None: + self._show_no_target() + return + + model_widget = self.query_one("#target-model", Label) + slug_widget = self.query_one("#target-slug", Label) + engine_widget = self.query_one("#target-engine", Label) + url_widget = self.query_one("#target-url", Label) + gpu0_widget = self.query_one("#target-gpu0", Label) + gpu1_widget = self.query_one("#target-gpu1", Label) + + if not target.model: + self._show_no_target() + return + + # Model line + status_glyph = STATUS_GLYPHS.get(target.health, STATUS_GLYPHS["unknown"]) + model_widget.update(f"Model {target.model}") + + # Slug + registry status + if target.slug: + reg_glyph = REGISTRY_STATUS_GLYPHS.get(target.status, "") + slug_widget.update(f"Slug {target.slug} {reg_glyph} {target.status}") + else: + slug_widget.update("") + + # Engine line + tp_str = f"TP {target.tp}" if target.tp else "" + kv_str = f"KV {target.kv_format}" if target.kv_format else "" + engine_widget.update(f"Engine {target.engine} {tp_str} {kv_str}".strip()) + + # URL + health + color = STATUS_COLORS.get(target.health, "dim") + url_widget.update(f"URL :{target.host_port} [{color}]{status_glyph}[/{color}]") + + # GPU lines + if target.gpus: + for i, gpu in enumerate(target.gpus[:2]): + widget = gpu0_widget if i == 0 else gpu1_widget + widget.update( + f"GPU{gpu.index} {gpu.mem_used_mib}/{gpu.mem_total_mib}M " + f"{gpu.utilization}% {gpu.power_draw_w:.0f}W/{gpu.power_limit_w:.0f}W " + f"{gpu.temp_c}°C" + ) + else: + gpu0_widget.update("") + gpu1_widget.update("") + + def _show_no_target(self) -> None: + model_widget = self.query_one("#target-model", Label) + model_widget.update("[yellow]No model serving — run gpu-mode [/yellow]") + for widget_id in ("#target-slug", "#target-engine", "#target-url", "#target-gpu0", "#target-gpu1"): + self.query_one(widget_id, Label).update("") diff --git a/tools/test-console/club3090_test_console/widgets/test_menu.py b/tools/test-console/club3090_test_console/widgets/test_menu.py new file mode 100644 index 00000000..2e080640 --- /dev/null +++ b/tools/test-console/club3090_test_console/widgets/test_menu.py @@ -0,0 +1,171 @@ +"""Test menu pane — lists available tests and their status.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +from textual.app import ComposeResult +from textual.message import Message +from textual.reactive import reactive +from textual.widget import Widget +from textual.widgets import Label, ListItem, ListView, Static + +from ..parsers import TestType + + +@dataclass +class TestEntry: + """A test menu entry.""" + test_type: TestType + display_name: str + duration_hint: str + description: str = "" + + +# The test catalog from spec Section 3 +TEST_CATALOG = [ + TestEntry(TestType.VERIFY, "Smoke (verify)", "~15s", "Quick server reachability check"), + TestEntry(TestType.VERIFY_FULL, "Functional (verify-full)", "~2min", "Full functional test suite"), + TestEntry(TestType.BENCH, "Speed bench (TPS)", "~5min", "Throughput benchmark"), + TestEntry(TestType.VERIFY_STRESS, "Stress / NIAH", "~15min", "Long-context + boundary tests"), + TestEntry(TestType.QUALITY, "Quality packs", "5-90min", "Behavioral quality testing"), + TestEntry(TestType.SOAK, "Soak / stability", "~20min", "Long-running stability test"), + TestEntry(TestType.REBENCH, "★ FULL rebench (macro)", "~45min", "Complete test pipeline"), +] + +STATUS_GLYPHS = { + "idle": "○", + "running": "▶", + "passed": "✓", + "failed": "✗", + "skipped": "⊘", + "queued": "◔", +} + + +class TestMenuPane(Static): + """Left-bottom pane showing the test menu.""" + + selected_index: reactive[int] = reactive(0) + + DEFAULT_CSS = """ + TestMenuPane { + width: 100%; + height: 1fr; + border: solid $primary; + padding: 0 1; + } + TestMenuPane .pane-title { + text-style: bold; + color: $accent; + dock: top; + } + TestMenuPane ListView { + height: 1fr; + } + TestMenuPane ListView > ListItem { + height: 2; + padding: 0 1; + } + TestMenuPane ListView > ListItem.--highlight { + background: $boost; + } + TestMenuPane .test-status { + width: 3; + } + TestMenuPane .test-name { + width: 1fr; + } + TestMenuPane .test-duration { + width: 8; + text-align: right; + color: $text-muted; + } + """ + + class TestSelected(Message): + """Fired when a test is selected.""" + def __init__(self, entry: TestEntry) -> None: + self.entry = entry + super().__init__() + + class TestActivated(Message): + """Fired when a test is activated (Enter pressed).""" + def __init__(self, entry: TestEntry) -> None: + self.entry = entry + super().__init__() + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._entries = TEST_CATALOG + self._statuses: dict[TestType, str] = {t.test_type: "idle" for t in TEST_CATALOG} + + def compose(self) -> ComposeResult: + yield Label("Tests", classes="pane-title") + with ListView(id="test-list"): + for entry in self._entries: + with ListItem(): + yield Label(self._format_entry(entry), classes="test-entry") + + def _format_entry(self, entry: TestEntry) -> str: + status = self._statuses.get(entry.test_type, "idle") + glyph = STATUS_GLYPHS.get(status, "○") + color = { + "idle": "", + "running": "[cyan]", + "passed": "[green]", + "failed": "[red]", + "skipped": "[yellow]", + "queued": "[dim]", + }.get(status, "") + end_color = f"[/{color.rstrip(']')}]" if color else "" + # Strip brackets for rich markup + if color: + color_clean = color.rstrip("]").lstrip("[") + end_clean = f"[/{color_clean}]" + else: + color_clean = "" + end_clean = "" + return f"{color}{glyph} {entry.display_name:<25} {entry.duration_hint}{end_clean}" + + def set_status(self, test_type: TestType, status: str) -> None: + """Update the status of a test entry.""" + self._statuses[test_type] = status + self._refresh_list() + + def _refresh_list(self) -> None: + """Refresh the list display.""" + try: + list_view = self.query_one("#test-list", ListView) + # Update labels + for i, entry in enumerate(self._entries): + items = list_view.children + if i < len(items): + label = items[i].query_one(Label) + label.update(self._format_entry(entry)) + except Exception: + pass + + def get_selected_entry(self) -> Optional[TestEntry]: + """Get the currently selected test entry.""" + try: + list_view = self.query_one("#test-list", ListView) + idx = list_view.index + if 0 <= idx < len(self._entries): + return self._entries[idx] + except Exception: + pass + return None + + def on_list_view_selected(self, event: ListView.Selected) -> None: + """Handle list selection.""" + entry = self.get_selected_entry() + if entry: + self.post_message(self.TestSelected(entry)) + + def on_list_view_highlighted(self, event: ListView.Highlighted) -> None: + """Handle list highlight change.""" + entry = self.get_selected_entry() + if entry: + self.post_message(self.TestSelected(entry)) diff --git a/tools/test-console/pyproject.toml b/tools/test-console/pyproject.toml new file mode 100644 index 00000000..cc630836 --- /dev/null +++ b/tools/test-console/pyproject.toml @@ -0,0 +1,34 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "club3090-test-console" +version = "0.1.0" +description = "A lazydocker-style TUI for the club-3090 AI inference stack test suite" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "textual>=0.60", + "rich>=13.0", + "httpx>=0.25", +] + +[project.scripts] +c3t = "club3090_test_console.__main__:main" + +[tool.hatch.build.targets.wheel] +packages = ["club3090_test_console"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +asyncio_mode = "auto" +filterwarnings = [ + "ignore::pytest.PytestCollectionWarning", +] + +[dependency-groups] +dev = [ + "pytest>=9.1.0", + "pytest-asyncio>=1.4.0", +] diff --git a/tools/test-console/test_summary.md b/tools/test-console/test_summary.md new file mode 100644 index 00000000..3c5d78e8 --- /dev/null +++ b/tools/test-console/test_summary.md @@ -0,0 +1,120 @@ +# Test Console Fixes Summary + +## Issues Fixed + +### 1. History View UI Distortion +**Problem**: Tabbing through the history detail screen caused UI distortion and the Close button would get cut off. + +**Solution**: +- Simplified the layout from complex nested Grid/Vertical containers to a single Vertical container +- Removed overly restrictive min/max height constraints +- Used standard Textual layout patterns: Vertical with 1fr content area and 3-unit button bar +- Made the layout more flexible and responsive + +**Files Changed**: +- `widgets/history_view.py`: Simplified RunDetailScreen layout + +### 2. Quality Tests Config Missing MAX_TOKENS +**Problem**: Quality tests config dialog didn't expose MAX_TOKENS and THINKING_MAX_TOKENS environment variables. + +**Solution**: +- Added `max_tokens` and `thinking_max_tokens` fields to TestConfig dataclass +- Added input fields to quality config dialog +- Updated `_read_config()` to read these values +- Updated runner to pass MAX_TOKENS and THINKING_MAX_TOKENS env vars + +**Files Changed**: +- `runner.py`: Added fields to TestConfig, updated quality command builder +- `app.py`: Added input fields to quality config, updated _read_config() + +### 3. Rebench Full Config Missing MAX_TOKENS +**Problem**: Rebench full config dialog didn't expose MAX_TOKENS and THINKING_MAX_TOKENS environment variables. + +**Solution**: +- Reused the same fields added for quality tests +- Added input fields to rebench config dialog +- Updated `_read_config()` to read these values +- Updated runner to pass MAX_TOKENS and THINKING_MAX_TOKENS env vars + +**Files Changed**: +- `runner.py`: Updated rebench command builder +- `app.py`: Added input fields to rebench config, updated _read_config() + +### 4. Rebench Full Config Buttons Truncated +**Problem**: The rebench full config dialog had so many fields that the Run/Cancel buttons at the bottom were being cut off. + +**Solution**: +- Increased max-height from 35 to 90% to allow more vertical space +- Wrapped config fields in a scrollable container (`#config-fields`) +- Used flex layout: title + hint + scrollable fields + button bar +- Button bar now has fixed height of 3 units, always visible + +**Files Changed**: +- `app.py`: Updated ConfigScreen CSS and compose() method + +## Test Results + +### Unit Tests +``` +83 tests passed ✅ +``` + +### Headless Integration Tests +``` +30/30 key interactions passed ✅ +- Help screen opens/closes +- History screen opens/closes +- Manual target screen opens/closes +- All 6 test configs open/close +- Tab navigation works in config dialogs +- All global keybindings work (r, f, x, q) +``` + +### Manual Verification +``` +✅ Quality config has MAX_TOKENS input field +✅ Quality config has THINKING_MAX_TOKENS input field +✅ Rebench config has MAX_TOKENS input field +✅ Rebench config has THINKING_MAX_TOKENS input field +✅ Rebench config Run/Cancel buttons are visible +✅ History detail screen opens without distortion +``` + +## Technical Details + +### Config Dialog Structure +``` +ConfigScreen > Vertical (max-height: 90%) +├── Label (config-title, height: 1) +├── Label (config-hint, height: 1) +├── Vertical (config-fields, height: 1fr, scrollable) +│ ├── Label + Select/Input pairs (varies by test type) +│ └── ... (up to 10+ fields for rebench) +└── Horizontal (button-bar, height: 3) + ├── Button (Run) + └── Button (Cancel) +``` + +### History Detail Screen Structure +``` +RunDetailScreen > Vertical (height: 100%) +├── Label (detail-title, height: 1) +├── RichLog (detail-content, height: 1fr, scrollable) +└── Horizontal (button-bar, height: 3) + ├── Button (Summary) + ├── Button (Report, conditional) + ├── Button (Log, conditional) + └── Button (Close) +``` + +### Environment Variables Passed +- **Quality Tests**: MAX_TOKENS, THINKING_MAX_TOKENS +- **Rebench Full**: MAX_TOKENS, THINKING_MAX_TOKENS +- **Bench Tests**: RUNS, WARMUPS, ONLY, FORCE_TOKENS, ENABLE_THINKING + +## Files Modified +1. `club3090_test_console/app.py` - Config dialogs and main app +2. `club3090_test_console/runner.py` - Test config and command builder +3. `club3090_test_console/widgets/history_view.py` - History detail screen + +All changes are backward compatible and don't break existing functionality. diff --git a/tools/test-console/tests/__init__.py b/tools/test-console/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/test-console/tests/test_detect.py b/tools/test-console/tests/test_detect.py new file mode 100644 index 00000000..9fcfa61b --- /dev/null +++ b/tools/test-console/tests/test_detect.py @@ -0,0 +1,214 @@ +"""Tests for the detection module.""" + +from __future__ import annotations + +import asyncio +import json +from unittest.mock import AsyncMock, patch, MagicMock + +import pytest + +from club3090_test_console.detect import ( + ServingTarget, + GpuInfo, + PORT_MAP_BROAD_RE, + _classify_engine, + _classify_engine_from_container, + match_target_to_registry, +) + + +# ============================================================================ +# Test port regex +# ============================================================================ + +class TestPortRegex: + def test_vllm_port(self): + m = PORT_MAP_BROAD_RE.search("0.0.0.0:8010->8000/tcp") + assert m is not None + assert m.group(1) == "8010" + assert m.group(2) == "8000" + + def test_llamacpp_port(self): + m = PORT_MAP_BROAD_RE.search("0.0.0.0:8020->8080/tcp") + assert m is not None + assert m.group(1) == "8020" + assert m.group(2) == "8080" + + def test_sglang_port(self): + m = PORT_MAP_BROAD_RE.search("0.0.0.0:30000->30000/tcp") + assert m is not None + assert m.group(1) == "30000" + assert m.group(2) == "30000" + + def test_ipv6_loopback(self): + m = PORT_MAP_BROAD_RE.search("[::]:8011->8000/tcp") + assert m is not None + assert m.group(1) == "8011" + + def test_localhost_only(self): + m = PORT_MAP_BROAD_RE.search("127.0.0.1:8011->8000/tcp") + assert m is not None + assert m.group(1) == "8011" + + def test_non_engine_port_ignored(self): + m = PORT_MAP_BROAD_RE.search("0.0.0.0:8188->8188/tcp") + assert m is None # 8188 is not an engine port + + def test_multiple_mappings(self): + line = "0.0.0.0:8010->8000/tcp, :::8010->8000/tcp" + matches = list(PORT_MAP_BROAD_RE.finditer(line)) + assert len(matches) == 2 + + +# ============================================================================ +# Test engine classification +# ============================================================================ + +class TestEngineClassification: + def test_from_port(self): + assert _classify_engine("8000") == "vllm" + assert _classify_engine("8080") == "llamacpp" + assert _classify_engine("30000") == "sglang" + assert _classify_engine("9999") == "unknown" + + def test_from_container_name(self): + assert _classify_engine_from_container("vllm-qwen36-27b") == "vllm" + assert _classify_engine_from_container("llama-cpp-pi-reasoning") == "llamacpp" + assert _classify_engine_from_container("ik-llama-cpp-dual") == "llamacpp" + assert _classify_engine_from_container("sglang-main") == "sglang" + assert _classify_engine_from_container("beellama-dflash") == "beellama" + assert _classify_engine_from_container("random-container") == "unknown" + + +# ============================================================================ +# Test ServingTarget +# ============================================================================ + +class TestServingTarget: + def test_is_localhost(self): + t = ServingTarget(url="http://localhost:8010") + assert t.is_localhost is True + + t = ServingTarget(url="http://127.0.0.1:8010") + assert t.is_localhost is True + + t = ServingTarget(url="http://192.168.1.50:8010") + assert t.is_localhost is False + + def test_is_active(self): + t = ServingTarget(url="http://localhost:8010", model="test-model", health="serving") + assert t.is_active is True + + t = ServingTarget(url="http://localhost:8010", model="test-model", health="unreachable") + assert t.is_active is False + + t = ServingTarget(url="", model="", health="unknown") + assert t.is_active is False + + +# ============================================================================ +# Test registry matching +# ============================================================================ + +class TestRegistryMatching: + def test_match_by_port(self): + target = ServingTarget(host_port=8010, container="vllm-test") + variants = [ + {"slug": "vllm/dual", "port": 8010, "model": "qwen", "engine": "vllm", + "kvcalc_key": "fp8", "status": "production", "container": "vllm_qwen", + "compose_dir": "", "file": "", "switch_engine": "", "launch_engine": "", + "compose_path": "", "ctx_label": "", "status_note": ""}, + ] + result = match_target_to_registry(target, variants) + assert result.slug == "vllm/dual" + assert result.status == "production" + + def test_match_by_container_name(self): + target = ServingTarget(host_port=9999, container="vllm-qwen36-27b") + variants = [ + {"slug": "vllm/dual", "port": 8010, "model": "qwen", "engine": "vllm", + "kvcalc_key": "fp8", "status": "production", "container": "vllm_qwen36_27b", + "compose_dir": "", "file": "", "switch_engine": "", "launch_engine": "", + "compose_path": "", "ctx_label": "", "status_note": ""}, + ] + result = match_target_to_registry(target, variants) + assert result.slug == "vllm/dual" + + def test_no_match(self): + target = ServingTarget(host_port=9999, container="unknown-thing") + variants = [ + {"slug": "vllm/dual", "port": 8010, "model": "qwen", "engine": "vllm", + "kvcalc_key": "fp8", "status": "production", "container": "vllm_qwen", + "compose_dir": "", "file": "", "switch_engine": "", "launch_engine": "", + "compose_path": "", "ctx_label": "", "status_note": ""}, + ] + result = match_target_to_registry(target, variants) + assert result.slug == "" + + +# ============================================================================ +# Test nothing-serving path +# ============================================================================ + +class TestNothingServing: + @pytest.mark.asyncio + async def test_empty_docker_ps(self): + """When docker ps returns nothing, health should be 'unreachable'.""" + from club3090_test_console.detect import detect_endpoint + + mock_proc = AsyncMock() + mock_proc.communicate = AsyncMock(return_value=(b"", b"")) + + with patch("asyncio.create_subprocess_exec", return_value=mock_proc): + target = await detect_endpoint() + assert target.health == "unreachable" + assert not target.model + assert not target.url + + +class TestDualStackDedup: + """Test that dual-stack Docker output is deduped and non-engine containers are filtered.""" + + def test_dual_stack_dedup(self): + """Same container with 0.0.0.0 and [::] mappings should produce one candidate.""" + from club3090_test_console.detect import PORT_MAP_BROAD_RE, ENGINE_PREFIXES + + # Simulate Docker dual-stack output + ports_str = "0.0.0.0:8010->8000/tcp, [::]:8010->8000/tcp" + matches = list(PORT_MAP_BROAD_RE.finditer(ports_str)) + assert len(matches) == 2 # Two raw matches + + # Dedup logic + seen = set() + unique = [] + container_name = "vllm-qwen36-27b" + for m in matches: + key = (container_name, int(m.group(1))) + if key not in seen: + seen.add(key) + unique.append(m) + assert len(unique) == 1 # Deduped to one + + def test_open_webui_filtered(self): + """Open WebUI maps 8080->8080 but isn't an engine container.""" + from club3090_test_console.detect import ENGINE_PREFIXES + + assert not ENGINE_PREFIXES.match("open-webui") + assert not ENGINE_PREFIXES.match("nginx-proxy") + assert not ENGINE_PREFIXES.match("litellm-gateway") + assert ENGINE_PREFIXES.match("vllm-qwen36-27b") + assert ENGINE_PREFIXES.match("llama-cpp-pi-reasoning") + assert ENGINE_PREFIXES.match("sglang-main") + assert ENGINE_PREFIXES.match("beellama-dflash") + + def test_multiple_containers_detected(self): + """Truly different containers should set health=multiple.""" + from club3090_test_console.detect import ENGINE_PREFIXES + + candidates = [ + ("vllm-qwen36-27b", 8010, 8000, "vllm"), + ("llama-cpp-pi-reasoning", 8063, 8080, "llamacpp"), + ] + unique_containers = set(c[0] for c in candidates) + assert len(unique_containers) == 2 # Two different containers diff --git a/tools/test-console/tests/test_parsers.py b/tools/test-console/tests/test_parsers.py new file mode 100644 index 00000000..870bb7d1 --- /dev/null +++ b/tools/test-console/tests/test_parsers.py @@ -0,0 +1,480 @@ +"""Tests for the parsers module — exercises all output formats from spec Section 10.""" + +from __future__ import annotations + +import pytest + +from club3090_test_console.parsers import ( + BenchParser, + VerifyParser, + StressParser, + QualityParser, + SoakParser, + RebenchParser, + Status, + strip_ansi, +) + + +# ============================================================================ +# Test strip_ansi +# ============================================================================ + +def test_strip_ansi(): + assert strip_ansi("\033[32m✓\033[0m test") == "✓ test" + assert strip_ansi("no codes here") == "no codes here" + assert strip_ansi("\033[1;31m✗\033[0m fail") == "✗ fail" + + +# ============================================================================ +# Test BenchParser +# ============================================================================ + +class TestBenchParser: + def test_section_header(self): + p = BenchParser() + event = p.parse_line("========== NARRATIVE (prompt=65 chars, max_tokens=1000) ==========") + assert event is not None + assert event.event_type == "bench_section" + assert event.data["section"] == "narrative" + + def test_warmup_line(self): + p = BenchParser() + p.parse_line("========== NARRATIVE (prompt=65 chars, max_tokens=1000) ==========") + event = p.parse_line(" warm-1 wall= 12.05s ttft= 120ms toks=1000 wall_TPS= 83.03 decode_TPS= 83.86") + assert event is not None + assert event.event_type == "bench_run" + assert event.data["type"] == "warm" + assert event.data["run"] == 1 + assert event.data["wall_s"] == 12.05 + assert event.data["ttft_ms"] == 120 + assert event.data["tokens"] == 1000 + assert event.data["wall_tps"] == 83.03 + assert event.data["decode_tps"] == 83.86 + + def test_measured_run_line(self): + p = BenchParser() + p.parse_line("========== NARRATIVE (prompt=65 chars, max_tokens=1000) ==========") + event = p.parse_line(" run-1 wall= 11.53s ttft= 118ms toks=1000 wall_TPS= 86.68 decode_TPS= 87.58") + assert event is not None + assert event.event_type == "bench_run" + assert event.data["type"] == "run" + assert event.data["wall_tps"] == 86.68 + assert event.data["decode_tps"] == 87.58 + assert len(p.runs["narrative"]) == 1 + + def test_summary_metric(self): + p = BenchParser() + p.current_section = "narrative" + event = p.parse_line(" wall_TPS mean= 84.14 std= 2.41 CV= 2.9% min=81.89 max=86.68") + assert event is not None + assert event.event_type == "summary_metric" + assert event.data["metric"] == "wall_TPS" + assert event.data["mean"] == 84.14 + assert event.data["cv"] == 2.9 + + def test_decode_tps_summary(self): + p = BenchParser() + p.current_section = "narrative" + event = p.parse_line(" decode_TPS mean= 84.99 std= 2.45 CV= 2.9% min=82.70 max=87.58") + assert event is not None + assert event.data["metric"] == "decode_TPS" + assert event.data["mean"] == 84.99 + + def test_ttft_summary(self): + p = BenchParser() + p.current_section = "narrative" + event = p.parse_line(" TTFT mean= 119ms std= 1ms min=118ms max=120ms") + assert event is not None + assert event.event_type == "summary_ttft" + assert event.data["ttft_mean_ms"] == 119 + + def test_multiple_runs_accumulate(self): + p = BenchParser() + p.parse_line("========== NARRATIVE (prompt=65 chars, max_tokens=1000) ==========") + p.parse_line(" run-1 wall= 11.53s ttft= 118ms toks=1000 wall_TPS= 86.68 decode_TPS= 87.58") + p.parse_line(" run-2 wall= 11.93s ttft= 119ms toks=1000 wall_TPS= 83.81 decode_TPS= 84.65") + assert len(p.runs["narrative"]) == 2 + + def test_unmatched_line_returns_none(self): + p = BenchParser() + assert p.parse_line("some random log line") is None + + +# ============================================================================ +# Test VerifyParser +# ============================================================================ + +class TestVerifyParser: + def test_step_header(self): + p = VerifyParser() + event = p.parse_line("[3/9] Basic completion — capital of France ...") + assert event is not None + assert event.event_type == "verify_step" + assert event.data["step"] == 3 + assert event.data["total"] == 9 + assert "capital of France" in event.data["name"] + + def test_pass_check(self): + p = VerifyParser() + event = p.parse_line(" ✓ reply contains 'Paris'") + assert event is not None + assert event.event_type == "verify_check" + assert event.data["status"] == "passed" + assert event.data["glyph"] == "✓" + + def test_fail_check(self): + p = VerifyParser() + event = p.parse_line(" ✗ tool-call request failed") + assert event is not None + assert event.data["status"] == "failed" + assert event.data["glyph"] == "✗" + + def test_skip_check(self): + p = VerifyParser() + event = p.parse_line(" ⊘ Genesis patches applied (skipped)") + assert event is not None + assert event.data["status"] == "skipped" + assert event.data["glyph"] == "⊘" + + def test_hint(self): + p = VerifyParser() + p.current_step = {"num": 1, "checks": [{"message": "test"}]} + event = p.parse_line(" → Check docker logs vllm-qwen36-27b") + assert event is not None + assert event.event_type == "verify_hint" + assert "docker logs" in event.data["hint"] + + def test_all_passed_verdict(self): + p = VerifyParser() + event = p.parse_line("All checks passed. Stack is ready for full-functionality use.") + assert event is not None + assert event.event_type == "verdict" + assert event.data["status"] == Status.PASSED + + def test_checks_failed_verdict(self): + p = VerifyParser() + event = p.parse_line("3 check(s) failed. See hints above.") + assert event is not None + assert event.event_type == "verdict" + assert event.data["status"] == Status.FAILED + assert event.data["failed"] == 3 + + def test_ansi_stripped(self): + p = VerifyParser() + event = p.parse_line(" \033[32m✓\033[0m reply contains 'Paris'") + assert event is not None + assert event.data["status"] == "passed" + + def test_counters(self): + p = VerifyParser() + p.parse_line(" ✓ test 1") + p.parse_line(" ✗ test 2") + p.parse_line(" ⊘ test 3") + assert p.passed == 1 + assert p.failed == 1 + assert p.skipped == 1 + + +# ============================================================================ +# Test StressParser +# ============================================================================ + +class TestStressParser: + def test_probe_header(self): + p = StressParser() + event = p.parse_line("[1/8] Long-context needle small rungs (10K / 30K) ...") + assert event is not None + assert event.event_type == "stress_probe" + assert event.data["probe"] == 1 + + def test_token_pass(self): + p = StressParser() + event = p.parse_line(" ✓ 10000 tokens: recalled '…' (got: …)") + assert event is not None + assert event.event_type == "niah_token" + assert event.data["tokens"] == 10000 + assert event.data["status"] == "passed" + + def test_token_partial(self): + p = StressParser() + event = p.parse_line(" △ 30000 tokens: recall MISS (…) — system OK, quality ceiling reached") + assert event is not None + assert event.data["status"] == "partial" + assert event.data["glyph"] == "△" + + def test_rung_pass(self): + p = StressParser() + event = p.parse_line(" ✓ rung 1/6: target=95K actual=95K tok (36%) recalled '…' prefill=… t/s (…s) VRAM_free=…MB") + assert event is not None + assert event.event_type == "niah_rung" + assert event.data["rung"] == 1 + assert event.data["total"] == 6 + assert event.data["target_k"] == 95 + assert event.data["status"] == "passed" + + def test_rung_partial(self): + p = StressParser() + event = p.parse_line(" △ rung 2/6: target=125K actual=125K tok (47%) recall MISS (…) — quality ceiling reached") + assert event is not None + assert event.data["status"] == "partial" + + def test_rung_failed(self): + p = StressParser() + event = p.parse_line(" ✗ rung 3/6: target=155K HTTP 500 (OOM at ~59% of n_ctx=262000)") + assert event is not None + assert event.data["status"] == "failed" + + def test_rung_skipped(self): + p = StressParser() + event = p.parse_line(" ⊘ rung 4/6: target=185K HTTP 400 (exceeds engine limit — clean rejection)") + assert event is not None + assert event.data["status"] == "skipped" + + def test_ladder_info(self): + p = StressParser() + event = p.parse_line(" n_ctx=262000 ladder: 95000 → 125000 → 155000 → 185000 → 215000 → 241000 (6 rungs)") + assert event is not None + assert event.event_type == "niah_ladder" + assert event.data["n_ctx"] == 262000 + + def test_vram_free(self): + p = StressParser() + event = p.parse_line(" VRAM free (ladder start): 12345 MB") + assert event is not None + assert event.data["vram_free_mb"] == 12345 + + def test_all_passed(self): + p = StressParser() + event = p.parse_line("All stress / boundary checks passed. KV-cache and prefill paths are sound for the deployed config.") + assert event is not None + assert event.data["status"] == Status.PASSED + + def test_some_failed(self): + p = StressParser() + event = p.parse_line("3 stress check(s) failed. See hints above.") + assert event is not None + assert event.data["status"] == Status.FAILED + assert event.data["failed"] == 3 + + +# ============================================================================ +# Test QualityParser +# ============================================================================ + +class TestQualityParser: + def test_scenario_pass(self): + p = QualityParser() + event = p.parse_line(" [1/15] TC-01 ✓ passed (2.3s)") + assert event is not None + assert event.event_type == "quality_scenario" + assert event.data["num"] == 1 + assert event.data["total"] == 15 + assert event.data["scenario_id"] == "TC-01" + assert event.data["passed"] is True + assert event.data["elapsed_s"] == 2.3 + assert event.data["pack_id"] == "toolcall-15" + + def test_scenario_fail(self): + p = QualityParser() + event = p.parse_line(" [7/15] TC-07 ✗ verifier_fail (3.1s)") + assert event is not None + assert event.data["passed"] is False + assert event.data["failure_mode"] == "verifier_fail" + assert event.data["elapsed_s"] == 3.1 + + def test_pack_id_mapping(self): + p = QualityParser() + # IF = instructfollow + event = p.parse_line(" [1/15] IF-01 ✓ passed (1.6s)") + assert event.data["pack_id"] == "instructfollow-15" + + # SO = structoutput + event = p.parse_line(" [1/15] SO-01 ✓ passed (1.2s)") + assert event.data["pack_id"] == "structoutput-15" + + # RM = reasonmath + event = p.parse_line(" [1/15] RM-01 ✓ passed (2.0s)") + assert event.data["pack_id"] == "reasonmath-15" + + def test_totals_accumulate(self): + p = QualityParser() + p.parse_line(" [1/15] TC-01 ✓ passed (2.3s)") + p.parse_line(" [2/15] TC-02 ✓ passed (1.5s)") + p.parse_line(" [3/15] TC-03 ✗ verifier_fail (3.1s)") + assert p.total_passed == 2 + assert p.total_count == 3 + + def test_pack_tracking(self): + p = QualityParser() + p.parse_line(" [1/15] TC-01 ✓ passed (2.3s)") + p.parse_line(" [2/15] TC-02 ✓ passed (1.5s)") + assert p.packs["toolcall-15"]["passed"] == 2 + assert p.packs["toolcall-15"]["total"] == 2 + + def test_total_line(self): + p = QualityParser() + event = p.parse_line("TOTAL 120/150") + assert event is not None + assert event.event_type == "quality_total" + assert event.data["passed"] == 120 + assert event.data["total"] == 150 + + +# ============================================================================ +# Test SoakParser +# ============================================================================ + +class TestSoakParser: + def test_config_line(self): + p = SoakParser() + event = p.parse_line("[soak] mode=fresh sessions=20 turns=5 max_growth=200MiB timeout=1800s") + assert event is not None + assert event.event_type == "soak_config" + assert event.data["mode"] == "fresh" + assert event.data["sessions"] == 20 + assert event.data["turns"] == 5 + + def test_session(self): + p = SoakParser() + event = p.parse_line("[soak] session 1/20") + assert event is not None + assert event.event_type == "soak_session" + assert event.data["session"] == 1 + assert event.data["total"] == 20 + + def test_turn(self): + p = SoakParser() + p.current_session = 1 + event = p.parse_line("[soak] turn 1/5: status=200 wall=5159ms ttft=481ms decode_tps=42.113 vram=43104MiB") + assert event is not None + assert event.event_type == "soak_turn" + assert event.data["turn"] == 1 + assert event.data["status"] == 200 + assert event.data["wall_ms"] == 5159 + assert event.data["ttft_ms"] == 481 + assert event.data["decode_tps"] == 42.113 + assert event.data["vram_mib"] == 43104 + + def test_baseline(self): + p = SoakParser() + event = p.parse_line("[soak] warm baseline after session 1: 43104 MiB") + assert event is not None + assert event.event_type == "soak_baseline" + assert event.data["vram_mib"] == 43104 + + def test_verdict_pass(self): + p = SoakParser() + event = p.parse_line("[soak] verdict PASS") + assert event is not None + assert event.event_type == "verdict" + assert event.data["status"] == Status.PASSED + assert event.data["verdict"] == "PASS" + + def test_verdict_fail(self): + p = SoakParser() + event = p.parse_line("[soak] verdict FAIL") + assert event.data["status"] == Status.FAILED + + def test_metrics(self): + p = SoakParser() + event = p.parse_line("[soak] silent_empty 0 / 100 (0.0%)") + assert event is not None + assert event.data["key"] == "silent_empty" + assert "0 / 100" in event.data["value"] + + event = p.parse_line("[soak] tps_retention 100.0%") + assert event.data["key"] == "tps_retention" + + event = p.parse_line("[soak] p50_decode_tps 42.23") + assert event.data["key"] == "p50_decode_tps" + + +# ============================================================================ +# Test RebenchParser +# ============================================================================ + +class TestRebenchParser: + def test_step_running(self): + p = RebenchParser() + event = p.parse_line("[verify-full] running…") + assert event is not None + assert event.event_type == "rebench_step_start" + assert event.data["step"] == "verify-full" + assert p.current_step == "verify-full" + + def test_step_passed(self): + p = RebenchParser() + event = p.parse_line("[verify-full] ✓ 96s — log: results/rebench/test-tag/verify-full.log") + assert event is not None + assert event.event_type == "rebench_step_done" + assert event.data["step"] == "verify-full" + assert event.data["status"] == Status.PASSED + assert event.data["elapsed_s"] == 96 + + def test_step_failed(self): + p = RebenchParser() + event = p.parse_line("[bench] ✗ 14s — failed (rc=1) — log: …") + assert event is not None + assert event.data["status"] == Status.FAILED + assert event.data["rc"] == 1 + + def test_step_skipped(self): + p = RebenchParser() + event = p.parse_line("[quality-full] skipped — 8-pack is opt-in (pass --with-8pack-thinking=off|both)") + assert event is not None + assert event.data["status"] == Status.SKIPPED + + def test_report_path(self): + p = RebenchParser() + event = p.parse_line(" report: results/rebench/test-tag/REPORT.md") + assert event is not None + assert event.data["path"] == "results/rebench/test-tag/REPORT.md" + + def test_artifacts_dir(self): + p = RebenchParser() + event = p.parse_line(" artifacts: results/rebench/test-tag") + assert event is not None + assert event.data["dir"] == "results/rebench/test-tag" + + def test_complete(self): + p = RebenchParser() + event = p.parse_line(" rebench complete") + assert event is not None + assert event.event_type == "rebench_complete" + assert p.complete is True + + def test_full_sequence(self): + """Test a complete rebench sequence.""" + p = RebenchParser() + p.parse_line("[verify-full] running…") + assert p.steps["verify-full"]["status"] == Status.RUNNING + + p.parse_line("[verify-full] ✓ 96s — log: results/rebench/tag/verify-full.log") + assert p.steps["verify-full"]["status"] == Status.PASSED + + p.parse_line("[bench] running…") + p.parse_line("[bench] ✓ 312s — log: results/rebench/tag/bench.log") + assert p.steps["bench"]["status"] == Status.PASSED + + p.parse_line("[quality-full] skipped — 8-pack is opt-in") + assert p.steps["quality-full"]["status"] == Status.SKIPPED + + p.parse_line(" rebench complete") + assert p.complete is True + + +# ============================================================================ +# Test parser factory +# ============================================================================ + +class TestParserFactory: + def test_get_parser(self): + from club3090_test_console.parsers import get_parser, TestType + assert isinstance(get_parser(TestType.BENCH), BenchParser) + assert isinstance(get_parser(TestType.VERIFY), VerifyParser) + assert isinstance(get_parser(TestType.VERIFY_FULL), VerifyParser) + assert isinstance(get_parser(TestType.VERIFY_STRESS), StressParser) + assert isinstance(get_parser(TestType.QUALITY), QualityParser) + assert isinstance(get_parser(TestType.SOAK), SoakParser) + assert isinstance(get_parser(TestType.REBENCH), RebenchParser) diff --git a/tools/test-console/tests/test_runner.py b/tools/test-console/tests/test_runner.py new file mode 100644 index 00000000..f6b3cf35 --- /dev/null +++ b/tools/test-console/tests/test_runner.py @@ -0,0 +1,215 @@ +"""Tests for the runner module — subprocess management and env injection.""" + +from __future__ import annotations + +import asyncio +import os +from pathlib import Path +from unittest.mock import AsyncMock, patch, MagicMock + +import pytest + +from club3090_test_console.runner import TestRunner, TestConfig, RunState +from club3090_test_console.detect import ServingTarget, GpuInfo +from club3090_test_console.parsers import TestType + + +# ============================================================================ +# Test command building +# ============================================================================ + +class TestCommandBuilding: + """Test that _build_command produces correct commands and env.""" + + def _make_runner_with_target(self, target: ServingTarget) -> TestRunner: + runner = TestRunner(repo_root=Path("/repo")) + state = RunState( + test_type=TestType.BENCH, + config=TestConfig(test_type=TestType.BENCH), + target=target, + ) + runner.current_run = state + return runner + + def test_bench_command(self): + target = ServingTarget(url="http://localhost:8010", model="qwen3.6-27b", container="vllm-test") + runner = self._make_runner_with_target(target) + config = TestConfig(test_type=TestType.BENCH, run_count=3, warmups=2, only="narr") + runner.current_run.config = config + cmd, env = runner._build_command(config) + + assert "scripts/bench.sh" in cmd + assert env["URL"] == "http://localhost:8010" + assert env["MODEL"] == "qwen3.6-27b" + assert env["CONTAINER"] == "vllm-test" + assert env["RUNS"] == "3" + assert env["WARMUPS"] == "2" + assert env["ONLY"] == "narr" + assert env["PYTHONUNBUFFERED"] == "1" + + def test_bench_with_thinking(self): + target = ServingTarget(url="http://localhost:8010", model="qwen", container="vllm-test") + runner = self._make_runner_with_target(target) + config = TestConfig(test_type=TestType.BENCH, enable_thinking=True) + cmd, env = runner._build_command(config) + assert env["ENABLE_THINKING"] == "1" + + def test_bench_with_force_tokens(self): + target = ServingTarget(url="http://localhost:8010", model="qwen", container="vllm-test") + runner = self._make_runner_with_target(target) + config = TestConfig(test_type=TestType.BENCH, force_tokens=2000) + cmd, env = runner._build_command(config) + assert env["FORCE_TOKENS"] == "2000" + + def test_verify_full_command(self): + target = ServingTarget(url="http://localhost:8010", model="qwen", container="vllm-test") + runner = self._make_runner_with_target(target) + config = TestConfig(test_type=TestType.VERIFY_FULL, skip_tools=True, run_bench=True) + cmd, env = runner._build_command(config) + + assert "scripts/verify-full.sh" in cmd + assert "--bench" in cmd + assert env["SKIP_TOOLS"] == "1" + + def test_verify_stress_command(self): + target = ServingTarget(url="http://localhost:8010", model="qwen", container="vllm-test") + runner = self._make_runner_with_target(target) + config = TestConfig( + test_type=TestType.VERIFY_STRESS, + skip_longctx=True, + skip_tool_prefill=True, + ) + cmd, env = runner._build_command(config) + + assert "scripts/verify-stress.sh" in cmd + assert env["SKIP_LONGCTX"] == "1" + assert env["SKIP_TOOL_PREFILL"] == "1" + + def test_quality_command(self): + target = ServingTarget(url="http://localhost:8010", model="qwen", container="vllm-test") + runner = self._make_runner_with_target(target) + config = TestConfig( + test_type=TestType.QUALITY, + quality_tier="full", + quality_pack="toolcall-15", + quality_repeat=3, + ) + cmd, env = runner._build_command(config) + + assert "scripts/quality-test.sh" in cmd + assert "--full" in cmd + assert "--pack" in cmd + assert "toolcall-15" in cmd + assert "--repeat" in cmd + assert "3" in cmd + assert env["BENCHLOCAL_HERMES_RESOLVE_LOCALHOST"] == "1" # localhost + + def test_quality_non_localhost_no_hermes(self): + target = ServingTarget(url="http://192.168.1.50:8887", model="qwen", container="vllm-test") + runner = self._make_runner_with_target(target) + config = TestConfig(test_type=TestType.QUALITY, quality_tier="medium") + cmd, env = runner._build_command(config) + + assert "BENCHLOCAL_HERMES_RESOLVE_LOCALHOST" not in env + + def test_soak_command(self): + target = ServingTarget(url="http://localhost:8010", model="qwen", container="vllm-test") + runner = self._make_runner_with_target(target) + config = TestConfig( + test_type=TestType.SOAK, + soak_mode="fresh", + soak_sessions=20, + soak_turns=10, + soak_max_growth=300, + ) + cmd, env = runner._build_command(config) + + assert "scripts/soak-test.sh" in cmd + assert "--fresh" in cmd + assert env["SOAK_SESSIONS"] == "20" + assert env["SOAK_TURNS"] == "10" + assert env["SOAK_MAX_GROWTH_MIB"] == "300" + + def test_rebench_command(self): + target = ServingTarget(url="http://localhost:8010", model="qwen", container="vllm-test") + runner = self._make_runner_with_target(target) + config = TestConfig( + test_type=TestType.REBENCH, + rebench_8pack="both", + rebench_skip=["soak"], + rebench_tag="test-run", + ) + cmd, env = runner._build_command(config) + + assert "scripts/rebench-full.sh" in cmd + assert "--with-8pack-thinking=both" in cmd + assert "--skip=soak" in cmd + assert "--tag=test-run" in cmd + + def test_rebench_external_endpoint(self): + target = ServingTarget(url="http://localhost:8010", model="qwen", container="vllm-test") + runner = self._make_runner_with_target(target) + config = TestConfig( + test_type=TestType.REBENCH, + external_url="http://192.168.1.50:8887", + external_model="Qwen3.6-27B", + external_engine="llama-cpp", + ) + cmd, env = runner._build_command(config) + + assert "--url" in cmd + assert "http://192.168.1.50:8887" in cmd + assert "--model" in cmd + assert "Qwen3.6-27B" in cmd + assert "--engine" in cmd + assert "llama-cpp" in cmd + assert env["PREFLIGHT_NO_AUTODETECT"] == "1" + assert env["CONTAINER"] == "none" + + def test_stdbuf_wrapping(self): + """All commands should be wrapped in stdbuf for line-buffered output.""" + target = ServingTarget(url="http://localhost:8010", model="qwen", container="vllm-test") + runner = self._make_runner_with_target(target) + for tt in TestType: + config = TestConfig(test_type=tt) + cmd, _ = runner._build_command(config) + assert cmd[0] == "stdbuf", f"{tt} command not wrapped in stdbuf" + + +# ============================================================================ +# Test RunState +# ============================================================================ + +class TestRunState: + def test_elapsed_while_running(self): + import time + state = RunState( + test_type=TestType.BENCH, + config=TestConfig(test_type=TestType.BENCH), + target=ServingTarget(), + started=time.time() - 10, + ) + assert state.elapsed_s >= 9 + assert state.is_running is True + assert state.is_finished is False + + def test_elapsed_after_finish(self): + state = RunState( + test_type=TestType.BENCH, + config=TestConfig(test_type=TestType.BENCH), + target=ServingTarget(), + started=1000.0, + finished=1060.0, + ) + assert state.elapsed_s == 60.0 + assert state.is_running is False + assert state.is_finished is True + + def test_not_started(self): + state = RunState( + test_type=TestType.BENCH, + config=TestConfig(test_type=TestType.BENCH), + target=ServingTarget(), + ) + assert state.elapsed_s == 0 + assert state.is_running is False diff --git a/tools/test-console/uv.lock b/tools/test-console/uv.lock new file mode 100644 index 00000000..761a0e94 --- /dev/null +++ b/tools/test-console/uv.lock @@ -0,0 +1,281 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "anyio" +version = "4.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/b5/001890774a9552aff22502b8da382593109ce0c95314abaebbb116567545/anyio-4.14.0.tar.gz", hash = "sha256:b47c1f9ccf73e67021df785332508f99379c68fa7d0684e8e3492cb1d4b23f89", size = 253586, upload-time = "2026-06-15T22:00:49.021Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/16/9826f089383c593cdfc4a6e5aca94d9e91ae1692c57af82c3b2aa5e810f7/anyio-4.14.0-py3-none-any.whl", hash = "sha256:dd9b7a2a9799ed6552fde617b2c5df02b7fdd7d88392fc48101e51bae46164d9", size = 123506, upload-time = "2026-06-15T22:00:47.595Z" }, +] + +[[package]] +name = "certifi" +version = "2026.6.17" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/c7/424b75da314c1045981bd9777432fad05a9e0c69daa4ed7e308bbaffe405/certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", size = 134594, upload-time = "2026-06-17T10:31:07.894Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db", size = 133289, upload-time = "2026-06-17T10:31:06.348Z" }, +] + +[[package]] +name = "club3090-test-console" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "httpx" }, + { name = "rich" }, + { name = "textual" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-asyncio" }, +] + +[package.metadata] +requires-dist = [ + { name = "httpx", specifier = ">=0.25" }, + { name = "rich", specifier = ">=13.0" }, + { name = "textual", specifier = ">=0.60" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=9.1.0" }, + { name = "pytest-asyncio", specifier = ">=1.4.0" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "linkify-it-py" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "uc-micro-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/c9/06ea13676ef354f0af6169587ae292d3e2406e212876a413bf9eece4eb23/linkify_it_py-2.1.0.tar.gz", hash = "sha256:43360231720999c10e9328dc3691160e27a718e280673d444c38d7d3aaa3b98b", size = 29158, upload-time = "2026-03-01T07:48:47.683Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/de/88b3be5c31b22333b3ca2f6ff1de4e863d8fe45aaea7485f591970ec1d3e/linkify_it_py-2.1.0-py3-none-any.whl", hash = "sha256:0d252c1594ecba2ecedc444053db5d3a9b7ec1b0dd929c8f1d74dce89f86c05e", size = 19878, upload-time = "2026-03-01T07:48:46.098Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + +[package.optional-dependencies] +linkify = [ + { name = "linkify-it-py" }, +] + +[[package]] +name = "mdit-py-plugins" +version = "0.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/59/fc/f8d0863f8862f25602c0404d75568e89fb6b4109804645e5cdfb1be5cf56/mdit_py_plugins-0.6.1.tar.gz", hash = "sha256:a2bca0f039f39dbd35fb74ae1b5f998608c437463371f0ff7f49a19a17a114d0", size = 56114, upload-time = "2026-05-13T09:03:38.91Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/69/6da5581c6a7fede7dc261bf4e67d6adca4196f176b43288b55b3db395b6e/mdit_py_plugins-0.6.1-py3-none-any.whl", hash = "sha256:214c82fb2ac524472ab6a5bcab1de80f73b50443e187f401bfd77efbc7c6481d", size = 66663, upload-time = "2026-05-13T09:03:37.76Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/47/e4501f49c178ae1d9f4a75073fda4204f52647993f075a9db4d14930e0c5/platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7", size = 31224, upload-time = "2026-05-28T03:32:53.587Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/84/0e/b5858858d74958632c49b72cb25a3976ff9f632397626715be71c89d3971/pytest-9.1.0.tar.gz", hash = "sha256:41dd9148c08072446394cefd3d79701701335a9f4cae69ba92e39f6c7f5c061c", size = 1634181, upload-time = "2026-06-13T18:52:45.983Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/5a/ba30a81239b909821b3153e303e7def45178bf353da4f72380e6c5e8793b/pytest-9.1.0-py3-none-any.whl", hash = "sha256:8ebb0e7888bdf2bdfc602ec51f8f62d50200af37356c74e503c79a94f5c81f32", size = 386453, upload-time = "2026-06-13T18:52:44.045Z" }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, +] + +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + +[[package]] +name = "textual" +version = "8.2.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py", extra = ["linkify"] }, + { name = "mdit-py-plugins" }, + { name = "platformdirs" }, + { name = "pygments" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/7a/c519db0aba5024f86e71e9631810bfdd6866ed2c8695bd7fa34b90e7ef59/textual-8.2.7.tar.gz", hash = "sha256:658f568ff81e30ed43890c3e07520390e5cf1b4763822006e060656b0a88f105", size = 1859249, upload-time = "2026-05-19T10:52:49.531Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/f5/c1e18bc0707300a0e90204343abbf7d7acd6fb7ebe03a6d4893b99a234b8/textual-8.2.7-py3-none-any.whl", hash = "sha256:4caaa13a90bc4cf9c6c862c067ccd34fe84e9c161710a2a907a8026313b6bd73", size = 731129, upload-time = "2026-05-19T10:52:51.773Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "uc-micro-py" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/78/67/9a363818028526e2d4579334460df777115bdec1bb77c08f9db88f6389f2/uc_micro_py-2.0.0.tar.gz", hash = "sha256:c53691e495c8db60e16ffc4861a35469b0ba0821fe409a8a7a0a71864d33a811", size = 6611, upload-time = "2026-03-01T06:31:27.526Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/73/d21edf5b204d1467e06500080a50f79d49ef2b997c79123a536d4a17d97c/uc_micro_py-2.0.0-py3-none-any.whl", hash = "sha256:3603a3859af53e5a39bc7677713c78ea6589ff188d70f4fee165db88e22b242c", size = 6383, upload-time = "2026-03-01T06:31:26.257Z" }, +]