From 75b29556c18ef62cf624244e57c1eee19130c3e6 Mon Sep 17 00:00:00 2001 From: noonghunna <10742901+noonghunna@users.noreply.github.com> Date: Sat, 4 Jul 2026 15:25:36 +0000 Subject: [PATCH] catalog-baselines slice 1: baselines.yml + registry-emit join + guards; catalog drops the BENCHMARKS scrape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catalog's measured columns (TPS / 8pk) now come from ONE productized source: scripts/lib/profiles/baselines.yml — the shipped, PR-reviewed 'bar' (accepted display projection of a validated gate run, with pin/rig/ power provenance) — joined per-slug into registry-emit --json with an emit-computed staleness verdict. Consumers never read baselines.yml or BENCHMARKS.md directly (BENCHMARKS stays the public human/cross-rig ledger). Design: the catalog-baselines note (2026-07-02, §2/§5 slice 1). - baselines.yml SEED WAVE 1: 10 rows with airtight traceability only (rebench tags on disk: agents-a1 golden specimen + gemma-31b-dual; unambiguous decode-class BENCHMARKS rows for the rest). 4 rows are HONESTLY born-stale with documented reasons (llamacpp rolling-tag-era x2, beellama pre-#296 image, 35B v0.22.0) — the guard's demo cases. 9 slugs still owed rows are listed as wave-2 gaps, no guessed numbers. - registry-emit join: per-variant 'baseline' field; current pin resolved the way launchers actually resolve it (engine-profile install.spec, compose-image-default fallback for ik/llama.cpp) → 'stale' = measured-pin != current-pin, null when undeterminable. - test-baselines.sh: schema + slug-membership + ctx-parity (compose ctx default == registry max_ctx, functional slugs) RED; pin-staleness WARN-only (pin bumps must not block on immediate re-bench — the debt stays visible). test-registry-json gains the 'baseline' contract key. - c3: enrich_measurements = pure in-memory map off the joined field — deletes BOTH the per-slug --explain fan-out (~4s/slug, the #439 option-3 leg) and the BENCHMARKS.md scrape from the catalog path (Explain modal + cross-rig explorer keep their readers). Stale rows render a † on the TPS cell + a status-line legend; full badge/overlay treatment is slice 2. Measurement gains source='baseline' + stale. - results/baselines/README: the three-store relationship (regression corpus vs measurement records vs display bar) + the no-drift rule the slice-2 induction tool enforces. Verified: guard test green (10 rows, 4 stale-warned); live join emits 57 variants / 10 baselines; live c3 catalog shows all 10 with daggers on exactly the stale 4 (A1 serving during the check: 154/154 - 105/150); c3 suite 764/764; full scripts gate green serial. (First gate pass tripped classifier/dedup by running CONCURRENTLY with the c3 suite — .pull-captures pollution class; serial rerun clean.) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EfF565T9eSLaqGzidyJ1Pm --- results/baselines/README.md | 9 + scripts/lib/profiles/baselines.yml | 188 ++++++++++++++++++ scripts/lib/registry-emit.sh | 73 ++++++- scripts/tests/test-baselines.sh | 126 ++++++++++++ scripts/tests/test-registry-json.sh | 15 +- tools/serve-cockpit/club3090_cockpit/app.py | 26 ++- tools/serve-cockpit/club3090_cockpit/data.py | 14 +- .../club3090_cockpit/services.py | 51 +++-- .../serve-cockpit/tests/test_app_headless.py | 33 +++ tools/serve-cockpit/tests/test_services.py | 28 ++- 10 files changed, 513 insertions(+), 50 deletions(-) create mode 100644 scripts/lib/profiles/baselines.yml create mode 100644 scripts/tests/test-baselines.sh diff --git a/results/baselines/README.md b/results/baselines/README.md index 35913aba..4da4261c 100644 --- a/results/baselines/README.md +++ b/results/baselines/README.md @@ -9,6 +9,15 @@ This is distinct from the runtime **measurement-record** TPS corpus (`scripts/lib/profiles/measurement_record.py`) — that tracks throughput; this tracks *behavioral quality* (ToolCall / InstructFollow / StructOutput / DataExtract / …). +It is ALSO distinct from the **shipped catalog baseline** +(`scripts/lib/profiles/baselines.yml`) — the slug-keyed *display* bar (decode TPS · +8-pack headline · ctx-validated · pin provenance) joined into `registry-emit --json` +for c3/switch. The three stores answer different questions (regression-diff vs raw +corpus vs accepted display row) but must not drift: the induction tool +(`catalog-baseline.sh`, slice 2) refreshes `baselines.yml`'s `quality_8pk` from the +same capture that lands here, so a slug's headline never disagrees with its +regression baseline. + ## What lives here One JSON per `(registry-slug, thinking-mode)`, named: diff --git a/scripts/lib/profiles/baselines.yml b/scripts/lib/profiles/baselines.yml new file mode 100644 index 00000000..cf05fb04 --- /dev/null +++ b/scripts/lib/profiles/baselines.yml @@ -0,0 +1,188 @@ +# baselines.yml — the shipped catalog baseline ("the bar"). +# =========================================================================== +# One row per registry slug: the ACCEPTED display projection of a validated +# gate run (full evidence stays in results/rebench// or the +# BENCHMARKS.md row it was reviewed from). PR-reviewed; written by +# scripts/catalog-baseline.sh (slice 2) or by hand at promotion time. +# Consumers read the registry-emit join, NEVER this file directly, and NEVER +# BENCHMARKS.md (which stays the public human/cross-rig ledger — publication, +# not machine source). +# +# Conventions: +# narr_tps / code_tps DECODE-class TPS means (bench.sh canonical prompts, +# warm, n>=3; the per-token rate, not wall). +# quality_8pk thinking-OFF arm "P/150"; _think_on = thinking-ON. +# 8-pack noise band is ±5–7 — separates classes, not +# fine rankings (badge semantics, design §2.1.4). +# ctx_validated highest ctx EXERCISED by the gate, with the recall +# verdict: {tokens: N, niah: "clean@XK" | "allocation- +# only"} — allocation alone is NOT validation (§2.1.2). +# engine_pin the pin the numbers were measured ON. If it differs +# from the slug's current engine pin, the row is +# PROVABLY STALE → c3 badges "re-bench owed" (§2.2). +# rig / power_cap_w hardware fingerprint class + per-card caps at bench. +# source_tag results/rebench// when a tag dir exists; +# omitted for rows reviewed from a BENCHMARKS.md entry +# (the row comment names the source). +# +# SEED WAVE 1 (2026-07-04): rows with airtight traceability only — a rebench +# tag on disk or an unambiguous decode-class BENCHMARKS.md row. Slugs still +# owed a row (evidence exists but needs review/archaeology) are listed at the +# bottom; DON'T guess numbers into them. +# =========================================================================== +schema_version: 1 +baselines: + + # ── Qwen3.6-27B ─────────────────────────────────────────────────────────── + llamacpp/default: + # BENCHMARKS.md 2026-05-23 row (decode, n=3, CV 0.6%/1.4%). + # Measured on the pre-2026-05-26 ROLLING tag (unknowable build) — the + # compose has pinned server-cuda-b9246 since → BORN-STALE, re-bench owed. + narr_tps: 50.27 + code_tps: 58.92 + date: 2026-05-23 + engine_pin: "ghcr.io/ggml-org/llama.cpp:server-cuda" + rig: "1x3090-pcie" + power_cap_w: [370] + submitted_by: "noonghunna" + + llamacpp/mtp: + # BENCHMARKS.md 2026-06-18 row (decode, n=5, CV <2%, thinking-off). + narr_tps: 47.9 + code_tps: 55.3 + date: 2026-06-18 + engine_pin: "ghcr.io/ggml-org/llama.cpp:server-cuda-b9246" + rig: "1x3090-pcie" + power_cap_w: [370] + submitted_by: "noonghunna" + + llamacpp/mtp-vision: + # BENCHMARKS.md 2026-05-20 row (decode, n=5, CV 1.6%/1.7%). + # Rolling-tag era (pre-b9246 pin) → BORN-STALE, re-bench owed. + narr_tps: 56.52 + code_tps: 66.17 + date: 2026-05-20 + engine_pin: "ghcr.io/ggml-org/llama.cpp:server-cuda" + rig: "1x3090-pcie" + power_cap_w: [370] + submitted_by: "noonghunna" + + ik-llama/iq4ks-mtp: + # BENCHMARKS.md 2026-05-23 row (decode 60.39/72.40, n=3; set+readback 370 W). + narr_tps: 60.39 + code_tps: 72.4 + date: 2026-05-23 + # ik composes pin by digest; measurement-era pin presumed == current cu13 + # digest (verify + exactify on the next gate via the induction tool). + engine_pin: "ghcr.io/ikawrakow/ik-llama-cpp@sha256:5f914f1ccade922417af58c94bd1cbb558052c8852d86678ead3fe693eec0143" + rig: "1x3090-pcie" + power_cap_w: [370] + submitted_by: "noonghunna" + + ik-llama/iq4ks-two-stage: + # BENCHMARKS.md 2026-05-24 row (decode, n=3, CV 1.9%/5.3%). + narr_tps: 59.4 + code_tps: 97.8 + date: 2026-05-24 + engine_pin: "ghcr.io/ikawrakow/ik-llama-cpp@sha256:5f914f1ccade922417af58c94bd1cbb558052c8852d86678ead3fe693eec0143" + rig: "1x3090-pcie" + power_cap_w: [370] + submitted_by: "noonghunna" + + beellama/dflash: + # BENCHMARKS.md 2026-05-30 row (decode 50.4/101.3, n=5, CV 5.6%/7.4%). + # Measured on the pre-#296 noonghunna multiarch image → PROVABLY STALE vs + # the current v0.3.2-preview engine pin (never re-benched on it) — the + # staleness badge is CORRECT here, keep it until a re-bench. + narr_tps: 50.4 + code_tps: 101.3 + date: 2026-05-30 + engine_pin: "ghcr.io/noonghunna/beellama-cpp:multiarch-b9459-07ac3ce" + rig: "1x3090-pcie" + power_cap_w: [370] + submitted_by: "noonghunna" + + ik-llama/byteshape-iq4xs-mtp: + # BENCHMARKS.md row (decode 115.60/137.07, n=5) + 8-pack 110/150 off — + # community intake #293/#299 reproduced on our rig 2026-06-02. + narr_tps: 115.6 + code_tps: 137.07 + quality_8pk: "110/150" + date: 2026-06-02 + engine_pin: "ghcr.io/ikawrakow/ik-llama-cpp@sha256:5f914f1ccade922417af58c94bd1cbb558052c8852d86678ead3fe693eec0143" + rig: "1x3090-pcie" + power_cap_w: [370] + submitted_by: "noonghunna" + + # ── Qwen3.6-35B-A3B ────────────────────────────────────────────────────── + vllm/qwen-35b-a3b-dual: + # BENCHMARKS.md promotion row (#259, decode 182.3/182.3; NIAH-clean 240K). + # Measured on v0.22.0; the slug now pins v0.24.0 (fast/max-tier era) → + # PROVABLY STALE — the born-stale demo row for the guard (§2.2). + # quality_8pk_think_on from the fresher #480-era measurement (incumbent + # baseline used in the Agents-A1 comparison). + narr_tps: 182.3 + code_tps: 182.3 + quality_8pk_think_on: "110/150" + ctx_validated: { tokens: 245760, niah: "clean@240K" } + date: 2026-05-30 + engine_pin: "vllm/vllm-openai:v0.22.0" + rig: "2x3090-pcie" + power_cap_w: [370, 420] + submitted_by: "noonghunna" + + # ── Agents-A1 ──────────────────────────────────────────────────────────── + vllm/agents-a1-dual: + # rebench tag agents-a1-fp8-dual (2026-07-03 gate): bench n=5 CV 0.1%; + # verify-stress 8/8 staggered NIAH to 240,635 tok exact-recall, VRAM Δ0; + # soak PASS (99.8% retention). Quality = post benchlocal #79+#81 harness: + # OFF 105/150 · ON 110/150 (cli-40 ON 23/40 fresh-image). + # Cross-rig confirmed 2026-07-04 (#552 @sumo-dandan, decode −1%). + narr_tps: 153.9 + code_tps: 154.0 + ttft_ms: 130 + quality_8pk: "105/150" + quality_8pk_think_on: "110/150" + ctx_validated: { tokens: 240635, niah: "clean@240K" } + date: 2026-07-03 + engine_pin: "vllm/vllm-openai:v0.24.0" + rig: "2x3090-pcie" + power_cap_w: [370, 420] + source_tag: "agents-a1-fp8-dual" + submitted_by: "noonghunna" + + # ── Gemma-4-31B ────────────────────────────────────────────────────────── + vllm/gemma-31b-dual: + # rebench tag gemma-31b-dual-bf16 (v0.24.0 consolidation gate #538/#539): + # decode 59.07/59.06, TTFT ~72 ms, soak PASS p50 58.69. + # TODO-review: quality_8pk + ctx_validated (gate artifacts have the NIAH + # ladder — extract the verdict; 224K claim pending #40391 for 262K). + narr_tps: 59.07 + code_tps: 59.06 + ttft_ms: 72 + date: 2026-07-02 + engine_pin: "vllm/vllm-openai:v0.24.0" + rig: "2x3090-pcie" + power_cap_w: [370, 420] + source_tag: "gemma-31b-dual-bf16" + submitted_by: "noonghunna" + +# ─────────────────────────────────────────────────────────────────────────── +# SEED WAVE 2 — rows owed, evidence needs review before numbers land here +# (do NOT guess; each needs its primary row disambiguated): +# vllm/minimal · vllm/dual · vllm/qwen-27b-dual-fast — rows exist under +# pre-rename sections/tags (fp8-mtp era + the #340 tier work); pick the +# current-pin (v0.24.0) or mark born-stale from the v0.22.0-era rows. +# ik-llama/iq4ks-mtp-vision — only a "carried" row exists (decode ≈ 60/72 +# carried from iq4ks-mtp); decide whether carried rows qualify. +# ik-llama/apex-fit-q8q5 — decode 105.63/156.80 (n=5) exists but the row's +# date needs confirming (APEX intake era). +# vllm/gemma-12b-dual-bf16-mtp (76.4/120.9) · vllm/gemma-12b-single-int8-mtp +# (117/122.5) — 2026-06-04 rows; confirm decode-vs-wall class + current-pin +# status after the v0.24.0 consolidation (both slugs re-pinned since). +# vllm/gemma-26ba4b-single — decode 169.2/219.9; confirm date + pin era. +# beellama/gemma-dflash — no direct BENCHMARKS row under the slug; numbers +# live in the promotion-era notes; needs archaeology. +# llamacpp/deckard40B-dual-mtp — 41.6 tok/s MTP in registry notes; no +# BENCHMARKS row; needs a proper gate run or an honest no-row. +# ─────────────────────────────────────────────────────────────────────────── diff --git a/scripts/lib/registry-emit.sh b/scripts/lib/registry-emit.sh index 0237971e..59e3cbc3 100755 --- a/scripts/lib/registry-emit.sh +++ b/scripts/lib/registry-emit.sh @@ -397,9 +397,72 @@ _spec.loader.exec_module(_tui_registry) from scripts.lib.profiles.compat import load_profiles # noqa: E402 from scripts.lib.profiles.compose_registry import COMPOSE_REGISTRY, DEFAULTS # noqa: E402 +from scripts.lib.profiles.launch_compat import ProfileError, resolve_variant_pin # noqa: E402 tab = os.environ.get("REGISTRY_TAB", "") +# --- profiles: sourced via the EXISTING loaders (never re-derived). Loaded +# HERE (not after the variants block) because the baselines join below +# resolves per-slug current pins through the engine profiles. --- +profiles = load_profiles() + +# --- baselines join (catalog-baselines slice 1): the shipped bar rows from +# scripts/lib/profiles/baselines.yml, joined per-slug with a computed +# staleness verdict. THIS is the single point where measured display +# numbers enter the contract — consumers never read baselines.yml (or +# BENCHMARKS.md) directly. --- +import re as _re # noqa: E402 + +import yaml as _yaml # noqa: E402 + +_bl_path = root / "scripts" / "lib" / "profiles" / "baselines.yml" +_baselines = {} +if _bl_path.exists(): + _baselines = (_yaml.safe_load(_bl_path.read_text()) or {}).get("baselines") or {} + +# First `image:` default in the compose (handles both a bare literal and the +# ${ENGINE_IMAGE:-literal} env-fallback form) — the pin truth for engines with +# no docker-image install.spec (ik / llama.cpp pin per-compose or roll by policy). +_IMG_RE = _re.compile(r"^\s*image:\s*[\"']?(?:\$\{[A-Z_0-9]+:-)?([^\s}\"']+)\}?", _re.M) + + +def _compose_image_default(compose_path: str): + try: + txt = (root / compose_path).read_text(encoding="utf-8") + except OSError: + return None + m = _IMG_RE.search(txt) + return m.group(1) if m else None + + +def _current_pin(slug: str, compose_path: str): + """The pin a launcher-started serve actually runs TODAY: the engine + profile's docker-image spec when it has one (launchers inject it), else + the compose's image default.""" + try: + exports = resolve_variant_pin(profiles, slug) + # Nightly pins export a bare SHA (VLLM_NIGHTLY_SHA) — not comparable to + # an image string; fall through to the compose default for those. + if "VLLM_NIGHTLY_SHA" not in exports: + return next(iter(exports.values())) + except ProfileError: + pass + return _compose_image_default(compose_path) + + +def _baseline_for(slug: str, compose_path: str): + row = _baselines.get(slug) + if not row: + return None + out = dict(row) + cur = _current_pin(slug, compose_path) + measured = row.get("engine_pin") + # stale: true/false when both pins are known; null = undeterminable + # (unpinned engine or unreadable compose) — badge only on TRUE. + out["stale"] = (cur != measured) if (cur and measured) else None + out["current_pin"] = cur + return out + # --- variants: exactly the fields parse_variant_rows produces from the tab form, # trimmed to the contract's variant schema (+ 'source' default "curated"). --- variants = [] @@ -444,6 +507,10 @@ for vr in _tui_registry.parse_variant_rows(tab): ), "status_note": d["status_note"], "source": "curated", + # The shipped baseline row ("the bar") + computed staleness — the + # ONLY measured-display source for consumers (replaces the c3-side + # BENCHMARKS.md scrape). None when the slug has no accepted row. + "baseline": _baseline_for(d["slug"], d["compose_path"]), } ) @@ -459,8 +526,7 @@ defaults = [ for (model, engine, topology), slug in DEFAULTS.items() ] -# --- profiles: sourced via the EXISTING loaders (never re-derived). --- -profiles = load_profiles() +# (profiles loaded above, before the variants block — the baselines join needs it.) def _engine(e): @@ -515,7 +581,8 @@ payload = { }, } -json.dump(payload, sys.stdout, sort_keys=True) +# default=str: baseline rows carry YAML-parsed datetime.date values. +json.dump(payload, sys.stdout, sort_keys=True, default=str) sys.stdout.write("\n") PY_JSON } diff --git a/scripts/tests/test-baselines.sh b/scripts/tests/test-baselines.sh new file mode 100644 index 00000000..fe5fba6a --- /dev/null +++ b/scripts/tests/test-baselines.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# test-baselines — guards for scripts/lib/profiles/baselines.yml + its +# registry-emit join (catalog-baselines slice 1). +# +# REDs on: schema violations · unknown slugs · ctx parity breaks (functional +# slugs: compose MAX_MODEL_LEN/CTX_SIZE default must equal registry max_ctx) · +# a seeded slug missing its joined baseline in the --json contract. +# WARNs (never reds) on: pin-staleness (engine_pin != current pin) — pin bumps +# must not block on immediate re-bench; the debt just stays visible. +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT_DIR" + +# --- 1-3. schema + slug membership + ctx parity (pure python asserts) -------- +python3 - <<'PY' +import re +import sys +from datetime import date +from pathlib import Path + +import yaml + +sys.path.insert(0, ".") +from scripts.lib.profiles.compose_registry import COMPOSE_REGISTRY # noqa: E402 + +doc = yaml.safe_load(Path("scripts/lib/profiles/baselines.yml").read_text()) +assert doc.get("schema_version") == 1, "schema_version must be 1" +rows = doc.get("baselines") or {} +assert rows, "baselines.yml has no rows" + +QUALITY_RE = re.compile(r"^\d{1,3}/150$") +errors = [] +for slug, row in rows.items(): + where = f"baselines[{slug}]" + if slug not in COMPOSE_REGISTRY: + errors.append(f"{where}: unknown registry slug"); continue + for k in ("narr_tps", "code_tps"): + if not isinstance(row.get(k), (int, float)): + errors.append(f"{where}.{k}: required numeric") + if not isinstance(row.get("date"), date): + errors.append(f"{where}.date: required YYYY-MM-DD") + for k in ("engine_pin", "rig", "submitted_by"): + if not (isinstance(row.get(k), str) and row[k].strip()): + errors.append(f"{where}.{k}: required non-empty string") + pw = row.get("power_cap_w") + if not (isinstance(pw, list) and pw and all(isinstance(x, int) for x in pw)): + errors.append(f"{where}.power_cap_w: required list of ints") + # optional, typed when present + if "ttft_ms" in row and not isinstance(row["ttft_ms"], (int, float)): + errors.append(f"{where}.ttft_ms: numeric") + for k in ("quality_8pk", "quality_8pk_think_on"): + if k in row and not QUALITY_RE.match(str(row[k])): + errors.append(f"{where}.{k}: must look like 'P/150'") + if "ctx_validated" in row: + cv = row["ctx_validated"] + ok = (isinstance(cv, dict) and isinstance(cv.get("tokens"), int) + and isinstance(cv.get("niah"), str)) + if not ok: + errors.append(f"{where}.ctx_validated: {{tokens: int, niah: str}}") + if "source_tag" in row and not isinstance(row["source_tag"], str): + errors.append(f"{where}.source_tag: string") + +# ctx parity (functional slugs): compose ctx-env default == registry max_ctx. +CTX_RE = re.compile(r"\$\{(?:MAX_MODEL_LEN|CTX_SIZE|MAX_CTX)[^:}]*:-(\d+)\}") +for slug, e in COMPOSE_REGISTRY.items(): + if e["status"] not in ("production", "caveats"): + continue + try: + txt = Path(e["compose_path"]).read_text() + except OSError: + errors.append(f"ctx-parity[{slug}]: compose unreadable: {e['compose_path']}") + continue + m = CTX_RE.search(txt) + if m and int(m.group(1)) != e["max_ctx"]: + errors.append( + f"ctx-parity[{slug}]: compose default {m.group(1)} != registry max_ctx {e['max_ctx']}" + ) + +if errors: + print("test-baselines: FAIL", file=sys.stderr) + for err in errors: + print(f" ✗ {err}", file=sys.stderr) + sys.exit(1) +print(f" ✓ schema + slug membership + ctx parity ({len(rows)} rows)") +PY + +# --- 4-5. the join contract + staleness WARNs -------------------------------- +# shellcheck source=/dev/null +source scripts/lib/registry-emit.sh +json="$(registry_variant_rows_json "$ROOT_DIR")" +# Env (not a pipe): the heredoc below owns the python interpreter's stdin — +# the same trick registry-emit itself uses for REGISTRY_TAB. +EMIT_JSON="$json" python3 - <<'PY' +import json +import os +import sys +from pathlib import Path + +import yaml + +d = json.loads(os.environ["EMIT_JSON"]) +by_slug = {v["slug"]: v for v in d["variants"]} +rows = yaml.safe_load(Path("scripts/lib/profiles/baselines.yml").read_text())["baselines"] + +missing = [s for s in rows if not (by_slug.get(s) or {}).get("baseline")] +if missing: + print(f"test-baselines: FAIL — seeded slugs missing joined baseline: {missing}", + file=sys.stderr) + sys.exit(1) + +# every joined row must carry the computed staleness verdict key +bad = [s for s in rows if "stale" not in by_slug[s]["baseline"]] +if bad: + print(f"test-baselines: FAIL — joined rows missing 'stale': {bad}", file=sys.stderr) + sys.exit(1) + +stale = [s for s in rows if by_slug[s]["baseline"]["stale"] is True] +for s in stale: + b = by_slug[s]["baseline"] + print(f" WARN: {s} baseline is STALE — measured on {b['engine_pin']!r}, " + f"current pin {b['current_pin']!r} (re-bench owed; row stays, badge shows)") +print(f" ✓ join contract ({len(rows)} joined, {len(stale)} stale-warned)") +PY + +echo "test-baselines: ok" diff --git a/scripts/tests/test-registry-json.sh b/scripts/tests/test-registry-json.sh index aa3ff4aa..0eac6804 100755 --- a/scripts/tests/test-registry-json.sh +++ b/scripts/tests/test-registry-json.sh @@ -44,16 +44,19 @@ need(isinstance(d["variants"], list) and d["variants"], "variants must be a non- need(isinstance(d["defaults"], list) and d["defaults"], "defaults must be a non-empty list") # variants — the parse_variant_rows fields (+ source + configured_ctx + -# weights_companions/drafter/vision); port is an int. configured_ctx is the EXACT -# numeric registry max_ctx int behind ctx_label (the cockpit's divergence badge -# compares the probe against it). weights_companions = the per-slug extra weight -# keys (DFlash draft / mmproj) the cockpit Download fetches alongside the core; -# drafter / vision are the per-slug facets (display + companion derivation). +# weights_companions/drafter/vision + baseline); port is an int. configured_ctx +# is the EXACT numeric registry max_ctx int behind ctx_label (the cockpit's +# divergence badge compares the probe against it). weights_companions = the +# per-slug extra weight keys (DFlash draft / mmproj) the cockpit Download +# fetches alongside the core; drafter / vision are the per-slug facets. +# baseline = the shipped catalog-baseline row joined from +# scripts/lib/profiles/baselines.yml with the emit-computed 'stale' verdict +# (catalog-baselines slice 1) — None when the slug has no accepted row. VARIANT_KEYS = { "slug", "switch_engine", "launch_engine", "compose_dir", "file", "port", "model", "engine", "kvcalc_key", "container", "compose_path", "status", "ctx_label", "configured_ctx", "status_note", "source", - "weights_companions", "drafter", "vision", + "weights_companions", "drafter", "vision", "baseline", } v0 = d["variants"][0] need(set(v0.keys()) == VARIANT_KEYS, diff --git a/tools/serve-cockpit/club3090_cockpit/app.py b/tools/serve-cockpit/club3090_cockpit/app.py index aaacb36f..5330b510 100644 --- a/tools/serve-cockpit/club3090_cockpit/app.py +++ b/tools/serve-cockpit/club3090_cockpit/app.py @@ -741,12 +741,14 @@ class CatalogPane(Container): serving = (self._serving_slug or "").strip() prev_model: Optional[str] = None # blank-on-repeat → the switch.sh --list grouped look for e in rows: - # source provenance — flag a coarse markdown scrape so a measurement - # from BENCHMARKS.md is never mistaken for a structured record. - meas_src = e.measurement.source + # Measured columns come from the SHIPPED BASELINE (registry-emit + # join) — the BENCHMARKS.md scrape is gone from the catalog path. + # Honesty marker: † = the row was measured on an OLDER engine pin + # than the slug currently runs (staleness guard §2.2; re-bench + # owed — full badge/overlay treatment lands in slice 2). tps = e.measurement.tps_label - if meas_src == "benchmarks.md" and tps != "—": - tps = f"{tps}*" + if e.measurement.stale is True and tps != "—": + tps = f"{tps}[yellow]†[/yellow]" # N3: mark the live-serving row so the running model is visible at a # glance in Run. Driven by the estate's matched_slug. slug_cell = e.slug @@ -797,8 +799,14 @@ class CatalogPane(Container): f"{banner}{scope}{len(rows)} / {len(self._entries)} variants{tail}{dep_note}" ) else: - star = " ([dim]*[/dim] = BENCHMARKS.md scrape)" if self._has_md_scrape() else "" - status_label.update(f"{banner}{len(self._entries)} variants loaded from registry{star}{dep_note}") + stale_note = ( + " ([yellow]†[/yellow][dim] = measured on an older engine pin — re-bench owed[/dim])" + if self._has_stale_baseline() + else "" + ) + status_label.update( + f"{banner}{len(self._entries)} variants loaded from registry{stale_note}{dep_note}" + ) # #9/A8 — keep the preview strip in sync with the cursor after a (re-)render # (enrichment mutates fit/measurement in place; the preview must reflect it). @@ -858,8 +866,8 @@ class CatalogPane(Container): if self._entries: self.refresh_enriched() - def _has_md_scrape(self) -> bool: - return any(e.measurement.source == "benchmarks.md" for e in self._entries) + def _has_stale_baseline(self) -> bool: + return any(e.measurement.stale is True for e in self._entries) def _filtered_entries(self) -> list[CatalogEntry]: # Hide 🗑️ deprecated slugs by default (mirrors `switch.sh --list`); [h] reveals them. diff --git a/tools/serve-cockpit/club3090_cockpit/data.py b/tools/serve-cockpit/club3090_cockpit/data.py index 775106b8..24dfb048 100644 --- a/tools/serve-cockpit/club3090_cockpit/data.py +++ b/tools/serve-cockpit/club3090_cockpit/data.py @@ -202,16 +202,22 @@ class FitVerdict: @dataclass class Measurement: - """A measured result for a slug, joined from a structured corpus or parsed - coarsely from BENCHMARKS.md. ``source`` records provenance so the UI can - distinguish a structured record from a best-effort markdown parse.""" + """A measured result for a slug. The catalog's source is the SHIPPED + BASELINE joined at registry-emit (source=="baseline"); "explain"/ + "benchmarks.md" remain only for non-catalog surfaces (Explain modal, + cross-rig explorer). ``source`` records provenance so the UI never + presents a coarse parse as an accepted number.""" narr_tps: Optional[float] = None code_tps: Optional[float] = None quality_8pk: Optional[str] = None # e.g. "107/150" max_ctx_label: str = "" date: str = "" - source: str = "" # "explain" | "corpus" | "benchmarks.md" | "" + source: str = "" # "baseline" | "explain" | "corpus" | "benchmarks.md" | "" + # Catalog-baselines: emit-computed pin-staleness for a baseline row — + # True = measured on an older engine pin (re-bench owed), False = current + # pin, None = undeterminable / not a baseline measurement. + stale: Optional[bool] = None @property def tps_label(self) -> str: diff --git a/tools/serve-cockpit/club3090_cockpit/services.py b/tools/serve-cockpit/club3090_cockpit/services.py index 01832489..4bec44ef 100644 --- a/tools/serve-cockpit/club3090_cockpit/services.py +++ b/tools/serve-cockpit/club3090_cockpit/services.py @@ -96,8 +96,6 @@ from .data import ( compute_promote_scaffold, measured_from_internal_json, measured_from_report_md, - measurement_from_explain_benchmarks, - parse_benchmarks_md_for_slug, parse_compute_apps, parse_df_output, parse_docker_ps_id_names, @@ -804,31 +802,28 @@ class CockpitData: e.fit = FitVerdict(verdict="skip", card=self.card) async def enrich_measurements(self, entries: list[CatalogEntry]) -> None: - # Read BENCHMARKS.md once up front — the per-slug md fallback below is - # then a pure in-memory parse (no I/O per slug). - bench_md = self._read_benchmarks_md() - sem = asyncio.Semaphore(self._ENRICH_CONCURRENCY) + """Catalog measured columns from the SHIPPED BASELINE joined into the + registry-emit --json contract (catalog-baselines slice 1). - async def _one(e: CatalogEntry) -> None: - # Preferred: structured benchmarks from the explain contract. The - # REAL shape is [{"row","columns"}]; measurement_from_explain_* - # parses TPS out of columns[]. Only COMMIT the explain result when - # it actually yields a TPS — otherwise an empty benchmarks[] (or a - # row that is stress/soak-only) must NOT suppress the markdown - # fallback (the `continue`-suppresses-fallback bug this fixes). - async with sem: - explain, _err = await self.explain(e.slug) - if explain and explain.get("benchmarks"): - m = measurement_from_explain_benchmarks(explain["benchmarks"]) - if m.narr_tps is not None or m.code_tps is not None: - e.measurement = m - return - # Fallback: coarse BENCHMARKS.md scrape (flagged in source). - m = parse_benchmarks_md_for_slug(bench_md or "", e.slug) - if m: - e.measurement = m - - await asyncio.gather(*(_one(e) for e in entries)) + Replaces BOTH prior sources: the per-slug ``--explain`` fan-out (the + ~4s/slug TPS leg, #439 option-3) and the coarse BENCHMARKS.md scrape — + BENCHMARKS.md is the public human/cross-rig ledger, never a machine + source. The baseline dict rides the variant row we already loaded, so + this is a pure in-memory map (no I/O, no subprocess). ``stale`` is the + emit-computed pin-staleness verdict (badged; slice 2 adds the local + measurement-record overlay).""" + for e in entries: + b = getattr(e.row, "baseline", None) + if not b: + continue + e.measurement = Measurement( + narr_tps=b.get("narr_tps"), + code_tps=b.get("code_tps"), + quality_8pk=b.get("quality_8pk"), + date=str(b.get("date") or ""), + source="baseline", + stale=b.get("stale"), + ) def _read_benchmarks_md(self) -> str: path = self.repo_root / "BENCHMARKS.md" @@ -3725,6 +3720,10 @@ def _variant_row_from_dict(d: dict[str, Any]) -> VariantRow: object.__setattr__(row, "weights_companions", [str(c) for c in comp]) object.__setattr__(row, "drafter", str(d.get("drafter") or "")) object.__setattr__(row, "vision", bool(d.get("vision"))) + # Catalog-baselines slice 1: the shipped baseline row joined at + # registry-emit (narr/code TPS · 8pk · ctx_validated · provenance · + # computed 'stale') — None when the slug has no accepted row. + object.__setattr__(row, "baseline", d.get("baseline") or None) except Exception: pass return row diff --git a/tools/serve-cockpit/tests/test_app_headless.py b/tools/serve-cockpit/tests/test_app_headless.py index 1c38e686..6c97267b 100644 --- a/tools/serve-cockpit/tests/test_app_headless.py +++ b/tools/serve-cockpit/tests/test_app_headless.py @@ -210,6 +210,13 @@ REGISTRY_JSON = json.dumps( "configured_ctx": 262144, "status_note": "", "source": "curated", + "baseline": { + "narr_tps": 174.0, "code_tps": 42.0, "quality_8pk": "109/150", + "date": "2026-07-01", "engine_pin": "vllm/vllm-openai:v0.24.0", + "current_pin": "vllm/vllm-openai:v0.24.0", "stale": False, + "rig": "2x3090-pcie", "power_cap_w": [370, 420], + "submitted_by": "noonghunna", + }, }, { "slug": "ik-llama/iq4ks-mtp", @@ -228,6 +235,13 @@ REGISTRY_JSON = json.dumps( "configured_ctx": 200000, "status_note": "", "source": "curated", + "baseline": { + "narr_tps": 60.4, "code_tps": 72.4, + "date": "2026-05-23", "engine_pin": "ghcr.io/ik-old@sha256:aaa", + "current_pin": "ghcr.io/ik-new@sha256:bbb", "stale": True, + "rig": "1x3090-pcie", "power_cap_w": [370], + "submitted_by": "noonghunna", + }, }, ], } @@ -964,6 +978,25 @@ class TestCatalogWired: assert entry.fit.glyph == "●" # fits-clean assert entry.measurement.tps_label == "174/42" assert entry.measurement.quality_label == "109/150" + # Catalog-baselines slice 1: the source is the shipped baseline. + assert entry.measurement.source == "baseline" + + @pytest.mark.asyncio + async def test_catalog_stale_baseline_dagger(self): + """Catalog-baselines slice 1 — a baseline measured on an OLDER engine + pin renders the † staleness marker on its TPS cell + the status-line + legend (re-bench owed); a current-pin row stays unmarked.""" + app, _, _ = make_app() + async with app.run_test(size=(120, 40)) as pilot: + await _settle(pilot) + tbl = app.query_one("#catalog-table", DataTable) + rows = [" ".join(str(c) for c in tbl.get_row_at(r)) for r in range(tbl.row_count)] + ik_row = next(r for r in rows if "iq4ks-mtp" in r) # stale fixture row + dual_row = next(r for r in rows if "vllm/dual" in r) # fresh fixture row + assert "†" in ik_row + assert "†" not in dual_row + status = str(app.query_one("#catalog-status", Label).render()) + assert "older engine pin" in status @pytest.mark.asyncio async def test_catalog_ik_llama_fit_is_skip(self): diff --git a/tools/serve-cockpit/tests/test_services.py b/tools/serve-cockpit/tests/test_services.py index 22c7eee8..e3aa146b 100644 --- a/tools/serve-cockpit/tests/test_services.py +++ b/tools/serve-cockpit/tests/test_services.py @@ -143,6 +143,13 @@ REGISTRY_JSON = json.dumps( "ctx_label": "262K", "status_note": "", "source": "curated", + "baseline": { + "narr_tps": 174.0, "code_tps": 42.0, "quality_8pk": "109/150", + "date": "2026-07-01", "engine_pin": "vllm/vllm-openai:v0.24.0", + "current_pin": "vllm/vllm-openai:v0.24.0", "stale": False, + "rig": "2x3090-pcie", "power_cap_w": [370, 420], + "submitted_by": "noonghunna", + }, }, { "slug": "ik-llama/iq4ks-mtp", @@ -160,6 +167,13 @@ REGISTRY_JSON = json.dumps( "ctx_label": "200K", "status_note": "", "source": "curated", + "baseline": { + "narr_tps": 60.4, "code_tps": 72.4, + "date": "2026-05-23", "engine_pin": "ghcr.io/ik-old@sha256:aaa", + "current_pin": "ghcr.io/ik-new@sha256:bbb", "stale": True, + "rig": "1x3090-pcie", "power_cap_w": [370], + "submitted_by": "noonghunna", + }, }, ], } @@ -487,13 +501,23 @@ class TestLoadCatalog: assert ik.fit.verdict == "skip" @pytest.mark.asyncio - async def test_catalog_enriches_measurement_from_explain(self): + async def test_catalog_enriches_measurement_from_baseline(self): + """Catalog-baselines slice 1: measured columns come from the shipped + baseline joined into the registry-emit contract — no per-slug explain + fan-out, no BENCHMARKS.md scrape (that stays a human ledger).""" cd = CockpitData(ROOT, runner=full_runner()) entries, _ = await cd.load_catalog(enrich_fit=False, enrich_measurement=True) vllm = next(e for e in entries if e.slug == "vllm/dual") - assert vllm.measurement.source == "explain" + assert vllm.measurement.source == "baseline" assert vllm.measurement.tps_label == "174/42" assert vllm.measurement.quality_label == "109/150" + assert vllm.measurement.stale is False + # The enrichment ran ZERO --explain subprocesses (the ~4s/slug leg is gone). + assert not any("--explain" in " ".join(c) for c in cd._runner.calls) + # A stale row carries the emit-computed verdict through. + ik = next(e for e in entries if e.slug == "ik-llama/iq4ks-mtp") + assert ik.measurement.source == "baseline" + assert ik.measurement.stale is True @pytest.mark.asyncio async def test_catalog_empty_registry_returns_error(self):