#!/usr/bin/env bash set -euo pipefail # test-generate-compose.sh — v0.8.0 STEP 4 (club-3090 #141 / PR #147). # # Golden-parity + refusal/degraded contract for scripts/generate-compose.sh. # This test is the contract; the generator is fixed to it (never the # reverse). It imports the STEP-2 patch_attribution.reaches() — it does NOT # reimplement compose parsing or reachability (RED-LINE). # # 5 golden triples, all verified genesis_equipped:false in # profile_runtime.yml, spanning every in-scope engine class: # vllm/minimal vllm-stable, tp1, fp8, drafter=None # vllm/dual vllm-stable, tp2, fp8, mtp # vllm/gemma-bf16-mtp vllm-gemma-stable, gemma, bf16 # vllm/gemma-int8-mtp vllm-gemma-stable, int8-PTH, multi-file overlay # vllm/gemma-mtp-tp1 vllm-gemma-stable, single-card fp8 risk path # # Per triple: generate -> semantic diff vs the shipped compose differs ONLY # at the two patch insertion points (image expression + every constant # reproduce verbatim) -> selected+wired subset-of shipped -> wired patches # pass reaches() on the GENERATED compose (actual wiring, not header text) # -> selected-but-undelivered NOT reachable -> 3-category header present -> # NO --trust-remote-code emitted. Plus the refusal/degraded matrix. ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" python3 - "$ROOT_DIR" <<'PY' from __future__ import annotations import os import sys import tempfile from pathlib import Path root = Path(sys.argv[1]) sys.path.insert(0, str(root)) from scripts.lib import generate_compose as gc # noqa: E402 from scripts.lib.profiles import patch_attribution as pa # noqa: E402 from scripts.lib.profiles.compose_registry import COMPOSE_REGISTRY # noqa: E402 errors: list[str] = [] def check(cond: bool, msg: str) -> None: if not cond: errors.append(msg) patches = gc.load_patches(root) pmap = {p["id"]: p for p in patches} runtime = gc.load_runtime(root) GOLDEN = [ "vllm/minimal", "vllm/dual", "vllm/gemma-bf16-mtp", "vllm/gemma-int8-mtp", "vllm/gemma-mtp-tp1", ] # -------------------------------------------------------------------------- # 0. kv_arg map is unit-tested (brief §STEP 3 step 9 — documented + tested). # -------------------------------------------------------------------------- KV_EXPECT = { "bf16": None, "fp16": None, "fp8_e5m2": "fp8_e5m2", "fp8_e4m3": "fp8", "int8_per_token_head": "auto+PTH", "q4_0": "q4_0", "k8v4": "k8v4", } for fmt, want in KV_EXPECT.items(): got = gc.kv_arg(fmt) check(got == want, f"kv_arg({fmt!r})={got!r}, expected {want!r}") def normalize(body: str, markers: list[str]) -> list[str]: """Comment-free service body minus (a) the governed --trust-remote-code token and (b) every patch-wiring insertion-point line for the profile's selected patches. What remains is the constant/param substrate that must reproduce verbatim between shipped and generated.""" out = [] for ln in body.splitlines(): s = ln.strip() if s in ("- --trust-remote-code", "--trust-remote-code"): continue if any(m in ln for m in markers): continue if s == "": continue out.append(ln.rstrip()) return out # -------------------------------------------------------------------------- # 1. Golden-parity per triple. # -------------------------------------------------------------------------- for prof in GOLDEN: prt = (runtime.get("profiles") or {}).get(prof) or {} check(prt.get("genesis_equipped") is False, f"{prof}: golden triple must be genesis_equipped:false") out_text, meta = gc.generate(root, prof) # 3-category header present, above `services:` (so service_body drops it). check("[1] selected + WIRED" in out_text, f"{prof}: header missing category [1]") check("[2] selected + UNDELIVERED" in out_text, f"{prof}: header missing category [2]") check("[3] EXCLUDED" in out_text, f"{prof}: header missing category [3]") check("GENERATED by scripts/generate-compose.sh" in out_text, f"{prof}: header missing provenance line") gen_body = pa.service_body(out_text) shipped_path = root / meta["source"] ship_body = pa.service_body(shipped_path.read_text(encoding="utf-8")) # Header lives above services: -> service_body() must have removed it. check("GENERATED by scripts/generate-compose.sh" not in gen_body, f"{prof}: generator header leaked into the service body") # Image expression reproduces verbatim (correction #2 — never rewritten). shipped_image = next((ln.strip() for ln in ship_body.splitlines() if ln.strip().startswith("image: ")), None) check(shipped_image is not None and shipped_image in gen_body, f"{prof}: image expression not reproduced verbatim") # NO --trust-remote-code emitted for an in-scope profile (correction #1). check("--trust-remote-code" not in gen_body, f"{prof}: --trust-remote-code emitted for in-scope profile") check(meta["trc_emitted"] is False, f"{prof}: meta.trc_emitted must be False") # Semantic diff differs ONLY at the two patch insertion points: with the # governed trc token and the selected patches' wiring lines normalized # away, the remaining body is byte-identical (constants verbatim). selected = gc.select_patches(patches, prof) markers: list[str] = [] for p in selected: markers.extend(gc._patch_wiring_markers(p)) a = normalize(ship_body, markers) b = normalize(gen_body, markers) check(a == b, f"{prof}: semantic diff is NOT confined to the two insertion points " f"(+{len(set(b)-set(a))} / -{len(set(a)-set(b))} non-wiring lines)") # selected+wired subset-of shipped patch set. shipped_reaches = { pid for pid in (meta["wired"] + meta["undelivered"] + meta["degraded_omitted"]) if pa.reaches(root, pmap[pid], meta["source"]) } for pid in meta["wired"]: check(pa.reaches(root, pmap[pid], meta["source"]), f"{prof}: wired patch {pid} is not in the shipped compose " f"(selected+wired must be a subset of shipped)") # wired patches pass reaches() on the GENERATED compose (actual wiring, # not header text — correction #4: the header WARNING block is above # services: and is comment-stripped, so it cannot create a hit). with tempfile.NamedTemporaryFile("w", suffix=".yml", delete=False) as fh: fh.write(out_text) gen_file = fh.name try: for pid in meta["wired"]: check(pa.reaches(root, pmap[pid], gen_file), f"{prof}: wired patch {pid} does not reach the GENERATED compose") # selected-but-undelivered (delivery-gap) patches are NOT reachable # in the generated compose (correction #4). The golden delivery-gap # case is the python_sidecar qwen3coder patch — sound spec markers. for pid in meta["undelivered"]: check(not pa.reaches(root, pmap[pid], gen_file), f"{prof}: undelivered patch {pid} is reachable in the " f"generated compose (must be omitted)") finally: os.unlink(gen_file) # At least one golden carries a real delivery-gap (the correction-#4 case). _, m_min = gc.generate(root, "vllm/minimal") check("qwen-qwen3coder-tool-parser-deferred-commit" in m_min["undelivered"], "vllm/minimal must surface the qwen3coder delivery-gap as undelivered") check("qwen-vllm-pr35936-required-fallback" in m_min["wired"], "vllm/minimal must wire the pr35936 fallback") # -------------------------------------------------------------------------- # 2. Refusals — scope gates + validation + foundational. # -------------------------------------------------------------------------- def expect_refuse(profile: str, code: int, needle: str, accept_degraded=False, env: dict | None = None) -> None: saved = {} if env: for k, v in env.items(): saved[k] = os.environ.get(k) os.environ[k] = v try: try: gc.generate(root, profile, accept_degraded=accept_degraded) errors.append(f"{profile}: expected Refuse({code}) [{needle}], generated OK") except gc.Refuse as r: check(r.code == code, f"{profile}: refuse code {r.code}, expected {code}") check(needle in str(r), f"{profile}: refuse message {str(r)!r} missing {needle!r}") finally: if env: for k, v in saved.items(): if v is None: os.environ.pop(k, None) else: os.environ[k] = v # genesis_equipped:true -> clean scope-gate refuse. No Genesis-equipped # compose remains in the registry post-#254, so synthesize the profile_runtime # bit that scope_gate consumes. genesis_runtime = { "profiles": { "synthetic/genesis": { "genesis_equipped": True, "genesis_equipped_evidence": "synthetic test fixture", } } } try: gc.scope_gate( "vllm-nightly-mtp", gc.load_engine(root, "vllm-nightly-mtp"), genesis_runtime, "synthetic/genesis", ) errors.append("synthetic/genesis: expected genesis_equipped:true refuse") except gc.Refuse as r: check("genesis_equipped:true" in str(r), f"synthetic/genesis: refuse message {str(r)!r} missing genesis_equipped:true") # llama.cpp profile is outside the generator scope; current fixture lacks its engine profile. expect_refuse("llamacpp/default", gc.EXIT_REFUSE, "engine profile not found") # foundational failed-guard -> hard-refuse even with --accept-degraded. expect_refuse("vllm/gemma-int8-mtp", gc.EXIT_REFUSE, "foundational drift-guard failed", accept_degraded=True, env={"CLUB3090_FORCE_GUARD_FAIL": "gemma-vllm-pr40391-rebased"}) # capability-scoped failed-guard -> DEGRADED, needs --accept-degraded. expect_refuse("vllm/gemma-int8-mtp", gc.EXIT_DEGRADED_NOACK, "DEGRADED", accept_degraded=False, env={"CLUB3090_FORCE_GUARD_FAIL": "gemma-vllm-gemma4-tool-parser-fixes"}) # capability-scoped failed-guard WITH --accept-degraded -> proceeds, the # patch is OMITTED (never wired) and the compose is flagged DEGRADED. os.environ["CLUB3090_FORCE_GUARD_FAIL"] = "gemma-vllm-gemma4-tool-parser-fixes" try: dtext, dmeta = gc.generate(root, "vllm/gemma-int8-mtp", accept_degraded=True) check(dmeta["degraded"] is True, "gemma-int8 forced-fail: meta.degraded must be True") check("gemma-vllm-gemma4-tool-parser-fixes" in dmeta["degraded_omitted"], "gemma-int8 forced-fail: tool-parser patch must be in degraded_omitted") check("gemma-vllm-gemma4-tool-parser-fixes" not in dmeta["wired"], "gemma-int8 forced-fail: a failed-guard patch must NEVER be wired") check("WARNING: DEGRADED" in dtext, "gemma-int8 forced-fail: header must carry the DEGRADED warning") dbody = pa.service_body(dtext) leaked = [l for l in dbody.splitlines() if "patches/vllm-gemma4-tool-parser-fixes/" in l and l.strip().startswith("-")] check(not leaked, f"gemma-int8 forced-fail: {len(leaked)} tool-parser overlay mount " f"line(s) leaked into a DEGRADED-omitted compose") finally: os.environ.pop("CLUB3090_FORCE_GUARD_FAIL", None) # -------------------------------------------------------------------------- # 3. Convenience tuple is NOT authoritative — prints matches, non-zero. # -------------------------------------------------------------------------- rc = gc.main(["--model", "gemma-4-31b", "--engine", "vllm-gemma-stable"]) check(rc == gc.EXIT_AMBIGUOUS, f"convenience tuple should exit EXIT_AMBIGUOUS, got {rc}") rc = gc.main([]) check(rc == gc.EXIT_USAGE, f"no args should exit EXIT_USAGE, got {rc}") rc = gc.main(["--profile", "vllm/does-not-exist"]) check(rc == gc.EXIT_USAGE, f"unknown profile should exit EXIT_USAGE, got {rc}") # -------------------------------------------------------------------------- # Verdict. # -------------------------------------------------------------------------- if errors: print("[generate-compose] FAIL") for e in errors: print(f" - {e}") sys.exit(1) print(f"[generate-compose] PASS: {len(GOLDEN)} golden triples, " f"semantic-diff confined to insertion points, reaches() wired/undelivered " f"verified, no --trust-remote-code emitted, refusal+degraded matrix green") PY