club-3090/scripts/tests/test-generate-compose.sh

#!/usr/bin/env bash
set -euo pipefail

# test-generate-compose.sh — v0.8.0 STEP 4 (club-3090 #141 / PR #147).
#
# Golden-parity + refusal/degraded contract for scripts/generate-compose.sh.
# This test is the contract; the generator is fixed to it (never the
# reverse). It imports the STEP-2 patch_attribution.reaches() — it does NOT
# reimplement compose parsing or reachability (RED-LINE).
#
# 5 golden triples, all verified genesis_equipped:false in
# profile_runtime.yml, spanning every in-scope engine class:
#   vllm/minimal      vllm-stable, tp1, fp8,  drafter=None
#   vllm/dual         vllm-stable, tp2, fp8,  mtp
#   vllm/gemma-bf16-mtp    vllm-gemma-stable, gemma, bf16
#   vllm/gemma-int8-mtp   vllm-gemma-stable, int8-PTH, multi-file overlay
#   vllm/gemma-mtp-tp1 vllm-gemma-stable, single-card fp8 risk path
#
# Per triple: generate -> semantic diff vs the shipped compose differs ONLY
# at the two patch insertion points (image expression + every constant
# reproduce verbatim) -> selected+wired subset-of shipped -> wired patches
# pass reaches() on the GENERATED compose (actual wiring, not header text)
# -> selected-but-undelivered NOT reachable -> 3-category header present ->
# NO --trust-remote-code emitted. Plus the refusal/degraded matrix.

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"

python3 - "$ROOT_DIR" <<'PY'
from __future__ import annotations

import os
import sys
import tempfile
from pathlib import Path

root = Path(sys.argv[1])
sys.path.insert(0, str(root))

from scripts.lib import generate_compose as gc  # noqa: E402
from scripts.lib.profiles import patch_attribution as pa  # noqa: E402
from scripts.lib.profiles.compose_registry import COMPOSE_REGISTRY  # noqa: E402

errors: list[str] = []


def check(cond: bool, msg: str) -> None:
    if not cond:
        errors.append(msg)


patches = gc.load_patches(root)
pmap = {p["id"]: p for p in patches}
runtime = gc.load_runtime(root)

GOLDEN = [
    "vllm/minimal",
    "vllm/dual",
    "vllm/gemma-bf16-mtp",
    "vllm/gemma-int8-mtp",
    "vllm/gemma-mtp-tp1",
]

# --------------------------------------------------------------------------
# 0. kv_arg map is unit-tested (brief §STEP 3 step 9 — documented + tested).
# --------------------------------------------------------------------------
KV_EXPECT = {
    "bf16": None,
    "fp16": None,
    "fp8_e5m2": "fp8_e5m2",
    "fp8_e4m3": "fp8",
    "int8_per_token_head": "auto+PTH",
    "q4_0": "q4_0",
    "k8v4": "k8v4",
}
for fmt, want in KV_EXPECT.items():
    got = gc.kv_arg(fmt)
    check(got == want, f"kv_arg({fmt!r})={got!r}, expected {want!r}")


def normalize(body: str, markers: list[str]) -> list[str]:
    """Comment-free service body minus (a) the governed --trust-remote-code
    token and (b) every patch-wiring insertion-point line for the profile's
    selected patches. What remains is the constant/param substrate that must
    reproduce verbatim between shipped and generated."""
    out = []
    for ln in body.splitlines():
        s = ln.strip()
        if s in ("- --trust-remote-code", "--trust-remote-code"):
            continue
        if any(m in ln for m in markers):
            continue
        if s == "":
            continue
        out.append(ln.rstrip())
    return out


# --------------------------------------------------------------------------
# 1. Golden-parity per triple.
# --------------------------------------------------------------------------
for prof in GOLDEN:
    prt = (runtime.get("profiles") or {}).get(prof) or {}
    check(prt.get("genesis_equipped") is False,
          f"{prof}: golden triple must be genesis_equipped:false")

    out_text, meta = gc.generate(root, prof)

    # 3-category header present, above `services:` (so service_body drops it).
    check("[1] selected + WIRED" in out_text, f"{prof}: header missing category [1]")
    check("[2] selected + UNDELIVERED" in out_text, f"{prof}: header missing category [2]")
    check("[3] EXCLUDED" in out_text, f"{prof}: header missing category [3]")
    check("GENERATED by scripts/generate-compose.sh" in out_text,
          f"{prof}: header missing provenance line")

    gen_body = pa.service_body(out_text)
    shipped_path = root / meta["source"]
    ship_body = pa.service_body(shipped_path.read_text(encoding="utf-8"))

    # Header lives above services: -> service_body() must have removed it.
    check("GENERATED by scripts/generate-compose.sh" not in gen_body,
          f"{prof}: generator header leaked into the service body")

    # Image expression reproduces verbatim (correction #2 — never rewritten).
    shipped_image = next((ln.strip() for ln in ship_body.splitlines() if ln.strip().startswith("image: ")), None)
    check(shipped_image is not None and shipped_image in gen_body,
          f"{prof}: image expression not reproduced verbatim")

    # NO --trust-remote-code emitted for an in-scope profile (correction #1).
    check("--trust-remote-code" not in gen_body,
          f"{prof}: --trust-remote-code emitted for in-scope profile")
    check(meta["trc_emitted"] is False, f"{prof}: meta.trc_emitted must be False")

    # Semantic diff differs ONLY at the two patch insertion points: with the
    # governed trc token and the selected patches' wiring lines normalized
    # away, the remaining body is byte-identical (constants verbatim).
    selected = gc.select_patches(patches, prof)
    markers: list[str] = []
    for p in selected:
        markers.extend(gc._patch_wiring_markers(p))
    a = normalize(ship_body, markers)
    b = normalize(gen_body, markers)
    check(a == b,
          f"{prof}: semantic diff is NOT confined to the two insertion points "
          f"(+{len(set(b)-set(a))} / -{len(set(a)-set(b))} non-wiring lines)")

    # selected+wired subset-of shipped patch set.
    shipped_reaches = {
        pid for pid in (meta["wired"] + meta["undelivered"] + meta["degraded_omitted"])
        if pa.reaches(root, pmap[pid], meta["source"])
    }
    for pid in meta["wired"]:
        check(pa.reaches(root, pmap[pid], meta["source"]),
              f"{prof}: wired patch {pid} is not in the shipped compose "
              f"(selected+wired must be a subset of shipped)")

    # wired patches pass reaches() on the GENERATED compose (actual wiring,
    # not header text — correction #4: the header WARNING block is above
    # services: and is comment-stripped, so it cannot create a hit).
    with tempfile.NamedTemporaryFile("w", suffix=".yml", delete=False) as fh:
        fh.write(out_text)
        gen_file = fh.name
    try:
        for pid in meta["wired"]:
            check(pa.reaches(root, pmap[pid], gen_file),
                  f"{prof}: wired patch {pid} does not reach the GENERATED compose")

        # selected-but-undelivered (delivery-gap) patches are NOT reachable
        # in the generated compose (correction #4). The golden delivery-gap
        # case is the python_sidecar qwen3coder patch — sound spec markers.
        for pid in meta["undelivered"]:
            check(not pa.reaches(root, pmap[pid], gen_file),
                  f"{prof}: undelivered patch {pid} is reachable in the "
                  f"generated compose (must be omitted)")
    finally:
        os.unlink(gen_file)

# At least one golden carries a real delivery-gap (the correction-#4 case).
_, m_min = gc.generate(root, "vllm/minimal")
check("qwen-qwen3coder-tool-parser-deferred-commit" in m_min["undelivered"],
      "vllm/minimal must surface the qwen3coder delivery-gap as undelivered")
check("qwen-vllm-pr35936-required-fallback" in m_min["wired"],
      "vllm/minimal must wire the pr35936 fallback")


# --------------------------------------------------------------------------
# 2. Refusals — scope gates + validation + foundational.
# --------------------------------------------------------------------------
def expect_refuse(profile: str, code: int, needle: str, accept_degraded=False,
                  env: dict | None = None) -> None:
    saved = {}
    if env:
        for k, v in env.items():
            saved[k] = os.environ.get(k)
            os.environ[k] = v
    try:
        try:
            gc.generate(root, profile, accept_degraded=accept_degraded)
            errors.append(f"{profile}: expected Refuse({code}) [{needle}], generated OK")
        except gc.Refuse as r:
            check(r.code == code,
                  f"{profile}: refuse code {r.code}, expected {code}")
            check(needle in str(r),
                  f"{profile}: refuse message {str(r)!r} missing {needle!r}")
    finally:
        if env:
            for k, v in saved.items():
                if v is None:
                    os.environ.pop(k, None)
                else:
                    os.environ[k] = v


# genesis_equipped:true -> clean scope-gate refuse. No Genesis-equipped
# compose remains in the registry post-#254, so synthesize the profile_runtime
# bit that scope_gate consumes.
genesis_runtime = {
    "profiles": {
        "synthetic/genesis": {
            "genesis_equipped": True,
            "genesis_equipped_evidence": "synthetic test fixture",
        }
    }
}
try:
    gc.scope_gate(
        "vllm-nightly-mtp",
        gc.load_engine(root, "vllm-nightly-mtp"),
        genesis_runtime,
        "synthetic/genesis",
    )
    errors.append("synthetic/genesis: expected genesis_equipped:true refuse")
except gc.Refuse as r:
    check("genesis_equipped:true" in str(r),
          f"synthetic/genesis: refuse message {str(r)!r} missing genesis_equipped:true")

# llama.cpp profile is outside the generator scope; current fixture lacks its engine profile.
expect_refuse("llamacpp/default", gc.EXIT_REFUSE, "engine profile not found")

# foundational failed-guard -> hard-refuse even with --accept-degraded.
expect_refuse("vllm/gemma-int8-mtp", gc.EXIT_REFUSE,
              "foundational drift-guard failed", accept_degraded=True,
              env={"CLUB3090_FORCE_GUARD_FAIL": "gemma-vllm-pr40391-rebased"})

# capability-scoped failed-guard -> DEGRADED, needs --accept-degraded.
expect_refuse("vllm/gemma-int8-mtp", gc.EXIT_DEGRADED_NOACK, "DEGRADED",
              accept_degraded=False,
              env={"CLUB3090_FORCE_GUARD_FAIL": "gemma-vllm-gemma4-tool-parser-fixes"})

# capability-scoped failed-guard WITH --accept-degraded -> proceeds, the
# patch is OMITTED (never wired) and the compose is flagged DEGRADED.
os.environ["CLUB3090_FORCE_GUARD_FAIL"] = "gemma-vllm-gemma4-tool-parser-fixes"
try:
    dtext, dmeta = gc.generate(root, "vllm/gemma-int8-mtp", accept_degraded=True)
    check(dmeta["degraded"] is True,
          "gemma-int8 forced-fail: meta.degraded must be True")
    check("gemma-vllm-gemma4-tool-parser-fixes" in dmeta["degraded_omitted"],
          "gemma-int8 forced-fail: tool-parser patch must be in degraded_omitted")
    check("gemma-vllm-gemma4-tool-parser-fixes" not in dmeta["wired"],
          "gemma-int8 forced-fail: a failed-guard patch must NEVER be wired")
    check("WARNING: DEGRADED" in dtext,
          "gemma-int8 forced-fail: header must carry the DEGRADED warning")
    dbody = pa.service_body(dtext)
    leaked = [l for l in dbody.splitlines()
              if "patches/vllm-gemma4-tool-parser-fixes/" in l and l.strip().startswith("-")]
    check(not leaked,
          f"gemma-int8 forced-fail: {len(leaked)} tool-parser overlay mount "
          f"line(s) leaked into a DEGRADED-omitted compose")
finally:
    os.environ.pop("CLUB3090_FORCE_GUARD_FAIL", None)


# --------------------------------------------------------------------------
# 3. Convenience tuple is NOT authoritative — prints matches, non-zero.
# --------------------------------------------------------------------------
rc = gc.main(["--model", "gemma-4-31b", "--engine", "vllm-gemma-stable"])
check(rc == gc.EXIT_AMBIGUOUS,
      f"convenience tuple should exit EXIT_AMBIGUOUS, got {rc}")
rc = gc.main([])
check(rc == gc.EXIT_USAGE, f"no args should exit EXIT_USAGE, got {rc}")
rc = gc.main(["--profile", "vllm/does-not-exist"])
check(rc == gc.EXIT_USAGE, f"unknown profile should exit EXIT_USAGE, got {rc}")


# --------------------------------------------------------------------------
# Verdict.
# --------------------------------------------------------------------------
if errors:
    print("[generate-compose] FAIL")
    for e in errors:
        print(f"  - {e}")
    sys.exit(1)

print(f"[generate-compose] PASS: {len(GOLDEN)} golden triples, "
      f"semantic-diff confined to insertion points, reaches() wired/undelivered "
      f"verified, no --trust-remote-code emitted, refusal+degraded matrix green")
PY