diff --git a/scripts/lib/profiles/envelopes.yml b/scripts/lib/profiles/envelopes.yml index dc2fd37b..97edd366 100644 --- a/scripts/lib/profiles/envelopes.yml +++ b/scripts/lib/profiles/envelopes.yml @@ -6,23 +6,41 @@ # context lever (bigger max_model_len) is deferred (single-card vLLM only, and # Cliff-2b-capped) — see /opt/ai/docs/phase2-32gb-envelope-design.md §6.3. # -# DISCIPLINE (identical to baselines.yml): a value here is a MEASURED, reviewed -# ceiling from a validated concurrency-soak run — NEVER a raw projection. The -# concurrency probe proposes; the soak validates (0 growth / 100% retention / -# 0 silent-empty); this file stores the validated number. A row without a -# `validated` block is a schema error (test-envelopes). +# PROVENANCE (two accepted bases — a bare row with neither is a schema error): +# +# computed The kv-calc pool CEILING at model-max context. Concurrency is a +# CAPACITY question — "do N full-context sequences' KV blocks fit +# the pool?" — not a performance one, so kv-calc answers it +# deterministically (arithmetic, arch-INDEPENDENT; the model math +# doesn't change across GPUs). We seed these because max_num_seqs +# is a cap with GRACEFUL PREEMPTION (never OOM): an over-optimistic +# ceiling costs an occasional preempt, not a crash. kv-calc is +# sm_120-calibrated (disc #571 paulp83's real-5090 verify-stress +# PASS), so a 32 GB projection is trusted arithmetic, not a guess. +# validated A real concurrency-soak on the card class (0 growth / 0 silent- +# empty / bounded VRAM). scripts/concurrency-probe.sh produces it. +# A `validated` block UPGRADES a `computed` row when a volunteer +# runs the soak; keep both if you want the audit trail. +# +# So: kv-calc proposes the ceiling (computed), the probe confirms it (validated). +# Every row MUST carry one block or the other (test-envelopes REDs otherwise). # # HOW IT INJECTS: launch.sh/switch.sh detect the card class (hardware profile # id). If a row exists for (slug, card-class) with a value ABOVE the compose # default, the launcher exports `MAX_NUM_SEQS=`; the compose consumes -# `${MAX_NUM_SEQS:-}`. 24 GB / no-row / heterogeneous / user-env-set -# -> no injection (compose default stands). Consumers read the registry-emit -# join, never this file directly. +# `${MAX_NUM_SEQS:-}`. no-row / user-env-set -> no injection (compose +# default stands). HETEROGENEOUS rigs clamp to the SMALLEST-VRAM card — that is +# exactly the pool vLLM allocates (TP symmetric-shards the KV cache, so the pool +# = min free-blocks across ranks), so a 24 GB card in the mix lands on the +# compose default while a 5090+bigger rig gets the 5090 ceiling. Consumers read +# the registry-emit join, never this file directly. # # Conventions: -# max_num_seqs measured concurrency cap the card's pool sustains. +# max_num_seqs concurrency cap the card's pool sustains (injected value). # compose_default the slug's shipped ${MAX_NUM_SEQS:-N} default (24 GB); # injection only fires when max_num_seqs > this. +# computed { basis: "", +# target_ctx: , date: YYYY-MM-DD, note: ... } # validated { concurrency_soak: "N streams @ , 0-growth", # vram_peak_gb: X, date: YYYY-MM-DD, source_tag: ... } # rig / submitted_by hardware fingerprint class + who ran it. @@ -30,12 +48,47 @@ schema_version: 1 envelopes: - # (empty — awaiting first validated 32 GB concurrency-soak runs) - # - # SEEDING PLAN (each needs a validated concurrency-soak on the card class): - # vllm/dual @ rtx-5090 (2x, 64 GB) — HARVEST from guybrush's fp8w dual - # ladder if it carries multi-stream, else a dedicated soak (disc #571). - # vllm/minimal @ rtx-5090 (1x, 32 GB) — single-5090 concurrency soak - # (the clean new ask, sequenced AFTER the fp8w round). - # 24 GB (rtx-3090) baseline is measured for REFERENCE (what the compose - # default should be) but carries NO row — the compose default already stands. + # --- computed seed (kv-calc pool ceilings @ model-max ctx) ----------------- + # Boundaries pinned 2026-07-05 by sweeping max_num_seqs through kv-calc until + # the KV pool caps. Target context = each slug's model-max (the no-preemption + # policy: N *full-context* concurrent sessions). 24 GB (rtx-3090) carries no + # row — the compose default already stands there. + + vllm/dual: + rtx-5090: + max_num_seqs: 4 # 2 -> 4 concurrent full-262K sessions on 2x32 GB + compose_default: 2 + computed: + basis: "kv-calc pool ceiling — tools/kv-calc.py --compose dual --vram 32 --tp 2 --max-ctx 262144: N=4 PASS, N=5 caps" + target_ctx: 262144 + date: 2026-07-05 + note: "no-preemption ceiling (2x RTX 5090, 64 GB). Arithmetic, sm_120-calibrated (disc #571 paulp verify-stress PASS); awaiting a concurrency-soak to upgrade to validated." + + vllm/minimal: + rtx-5090: + max_num_seqs: 9 # 1 -> 9 concurrent full-32K sessions on 1x32 GB + compose_default: 1 + computed: + basis: "kv-calc pool ceiling — tools/kv-calc.py --compose minimal --vram 32 --tp 1 --max-ctx 32768: N=9 PASS, N=10 caps" + target_ctx: 32768 + date: 2026-07-05 + note: "no-preemption ceiling (1x RTX 5090, 32 GB). Arithmetic, sm_120-calibrated (disc #571 paulp verify-stress PASS); awaiting a concurrency-soak to upgrade to validated." + + # NOT SEEDED yet (computable, deferred on purpose): + # rtx-6000-pro-blackwell (96 GB): pool ceilings run very high (minimal ~40+, + # dual 16+) — those need a latency-aware cap (concurrency past a point + # trades single-stream TPS), not the raw pool ceiling. Seed once we pick + # the policy. Rare card; 5090 is the realistic target first. + # DGX Spark (GB10, sm_121, 128 GB): folds into the SAME framework, but has + # no hardware profile yet — add scripts/lib/profiles/hardware/dgx-spark.yml + # before a row can inject (the guard requires card ∈ hardware profiles). + # MULTI-GPU (TP>2, e.g. vllm/qwen-27b-multi-fast/max @ TP=4): the injection + # seam ALREADY handles homogeneous >2-card rigs — it collapses N identical + # cards to one class, so `vllm/dual` on 4x5090 injects the dual ceiling and + # a TP=4 slug would inject the 4-card ceiling. kv-calc computes it too + # (--tp 4 --gpus 4: dual4 @ 4x5090/128 GB → N=8 ceiling @262K). Not seeded + # because those TP=4 slugs are `experimental` and 4x/8x-5090 rigs are rare + # — same kv-calc sweep seeds them the day one graduates or a volunteer + # appears. Heterogeneous rigs (mixed cards) clamp to the smallest-VRAM + # card (the pool vLLM actually allocates), so a 5090+bigger rig gets the + # 5090 ceiling while any 24 GB card in the mix falls back to the default. diff --git a/scripts/lib/profiles/launch_compat.py b/scripts/lib/profiles/launch_compat.py index 4c0000d5..65db8d2c 100755 --- a/scripts/lib/profiles/launch_compat.py +++ b/scripts/lib/profiles/launch_compat.py @@ -195,9 +195,11 @@ def _arch_aware_env(profiles, variant: str, entry: dict, gpu_spec: str, # --- #246 Phase 2: memory-envelope injection (concurrency-only first pass) --- -# Weights-invariant. Injects MAX_NUM_SEQS from a MEASURED per-(slug, card-class) -# ceiling in envelopes.yml, to spend a bigger card's KV pool on concurrency. -# 24 GB / no-row / heterogeneous / user-env-set -> no injection. +# Weights-invariant. Injects MAX_NUM_SEQS from a per-(slug, card-class) ceiling +# in envelopes.yml (validated soak OR computed kv-calc), to spend a bigger +# card's KV pool on concurrency. no-row / user-env-set -> no injection. +# Heterogeneous rigs clamp to the smallest-VRAM card (which is exactly the pool +# vLLM allocates); a 24 GB card in the mix therefore lands on the compose default. _ENVELOPES_PATH = Path(__file__).with_name("envelopes.yml") @@ -224,10 +226,16 @@ def _envelope_env(profiles, variant: str, gpu_spec: str) -> dict[str, str]: hardware = _parse_gpu_specs(gpu_spec, profiles) except LaunchCompatError: return {} # unmapped card -> compose default - card_ids = {hw.id for hw in hardware} - if len(card_ids) != 1: - return {} # heterogeneous rig -> no single card-class row applies - card_row = row.get(card_ids.pop()) + # Heterogeneous rigs: clamp to the SMALLEST-VRAM card. This mirrors vLLM + # rather than guessing — with TP the KV cache is symmetric-sharded, so the + # engine sizes the pool to min(free blocks) across ranks: the smallest card + # already dictates the pool. A smallest-card ceiling is therefore the real + # ceiling, not a conservative fudge. (Homogeneous rigs collapse to their one + # class. TP=1 stays safe too: the ceiling fits whichever single card vLLM + # runs on — all are >= the smallest.) No-op falls out for free when the + # smallest card has no row: a 24 GB card in the mix -> compose default holds. + smallest = min(hardware, key=lambda hw: hw.vram_gb) + card_row = row.get(smallest.id) if not isinstance(card_row, dict): return {} seqs = card_row.get("max_num_seqs") diff --git a/scripts/tests/test-envelopes.sh b/scripts/tests/test-envelopes.sh index 0029a170..f36f2a73 100644 --- a/scripts/tests/test-envelopes.sh +++ b/scripts/tests/test-envelopes.sh @@ -3,8 +3,9 @@ # injection (#246 Phase 2, concurrency-only first pass). # # REDs on: schema violations · unknown slugs · unknown card-class ids · a row -# missing its `validated` block (born-from-measurement discipline) · a broken -# injection contract. An EMPTY envelopes file is valid (rows land from probes). +# with neither a `validated` (soak) nor a `computed` (kv-calc) provenance block, +# or a `computed` block missing its `basis` (born-from-a-basis discipline) · a +# broken injection contract. An EMPTY envelopes file is valid. set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" @@ -39,9 +40,19 @@ for slug, byc in rows.items(): errors.append(f"{w}.max_num_seqs: required int") if "compose_default" in row and not isinstance(row["compose_default"], int): errors.append(f"{w}.compose_default: int") - # born-from-measurement: a value MUST cite a validated soak - if not isinstance(row.get("validated"), dict) or not row["validated"]: - errors.append(f"{w}.validated: required {{concurrency_soak, ...}} — no guessed rows") + # born-from-a-basis: a value MUST cite a `validated` soak OR a `computed` + # kv-calc ceiling — never a bare guess. (concurrency is a capacity + # question kv-calc answers deterministically; see envelopes.yml header.) + validated = row.get("validated") + computed = row.get("computed") + if not (isinstance(validated, dict) and validated) and \ + not (isinstance(computed, dict) and computed): + errors.append(f"{w}: requires a `validated` (soak) OR `computed` " + "(kv-calc) provenance block — no guessed rows") + # a computed row must name its kv-calc basis (the invocation + boundary) + if isinstance(computed, dict) and computed and not computed.get("basis"): + errors.append(f"{w}.computed.basis: required — cite the kv-calc " + "invocation + PASS/cap boundary") if errors: print("test-envelopes: FAIL", file=sys.stderr) for e in errors: print(f" ✗ {e}", file=sys.stderr) @@ -68,11 +79,38 @@ EOF pin() { python3 "$HELPER" resolve-variant-pin --variant "$1" --format shell --gpu-spec "$2" 2>/dev/null; } S5090="0|RTX 5090|32607|12.0;1|RTX 5090|32607|12.0" S3090="0|RTX 3090|24576|8.6;1|RTX 3090|24576|8.6" -HET="0|RTX 5090|32607|12.0;1|RTX 3090|24576|8.6" +HET_SMALL="0|RTX 5090|32607|12.0;1|RTX 3090|24576|8.6" # smallest = 3090 (no row) +HET_BIG="0|RTX 5090|32607|12.0;1|NVIDIA H100|81920|9.0" # smallest = 5090 (seeded) grep -q "MAX_NUM_SEQS=4" <(pin vllm/dual "$S5090") || fail "5090 with a row must inject MAX_NUM_SEQS=4" grep -q "MAX_NUM_SEQS" <(pin vllm/dual "$S3090") && fail "3090 (no row) must NOT inject" -grep -q "MAX_NUM_SEQS" <(pin vllm/dual "$HET") && fail "heterogeneous rig must NOT inject" +# heterogeneous clamps to the SMALLEST-VRAM card (= the pool vLLM allocates): +# 5090+3090 -> smallest 3090 has no row -> compose default (no inject) +# 5090+H100 -> smallest 5090 is seeded -> inject its ceiling (4) +grep -q "MAX_NUM_SEQS" <(pin vllm/dual "$HET_SMALL") && fail "het rig w/ smallest=3090 (no row) must NOT inject" +grep -q "MAX_NUM_SEQS=4" <(pin vllm/dual "$HET_BIG") || fail "het rig must clamp to smallest-VRAM card (5090) and inject its ceiling" + +# injection is provenance-agnostic: a `computed` row injects exactly like a +# `validated` one (the guard cares about provenance; the launcher does not). +cat > "$REAL" <<'EOF' +schema_version: 1 +envelopes: + vllm/dual: + rtx-5090: + max_num_seqs: 4 + compose_default: 2 + computed: { basis: "kv-calc N=4 PASS, N=5 caps", target_ctx: 262144 } +EOF +grep -q "MAX_NUM_SEQS=4" <(pin vllm/dual "$S5090") || fail "computed row must inject like a validated one" +cat > "$REAL" <<'EOF' +schema_version: 1 +envelopes: + vllm/dual: + rtx-5090: + max_num_seqs: 4 + compose_default: 2 + validated: { concurrency_soak: "4 @262K, 0-growth" } +EOF grep -q "MAX_NUM_SEQS" <(MAX_NUM_SEQS=9 pin vllm/dual "$S5090" | grep "MAX_NUM_SEQS=4") && fail "user env must win" # value at/below compose_default must not fire cat > "$REAL" <<'EOF' @@ -82,7 +120,7 @@ envelopes: rtx-5090: { max_num_seqs: 2, compose_default: 2, validated: { concurrency_soak: "x" } } EOF grep -q "MAX_NUM_SEQS" <(pin vllm/dual "$S5090") && fail "value == compose_default must NOT inject (no gain)" -echo " ✓ injection contract (inject · no-row · heterogeneous · user-env · no-gain)" +echo " ✓ injection contract (inject · no-row · het-clamp-to-smallest · computed-parity · user-env · no-gain)" cp "$BACKUP" "$REAL" [[ "$(sha256sum "$REAL" | cut -d' ' -f1)" == "$REAL_SUM" ]] || fail "real envelopes.yml not restored"