diff --git a/models/tess-4-27b/vllm/cache/.gitignore b/models/tess-4-27b/vllm/cache/.gitignore new file mode 100644 index 00000000..0f678c06 --- /dev/null +++ b/models/tess-4-27b/vllm/cache/.gitignore @@ -0,0 +1,5 @@ +# Per-variant compile/Triton artifacts. These get regenerated on first boot +# of any variant whose config changed; not source. Don't commit. +* +!.gitignore +!README.md diff --git a/models/tess-4-27b/vllm/compose/dual/nvfp4/fp8.yml b/models/tess-4-27b/vllm/compose/dual/nvfp4/fp8.yml new file mode 100644 index 00000000..c146352f --- /dev/null +++ b/models/tess-4-27b/vllm/compose/dual/nvfp4/fp8.yml @@ -0,0 +1,169 @@ +# =========================================================================== +# Profile (at-a-glance): +# Model: Tess-4-27B NVFP4 (migtissera compressed-tensors: W4A4 recipe, +# FP8-dynamic attention/linear_attn + NVFP4 FFNs; ~19 GB. +# Executes as Marlin W4A16 weight-only on Ampere) +# Topology: Dual 3090 PCIe (TP=2, no NVLink) — AMPERE-VALIDATED serve; +# native FP4 (sm_90+) untested. required_sm=9.0 (native), +# fallback_sm=7.5. +# Drafter: none — DELIBERATE, forensically settled 2026-07-11: the grafted +# base-Qwen MTP head accepts 0% in vLLM (feature-space misaligned; +# works only token-fed in llama.cpp draft-mtp) and the EAGLE3 +# head (~40% accept) is NET-NEGATIVE on this fast trunk (−27%). +# See learnings/tess-4-27b.md "2026-07-11" + club #662. Don't +# re-add without a retrained head. +# KV: fp8 → e4m3 at scale=1.0 (no scale tensors in this checkpoint; +# runtime KV calibration force-disabled on qwen3_5 hybrids — +# measured quality-equivalent regime on this family) +# Vision: yes (tower unquantized in-checkpoint) — UNTESTED on this serve; +# all validation is text-only +# Max ctx: 131K (validated envelope; weights ~9.5 GB/card leave headroom — +# raise MAX_MODEL_LEN at your own risk, unprofiled) +# Genesis: N/A — Genesis is Qwen3-Next-specific (this is qwen3_5) +# Status: 🧪 Experimental +# Best for: the FASTEST Tess on 2× 24 GB — 62.4 tok/s spec-off vs the +# llama.cpp catalog entry's 57.9 with MTP; first vLLM-servable +# Tess on consumer cards +# --------------------------------------------------------------------------- +# FIRST-PARTY VALIDATED ON AMPERE (2026-07-11 forensics week, reference 2×3090): +# - bench: **62.4 tok/s decode spec-off (CV 0.2%)** @131K + fp8 KV — beats +# the shipped llama.cpp Q4+MTP entry (57.9). BENCHMARKS.md Tess section. +# - Boot/serving path exercised heavily across the EAGLE3/MTP forensics +# (dozens of boots, coherence-clean), KV-scale probes, and template A/Bs. +# - 🧪 because the 8-PACK HAS NOT RUN on this serve (scheduled — "A0" in the +# vLLM quality program) and verify-stress/soak are unrun. Promotion gate: +# A0 quality vs the GGUF tier bar (115/118 ±5-7 noise) + stress + soak. +# - PIN RULE (maintainer, 2026-07-11): this compose pins migtissera's NVFP4 +# (the artifact all first-party numbers were measured on). If A0 quality +# undershoots the GGUF bar beyond noise, the fallback pin is +# huginnfork/Tess-4-27B-NVFP4A16 (on-disk; W4A16-calibrated, measured +# KLD 0.0418 vs bf16, regrafted MTP slots) — mechanism: W4A4 weight +# rounding can cost quality even under Ampere's weight-only execution. +# - Template: the repo's chat_template.jinja is STOCK-BROKEN (developer-role +# crash; HF PR with the fix is in flight) — this compose pins the vendored +# froggeric v19 family template regardless, same as the qwen3.6 composes. +# +# ⚠️ Native-FP4 rigs (5090/Hopper/Blackwell): Ampere executes weight-only +# W4A16 — your silicon also quantizes activations (true W4A4), reported to +# cost real quality on this family (~2× KLD). All numbers here are the W4A16 +# bound. If you run this natively, rebench-full + 8-pack and report — the +# native-W4A4 Tess datapoint doesn't exist yet. +# +# Run: +# cd /models/tess-4-27b/vllm/compose +# docker compose -f dual/nvfp4/fp8.yml up -d +# =========================================================================== +# Hardware metadata (parsed by scripts/preflight.sh): +# Requires-min-vram-gb: 24 +# Engine-profile: vllm-stable +# Requires-min-gpu-count: 2 +# Tensor-parallel: 2 +services: + vllm-tess-4-27b-dual-nvfp4: + # Pinned to a vLLM STABLE release (immutable, #407); the launchers inject + # the engine-profile image over this default (#254). + image: ${VLLM_IMAGE:-vllm/vllm-openai:v0.24.0} + container_name: "${ESTATE_CONTAINER:-vllm-tess-4-27b-dual-nvfp4}" + restart: ${CLUB3090_RESTART:-unless-stopped} + ports: + - "${BIND_HOST:-0.0.0.0}:${ESTATE_PORT:-${PORT:-8082}}:8000" + volumes: + - ${MODEL_DIR:-../../../../../../models-cache}:/root/.cache/huggingface + # torch.compile + Triton kernel caches — warm-start across boots. + - ../../../cache/torch_compile:/root/.cache/vllm/torch_compile_cache + - ../../../cache/triton:/root/.triton/cache + # froggeric v19 family chat template — SHARED with the qwen3.6-27b + # composes (same qwen3_5/qwen3.6 template family; single vendored copy, + # see patches.yml qwen-froggeric-chat-template). Required: the model + # repo's own template is stock-broken (developer-role crash). + - ../../../../../qwen3.6-27b/vllm/patches/froggeric-chat-template/chat_template.jinja:/etc/qwen-froggeric-chat-template.jinja:ro + # NVLink auto-detection — runs inside container at boot. + - ../../../../../../scripts/detect_nvlink.sh:/etc/club3090/detect_nvlink.sh:ro + environment: + - NVIDIA_VISIBLE_DEVICES=${ESTATE_GPUS:-${NVIDIA_VISIBLE_DEVICES:-all}} + # CUDA-level GPU mask — see #610; UUID form is renumbering-proof and is + # what CDI runtimes honor. Unset → absent (no masking). + - CUDA_VISIBLE_DEVICES + - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} + - HF_HUB_OFFLINE=${HF_HUB_OFFLINE:-${OFFLINE:-0}} + - TRANSFORMERS_OFFLINE=${TRANSFORMERS_OFFLINE:-${OFFLINE:-0}} + - VLLM_WORKER_MULTIPROC_METHOD=spawn + - NVLINK_MODE=${NVLINK_MODE:-auto} + - NCCL_CUMEM_ENABLE=0 + # Safe PCIe default (mirrors the validated dual composes); rigs with + # working P2P set NCCL_P2P_DISABLE=0. + - NCCL_P2P_DISABLE=${NCCL_P2P_DISABLE:-1} + - VLLM_NO_USAGE_STATS=1 + - OMP_NUM_THREADS=1 + # FP8-GEMM lib guard for consumer Blackwell — launchers auto-inject + # VLLM_USE_DEEP_GEMM=0 there (disc #571); raw compose users set it. + - VLLM_USE_DEEP_GEMM + - PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True} + shm_size: "16gb" + ipc: host + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + entrypoint: + - bash + - -c + - | + # NVLink auto-detection (sets NCCL env vars, _NVLINK_ENABLED). + source /etc/club3090/detect_nvlink.sh + if [ "$${_NVLINK_ENABLED:-0}" = "1" ]; then + exec vllm serve "$$@" + else + exec vllm serve --disable-custom-all-reduce "$$@" + fi + - -- + command: + - --model + - /root/.cache/huggingface/tess-4-27b-nvfp4 + - --served-model-name + - tess-4-27b + - tess-4-27b-nvfp4 + # NO --quantization flag — compressed-tensors auto-detects. Do NOT pass + # `--quantization modelopt` (that's the nvidia-export recipe; it fails + # config validation on this checkpoint). + - --dtype + - bfloat16 + - --tensor-parallel-size + - "${TP:-2}" + - --pipeline-parallel-size + - "${PP:-1}" + - --max-model-len + - "${MAX_MODEL_LEN:-131072}" + - --gpu-memory-utilization + - "${GPU_MEMORY_UTILIZATION:-0.92}" + - --max-num-seqs + - "${MAX_NUM_SEQS:-2}" + - --max-num-batched-tokens + - "8192" + # fp8 (= e4m3) at scale=1.0 — no scale tensors in this checkpoint; + # measured quality-equivalent regime on this family. + - --kv-cache-dtype + - "${KV_CACHE_DTYPE:-fp8}" + - --trust-remote-code + # froggeric v19 pin (see volumes) — the model-dir template is broken. + - --chat-template + - /etc/qwen-froggeric-chat-template.jinja + - --reasoning-parser + - qwen3 + - --default-chat-template-kwargs + - '{"enable_thinking": false}' + - --enable-auto-tool-choice + - --tool-call-parser + - qwen3_coder + - --enable-chunked-prefill + # NO --speculative-config: grafted MTP head = 0% accept in vLLM; + # EAGLE3 = net-negative. Forensically settled — see header. + - --override-generation-config + - '{"temperature":${TEMP:-${TEMPERATURE:-0.6}},"top_p":${TOP_P:-0.95},"top_k":${TOP_K:-20},"min_p":${MIN_P:-0.0},"repetition_penalty":${REPEAT_PENALTY:-1.0}}' + - --host + - 0.0.0.0 + - --port + - "8000" diff --git a/scripts/lib/profiles/compose_registry.py b/scripts/lib/profiles/compose_registry.py index bf224afd..d5af08d3 100644 --- a/scripts/lib/profiles/compose_registry.py +++ b/scripts/lib/profiles/compose_registry.py @@ -893,7 +893,18 @@ COMPOSE_REGISTRY = { default_port=8020, kvcalc_key="SKIP", status="caveats", - status_note="Tess-4-27B (migtissera Q4_K_M GGUF, 16 GB) — Qwen3.5-based dense 27B instruct/agentic fine-tune on dual 3090 llama.cpp. Arch CONFIRMED qwen35-dense (standard GQA, 64 layers) from the GGUF header — same family as Deckard-40B. EXTERNAL MTP n=2 (separate mtp-*.gguf draft via --spec-draft-model, spec_method mtp_gguf) — first external-draft compose in the catalog. q4_0 KV, 262K ctx. Live-validated 2026-07-09 on server-cuda-b9246: decode ~52 narrative / 68 code tok/s (TTFT 233 ms), prefill ~1.3K tok/s; verify-stress 8/8 (NIAH ladder clean to 240,634 tok = 91% of 262K, ~5.9 GB free at deepest fill); soak-continuous PASS (0 err, 0/100 silent-empty, p50 66.4 tok/s, 96.3% retention). Quality (benchlocal --full): core 8-pack 115/150 (77%) think-off, 118/150 (79%) think-on — ties-to-edges the qwen3.6-27b dual-max reference (109) and LEADS the agentic packs (hermesagent 15/20 vs 9, cli-40 25/40 vs 20). CAVEAT: streaming tool-calls + thinking-ON -> finish=length (heavy reasoner blows the token budget before emitting the call); use thinking-OFF (shipped default) for tool/agent streaming. Trades ~1/2 the qwen-dual throughput for a quality tie/edge + vision-capable base + smaller footprint (~12.7+17.2 GB layer-split vs ~22 GB/card TP=2).", + status_note="Tess-4-27B (migtissera Q4_K_M GGUF, 16 GB) — Qwen3.5-based dense 27B instruct/agentic fine-tune on dual 3090 llama.cpp. Arch qwen35-dense (dense = non-MoE; HYBRID attention, 48 linear + 16 full — corrected 2026-07-11) — same family as Deckard-40B. EXTERNAL MTP n=2 (separate mtp-*.gguf draft via --spec-draft-model, spec_method mtp_gguf) — first external-draft compose in the catalog. q4_0 KV, 262K ctx. Live-validated 2026-07-09 on server-cuda-b9246: decode ~52 narrative / 68 code tok/s (TTFT 233 ms), prefill ~1.3K tok/s; verify-stress 8/8 (NIAH ladder clean to 240,634 tok = 91% of 262K, ~5.9 GB free at deepest fill); soak-continuous PASS (0 err, 0/100 silent-empty, p50 66.4 tok/s, 96.3% retention). Quality (benchlocal --full): core 8-pack 115/150 (77%) think-off, 118/150 (79%) think-on — ties-to-edges the qwen3.6-27b dual-max reference (109) and LEADS the agentic packs (hermesagent 15/20 vs 9, cli-40 25/40 vs 20). CAVEAT: streaming tool-calls + thinking-ON -> finish=length (heavy reasoner blows the token budget before emitting the call); use thinking-OFF (shipped default) for tool/agent streaming. Trades ~1/2 the qwen-dual throughput for a quality tie/edge + vision-capable base + smaller footprint (~12.7+17.2 GB layer-split vs ~22 GB/card TP=2).", + ), + + "vllm/tess-dual-nvfp4": _entry( + model="tess-4-27b", weights_variant="nvfp4", workload="fast-chat", + engine="vllm-stable", drafter=None, kv_format="fp8_e4m3", + tp=2, max_ctx=131072, max_num_seqs=2, mem_util=0.92, + compose_path="models/tess-4-27b/vllm/compose/dual/nvfp4/fp8.yml", + default_port=8082, required_sm=9.0, fallback_sm=7.5, + kvcalc_key="SKIP", + status="experimental", + status_note="Tess-4-27B NVFP4 (migtissera compressed-tensors, W4A4 recipe; Marlin W4A16 weight-only on Ampere) at TP=2 @131K + fp8 KV, spec-off — THE FASTEST TESS on 2x24GB: 62.4 tok/s decode (CV 0.2%) vs the llama.cpp catalog entry's 57.9 with MTP, and the first vLLM-servable Tess on consumer cards (validated 2026-07-11, BENCHMARKS). Froggeric template pinned (repo template stock-broken). Drafter-less BY FORENSIC RESULT: grafted MTP head = 0% accept in vLLM (works only token-fed in llama.cpp), EAGLE3 ~40% = net-negative on this trunk (club #662). kvcalc SKIP: Tess is a qwen35 HYBRID (KV on 16/64 layers) and no hybrid kv-calc model exists for it yet — follow-up at promotion. 8-pack NOT yet run on this serve (the vLLM quality program A0 baseline) + stress/soak unrun = the promotion gate; pin-fallback rule to huginnfork NVFP4A16 if A0 undershoots the GGUF bar (learnings). No DEFAULTS row (llamacpp/tess-dual-mtp stays the tess recommendation until A0 lands).", ), # Ornith-1.0-9B — DeepReinforce agentic-coding RL fine-tune. Qwen3-Next DENSE-FFN diff --git a/scripts/lib/profiles/engines/vllm-stable.yml b/scripts/lib/profiles/engines/vllm-stable.yml index 483a2191..e04e09d1 100644 --- a/scripts/lib/profiles/engines/vllm-stable.yml +++ b/scripts/lib/profiles/engines/vllm-stable.yml @@ -15,6 +15,10 @@ supported_model_families: # the ephemeral vllm-gemma4-unified image onto this engine (2026-07-01). - qwen3-next-hybrid - qwen3-next-moe + - qwen35-dense # Tess-4-27B (qwen3_5 hybrid) — LIVE-VALIDATED on stock v0.24.0 + # 2026-07-11 (62.4 tok/s TP=2, forensics-week boots; slug + # vllm/tess-dual-nvfp4). Family also covers Deckard-40B if a + # vLLM path is ever added there. features: int8_per_token_head: true # NATIVE for qwen3-next (uniform head dims) since v0.22.0 — that's why the # flag is true. NOT correct for Gemma-4 (hetero heads): on v0.24.0 int8-PTH diff --git a/scripts/lib/profiles/models/tess-4-27b.yml b/scripts/lib/profiles/models/tess-4-27b.yml index f81e4a54..81e446bb 100644 --- a/scripts/lib/profiles/models/tess-4-27b.yml +++ b/scripts/lib/profiles/models/tess-4-27b.yml @@ -3,11 +3,16 @@ id: tess-4-27b display_name: Tess 4 27B family: qwen35-dense # Migtissera's Tess-4-27B — a Qwen3.5-based dense 27B instruct/agentic fine-tune. -# Arch CONFIRMED from the GGUF header (general.architecture=qwen35): STANDARD GQA -# attention, uniform across all 64 layers — NOT a Qwen3-Next/DeltaNet hybrid (no -# GDN / linear-attention layers). Same qwen35-dense family as Qwen3.6-40B-Deckard, -# 64 layers instead of 97. The base is VL-capable (an F16 mmproj ships alongside -# the weights), but this catalog entry serves TEXT-ONLY — no mmproj is mounted. +# Arch: qwen35 — ⚠️ CORRECTED 2026-07-11: this is a HYBRID (48 linear_attention +# SSM-style + 16 full_attention layers, 3:1 interleave; model_type qwen3_5_text, +# verified from safetensors config). The earlier "standard GQA uniform" claim +# came from the GGUF header, which only describes the full-attention geometry. +# NOTE for kv-calc: the attn fields below describe the 16 FULL-ATTENTION layers; +# KV cache exists on those 16 only — naive 64-layer KV math overestimates ~4× +# (why the vLLM entry ships kvcalc_key=SKIP pending real hybrid modeling). +# Same qwen35-dense family as Qwen3.6-40B-Deckard (also hybrid), 64 layers vs 97. +# The base is VL-capable (an F16 mmproj ships alongside the GGUF weights), but +# the GGUF catalog entry serves TEXT-ONLY — no mmproj is mounted. # A vision variant (mtp-vision.yml) can be added once validated. hidden_size: 5120 num_hidden_layers: 64 @@ -17,12 +22,26 @@ head_dim_attn: 256 # key_length == value_length == 256 intermediate_size: 17408 # feed_forward_length attention_k_eq_v: false # standard GQA — K and V are separate caches (same as Deckard/27B) max_ctx_supported: 262144 +kv_calc_supported: false # HYBRID (KV on 16/64 layers) — not wired into + # tools/kv-calc.py; naive 64-layer math overestimates + # ~4x. Wire + calibrate at vLLM-slug promotion time. # rope_theta = 1e7. MTP: migtissera ships the nextn (MTP) predict-layer as a # SEPARATE GGUF (mtp-Tess-4-27B-Q4_K_M.gguf) — an EXTERNAL draft, engaged via # `--spec-draft-model --spec-type draft-mtp` (drafter: tess-mtp-gguf). # The main quant carries NO embedded nextn head (contrast Deckard, embedded). vision_capable: false weights: + nvfp4: + path: tess-4-27b-nvfp4 + local_subdir: tess-4-27b-nvfp4 + size_gb: 19 + format: compressed-tensors + status: experimental + hf_repo: migtissera/Tess-4-27B-NVFP4 + engine: vllm + kind: main + verify_glob: "*.safetensors" + manual_note: "migtissera's llm-compressor NVFP4 (TRUE W4A4 recipe: 4-bit float gs16 weights AND activations on FFNs, FP8-dynamic attention/linear_attn; single safetensors, 2,672 tensors, 0 mtp — llm-compressor drops the head). On Ampere executes as Marlin W4A16 weight-only → 62.4 tok/s TP=2 @131K (2026-07-11, BENCHMARKS). Quant AUTO-DETECTS (do NOT pass --quantization modelopt). Repo chat_template is STOCK-BROKEN (developer-role crash; our HF fix PR in flight) → composes must pin froggeric. PIN RULE: this is the shipped vLLM pin (all first-party numbers measured on it); fallback = huginnfork/Tess-4-27B-NVFP4A16 (on-disk, W4A16-calibrated, KLD 0.0418) ONLY IF the A0 8-pack undershoots the GGUF tier bar beyond noise. Native-FP4 (sm_90+) quality unmeasured." migtissera-q4km: path: tess-4-27b-gguf/migtissera-q4km local_subdir: tess-4-27b-gguf/migtissera-q4km diff --git a/scripts/lib/profiles/patches.yml b/scripts/lib/profiles/patches.yml index 204323b2..ec8b80ce 100644 --- a/scripts/lib/profiles/patches.yml +++ b/scripts/lib/profiles/patches.yml @@ -236,14 +236,15 @@ patches: status: verified - id: qwen-froggeric-chat-template - model: [qwen3.6-27b] + model: [qwen3.6-27b, tess-4-27b] files: - models/qwen3.6-27b/vllm/patches/froggeric-chat-template/chat_template.jinja load_bearing_when: - composes: - vllm/minimal - vllm/dual - reason: "Default Qwen3.6 chat template mis-handles tool-call XML / reasoning delimiters / streaming (the #145 silent-break class); the froggeric override fixes 7 default-template defects and is behavior-critical for every tool/agent compose. Carnice/Qwopus are intentional excludes (they ship their own carnice template)." + - vllm/tess-dual-nvfp4 + reason: "Default Qwen3.6 chat template mis-handles tool-call XML / reasoning delimiters / streaming (the #145 silent-break class); the froggeric override fixes 7 default-template defects and is behavior-critical for every tool/agent compose. Tess-4-27B (same qwen3_5 template family) additionally NEEDS it because its repo template is stock-broken (developer-role crash). Carnice/Qwopus are intentional excludes (they ship their own carnice template)." evidence: "models/qwen3.6-27b/vllm/patches/froggeric-chat-template/PROVENANCE.md; docs/UPSTREAM.md (froggeric template re-eval #150 / v19 adopted PR #157)" delivery: # DEPRECATED/READ-ONLY (test-only) — see header dockerfile_bake: false diff --git a/scripts/tests/test-compose-registry-disk.sh b/scripts/tests/test-compose-registry-disk.sh index 50ada67e..dfd85ef1 100755 --- a/scripts/tests/test-compose-registry-disk.sh +++ b/scripts/tests/test-compose-registry-disk.sh @@ -25,8 +25,8 @@ def check(cond, msg): print(f"FAIL: {msg}") failures.append(msg) -check(len(COMPOSE_REGISTRY) == 63, f"registry has 63 entries (got {len(COMPOSE_REGISTRY)})") -check(len(disk_paths) == 64, f"disk has 64 compose files (got {len(disk_paths)})") +check(len(COMPOSE_REGISTRY) == 64, f"registry has 64 entries (got {len(COMPOSE_REGISTRY)})") +check(len(disk_paths) == 65, f"disk has 65 compose files (got {len(disk_paths)})") check(registry_paths <= disk_paths, "all registry compose_path values exist on disk") parked_disk_only = disk_paths - registry_paths # Disk-only (non-registry) composes allowed: parked SGLang archives, plus the experimental