club-3090/.env.example

# club-3090 environment overrides
#
# Copy to `.env` (or export inline) and edit only the values you want to
# change. Every variable below has a sensible default in the compose files
# and scripts — `.env` is purely for opt-in overrides.
#
#   cp .env.example .env
#   $EDITOR .env
#
# Variables are read by `docker compose ...` automatically when run from the
# directory that contains a `.env`. For scripts/setup.sh you can either
# `export` them in your shell or prefix the command:
#
#   MODEL_DIR=/scratch/models bash scripts/setup.sh qwen3.6-27b
#
# ⚠ Variable names below are CASE-SENSITIVE. Check carefully:
#     MODEL_DIR    — singular, NOT MODELS_DIR (plural) — that name is silently ignored
#     PORT         — host port for the OpenAI API
#     HF_TOKEN     — HuggingFace token


# -----------------------------------------------------------------------------
# Model storage
# -----------------------------------------------------------------------------

# Where weights live (SINGULAR — `MODEL_DIR`, not `MODELS_DIR`).
# Default is <repo>/models-cache. Override if you keep weights on a separate
# disk (e.g. /mnt/models). Path is resolved as-is — absolute paths recommended.
# MODEL_DIR=/mnt/models

# HuggingFace token. Only needed for gated/private repos. The Lorbus
# AutoRound INT4 weights and unsloth GGUFs we use are public, so this is
# usually blank.
# HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxxxxxxx


# -----------------------------------------------------------------------------
# GPU selection
# -----------------------------------------------------------------------------

# Which GPUs to expose to docker. Single-card composes use just `0`,
# dual-card composes use `0,1`. The compose files set this themselves;
# override here if your physical layout differs (e.g. you run dual-card
# on cards 2,3).
# CUDA_VISIBLE_DEVICES=0,1


# -----------------------------------------------------------------------------
# vLLM tuning knobs
# -----------------------------------------------------------------------------

# GPU memory utilization fraction. The shipped composes use:
#   single-card default (48K):  0.92
#   single-card long-vision/text: 0.98
#   dual-card:                  0.92
# Lowering this leaves more headroom for prefill spikes — see
# docs/INTERNALS.md "prefill cliffs". Raising above 0.95 is risky on
# the 48K config but fine on long-* configs because the model + KV
# fully consume the card anyway.
# MEM_UTIL=0.92

# Max sequence length. Shipped defaults: 48K (chat-safe), 192K (long-vision),
# 205K (long-text). Going beyond these on a 24GB card needs --mem-fraction
# tuning; see docs/USE_CASES.md for the curve.
# MAX_MODEL_LEN=48000

# PyTorch CUDA allocator config. Default keeps `expandable_segments:True`.
# On some setups the engine crashes at boot inside `gptq_marlin_repack` with
# `RuntimeError: CUDA driver error: device not ready`; setting this to
# `expandable_segments:False` resolves it. Known occurrences: JusefPol on
# NVLink (PR #31), this repo on single-card 3090 Ti / WSL2 / driver 596.36.
# See docs/HARDWARE.md "disable PyTorch expandable_segments" for the full
# failure signature.
# PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False


# -----------------------------------------------------------------------------
# Genesis patch tree (vLLM only)
# -----------------------------------------------------------------------------

# Pin Sandermage/genesis-vllm-patches to a specific ref. Default is the exact
# commit our published TPS numbers were measured against. Bumping requires
# re-running scripts/verify-full.sh to confirm your new ref still passes.
# GENESIS_PIN=bf667c7

# Skip cloning Genesis (e.g. you only intend to run llama.cpp / SGLang).
# SKIP_GENESIS=1

# VLLM_ENFORCE_EAGER — disable CUDA graph capture. Costs ~20-30% TPS but
# eliminates graph-capture OOM and Cliff 2 GDN activation spikes at runtime.
# Required on WSL2 at 180K+ context (CUDA graph reservation leaves insufficient
# KV cache headroom). Also useful when CUDA graph capture itself OOMs on boot.
# VLLM_ENFORCE_EAGER=1

# KV cache quantisation. Each compose variant ships a calibrated default:
#   turboquant_3bit_nc — 3-bit TQ k8v4, lowest memory per KV slot, best for
#                        long-context on 24 GB (bounded-thinking, dual-turbo,
#                        long-text, long-vision, docker-compose.yml).
#   fp8_e5m2           — 8-bit FP8, higher quality, used on variants with more
#                        KV headroom (tools-text, minimal, dual, dual-nvlink).
# Override only if you have a specific reason — switching fp8→TQ3 frees ~50%
# KV memory (important at 180K ctx), switching TQ3→fp8 costs that headroom.
# KV_CACHE_DTYPE=turboquant_3bit_nc

# Genesis P40 — TQ k8v4 GQA grouping kernel. Gain: +15-30% TPS on
# compute-regime GPUs (L2 cache >= 24 MB: RTX 5090, A100, H100).
# No gain on RTX 3090 (6 MB L2). Default off.
# GENESIS_ENABLE_P40=1

# Genesis P82 — SGLang-style acceptance threshold OR-clause for MTP spec-decode.
# Biased toward small-batch single-stream workloads (bounded-thinking, default).
# Cross-rig data: +10.5% on 3090 INT4, +12% on A5000 FP8. Default off.
# Note: Genesis will skip this automatically if upstream has absorbed the patch.
# GENESIS_ENABLE_P82=1

# Genesis PN54 — GDN contiguous-call deduplication (Cliff 2b OOM mitigation).
# Reduces peak activation memory during long-context GDN forward passes.
# Default off (opt-in while cross-rig data is still being collected).
# GENESIS_ENABLE_PN54=1


# -----------------------------------------------------------------------------
# Host port mapping (PORT)
# -----------------------------------------------------------------------------

# Host port the OpenAI-compatible API listens on. Each compose binds this
# host port to the container's port 8000 (vLLM) or 8080 (llama.cpp).
#
# Defaults per variant (used when PORT is not set):
#   vllm/default, long-vision, long-text, tools-text, minimal: 8020
#   vllm/dual:                                                  8010
#   vllm/dual-turbo:                                            8011
#   vllm/dual-dflash:                                           8012
#   vllm/dual-dflash-noviz:                                     8013
#   llamacpp/default, llamacpp/concurrent:                      8020
#
# Setting PORT here overrides the default for whichever variant you boot.
# Useful if 8020 is taken by something else, or you want all variants on
# one well-known port.
# PORT=9876

# Host interface to bind the API port on. Default is 0.0.0.0 (all interfaces).
# Set to 127.0.0.1 to restrict to localhost — prevents IDE agents (Cline,
# Cursor) from competing for the max-num-seqs=1 slot and causing verify-stress
# HTTP 000 failures during benchmark runs.
# BIND_HOST=127.0.0.1


# -----------------------------------------------------------------------------
# verify-full.sh / verify-stress.sh
# -----------------------------------------------------------------------------

# Endpoint to test against. Defaults to http://localhost:${PORT:-8020} for
# whichever variant you booted. Override only if your reverse-proxy /
# tunnel sits in between.
# URL=http://localhost:8020

# Number of warmup + measured runs in the bench script. Defaults are
# 3 warmup + 5 measured. Lower these for a faster smoke test.
# WARMUPS=3
# RUNS=5

# Hard wall-clock cap for soak-test.sh. Default: 1800s. On setups using
# VLLM_ENFORCE_EAGER=1 (e.g. WSL2 or 5090 Laptop), long-context prefills
# can exceed the 1800s budget before all 5 sessions complete. 3600s is
# sufficient for the RTX 5090 Laptop with enforce_eager.

# =============================================================================
# Validated rig profiles
# =============================================================================
# Copy the relevant block into your .env and adjust as needed.
#
# ── RTX 5090 Laptop + WSL2 (driver 596.36, 24 GB, EC-managed power) ─────────
# Validated 2026-05-08 against bounded-thinking.yml (issue #102).
# Results: 35.91 narr / 46.43 code wall TPS, TTFT ~165ms, soak PASS 5/5,
#          MTP acceptance 3.32-3.56, avg draft rate 77-85%.
#
# Notes:
#   - VLLM_ENFORCE_EAGER required: CUDA graph reservation eats ~4 GiB, leaving
#     insufficient KV headroom for 180K context on 24 GB VRAM.
#   - KV_CACHE_DTYPE: keep turboquant_3bit_nc (3-bit < fp8 8-bit in KV memory).
#     Switching to fp8_e5m2 raises the KV floor from ~2 GiB to 6.64 GiB and
#     prevents the engine from starting at 180K ctx.
#   - GENESIS_ENABLE_P40: RTX 5090 has 88 MB L2 — well above the 24 MB
#     threshold for the GQA grouping kernel to pay off.
#   - GENESIS_ENABLE_P82: upstream drift in rejection_sampler.py caused Genesis
#     to skip this patch on the 2026-05-08 run (no TPS delta). Re-enable when
#     the pin is bumped and verify with bench.sh.
#   - Software power cap (nvidia-smi -pl): N/A — laptop EC manages the power
#     envelope. Observed load draw ~94 W (95 W EC profile), max 175 W TDP.
#     Use -lgc/-lmc clock-lock (see docs/HARDWARE.md) for sweep characterisation.
#   - SOAK_TIMEOUT_S: 3600 needed because eager-mode prefills at accumulated
#     long context regularly hit 200-290s, exceeding the 1800s default budget
#     before all 5 sessions complete.
#
# PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False,max_split_size_mb:512
# GPU_MEMORY_UTILIZATION=0.94
# VLLM_ENFORCE_EAGER=1
# GENESIS_ENABLE_P40=1
# GENESIS_ENABLE_P82=1
# SOAK_TIMEOUT_S=3600