club-3090/models/qwen3.6-27b/vllm/compose/single/autoround-int4/minimal.yml

# ===========================================================================
# Profile (at-a-glance):
#   Model:     Qwen3.6-27B (Lorbus AutoRound INT4 + BF16 mtp.fc preserved)
#   Topology:  Single 3090 (TP=1)
#   Drafter:   none (no spec-decode — simplest path)
#   KV:        fp8_e5m2 (no TQ3 to avoid cudagraph cascade bugs)
#   Vision:    no
#   Max ctx:   65K (mem-util 0.95)
#   Genesis:   none — bare vLLM, isolation control
#   Status:    ✅ Production
#   Best for:  Debugging baseline / 20 GB Ampere fallback when TQ3 paths
#              don't fit / first-time setup verification
# ---------------------------------------------------------------------------
# Minimal — no spec-decode, no Genesis patches, no .tolist() patcher.
#
# When to pick this variant:
#   - You want the simplest possible stack (fewer moving parts to debug)
#   - You don't need MTP / spec-decode speedup
#   - You're OK with ~32 TPS instead of the ~50-70 TPS of the spec-decode variants
#   - You want vision + tool calls to "just work" without any patches
#
# Trade-off vs the default `docker-compose.yml`:
#   - This: no MTP, no Genesis → 32/33 TPS narr/code, 32K ctx, vision ✓, tools ✓
#   - Default: MTP + Genesis P65 + TQ3 KV → 51/68 TPS, 48K ctx, vision ✓, tools ✓
#
# This file exists because community feedback (predecessor repo issue #1)
# showed users sometimes hit the spec-decode tool-call cascade by booting
# the wrong compose. This minimal path removes that whole class of failure
# modes by never enabling spec-decode in the first place.
#
# Run:
#   cd compose && docker compose -f single/minimal.yml up -d
# ===========================================================================
# Hardware metadata (parsed by scripts/preflight.sh):
# Requires-min-vram-gb: 20
# Engine-profile: vllm-stable
# Requires-min-gpu-count: 1
# Tensor-parallel: 1
services:
  vllm-qwen36-27b-minimal:
    image: ${VLLM_IMAGE:-vllm/vllm-openai:v0.24.0}
    container_name: "${ESTATE_CONTAINER:-vllm-qwen36-27b-minimal}"
    restart: ${CLUB3090_RESTART:-unless-stopped}
    ports:
      - "${BIND_HOST:-0.0.0.0}:${ESTATE_PORT:-${PORT:-8020}}:8000"
    volumes:
      - ${MODEL_DIR:-../../../../../../models-cache}:/root/.cache/huggingface
      # torch.compile + Triton kernel caches — first boot warms (~60-90 sec);
      # subsequent boots reuse cached graphs. Pattern from Sander's PROD launch.
      # Closes club-3090 #22.
      - ../../../cache/torch_compile:/root/.cache/vllm/torch_compile_cache
      - ../../../cache/triton:/root/.triton/cache
      # (vLLM PR #35936 required-tool-fallback overlay dropped 2026-05-31 with the
      # move to stable v0.22.0 — its serving.py did `import vllm.beam_search`, removed
      # in newer vLLM, so it crashes on v0.22.0; the bug it patched doesn't trip on
      # current stable. See docs/UPSTREAM.md #35936.)
      # froggeric/Qwen-Fixed-Chat-Templates qwen3.6 — fixes 7 default-template
      # bugs (empty <think></think> spam, </thinking> hallucination, unclosed
      # think before tool call, no-user-query crash, developer role, etc.).
      # See docs/UPSTREAM.md "Community templates / model assets" + the row at
      # https://huggingface.co/froggeric/Qwen-Fixed-Chat-Templates
      - ../../../patches/froggeric-chat-template/chat_template.jinja:/etc/qwen-froggeric-chat-template.jinja:ro
    environment:
      - NVIDIA_VISIBLE_DEVICES=${ESTATE_GPUS:-${NVIDIA_VISIBLE_DEVICES:-all}}
      # CUDA-level GPU mask — the runtime-agnostic half of GPU selection
      # (#610): launch.sh --gpus exports GPU UUIDs here. CDI runtimes (NixOS,
      # nvidia-ctk cdi) IGNORE NVIDIA_VISIBLE_DEVICES, so this mask is what
      # actually pins cards there; on the classic runtime the UUID form is
      # renumbering-proof. Unset → absent (no masking).
      - CUDA_VISIBLE_DEVICES
      - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
      - HF_HUB_OFFLINE=${HF_HUB_OFFLINE:-${OFFLINE:-0}}
      - TRANSFORMERS_OFFLINE=${TRANSFORMERS_OFFLINE:-${OFFLINE:-0}}
      - VLLM_WORKER_MULTIPROC_METHOD=spawn
      - NCCL_CUMEM_ENABLE=0
      - NCCL_P2P_DISABLE=1
      - VLLM_NO_USAGE_STATS=1
      # expandable_segments:True crashes boot on some setups (likely cuMemMap path).
      # Known: JusefPol on NVLink (PR #31), WSL2 single-card 3090 Ti.
      # Override via .env: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False
      - PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}
      - OMP_NUM_THREADS=1
    shm_size: "16gb"
    ipc: host
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    entrypoint:
      - /bin/bash
      - -c
      - |
        # VLLM_ENFORCE_EAGER=1 in .env disables CUDA graphs — use on
        # hardware where graph capture causes OOM or instability (e.g. WSL2).
        exec vllm serve ${VLLM_ENFORCE_EAGER:+--enforce-eager} "$@"
      - --
    command:
      - --model
      - /root/.cache/huggingface/qwen3.6-27b-autoround-int4
      - --served-model-name
      - qwen3.6-27b
      - qwen3.6-27b-autoround
      - --quantization
      - auto_round
      - --dtype
      - float16
      - --tensor-parallel-size
      - "${TP:-1}"
      - --pipeline-parallel-size
      - "${PP:-1}"
      - --max-model-len
      - "${MAX_MODEL_LEN:-32768}"
      - --gpu-memory-utilization
      - "${GPU_MEMORY_UTILIZATION:-0.92}"
      - --max-num-seqs
      - "${MAX_NUM_SEQS:-1}"
      - --kv-cache-dtype
      - "${KV_CACHE_DTYPE:-fp8_e5m2}"
      - --trust-remote-code
      # froggeric chat-template override (see volumes block + docs/UPSTREAM.md)
      - --chat-template
      - /etc/qwen-froggeric-chat-template.jinja
      - --reasoning-parser
      - qwen3
      - --default-chat-template-kwargs
      - '{"enable_thinking": false}'
      - --enable-auto-tool-choice
      - --tool-call-parser
      - qwen3_coder
      - --enable-prefix-caching
      - --enable-chunked-prefill
      # No --speculative-config (this is the minimal variant — no spec-decode)
      - --override-generation-config
      - '{"temperature":${TEMP:-${TEMPERATURE:-0.6}},"top_p":${TOP_P:-0.95},"top_k":${TOP_K:-20},"min_p":${MIN_P:-0.0},"repetition_penalty":${REPEAT_PENALTY:-1.0}}'
      - --host
      - 0.0.0.0
      - --port
      - "8000"