diff --git a/models/diffusiongemma-26b-a4b/vllm/compose/dual/fp8/base.yml b/models/diffusiongemma-26b-a4b/vllm/compose/dual/fp8/base.yml index 5ba8043f..12ab5b33 100644 --- a/models/diffusiongemma-26b-a4b/vllm/compose/dual/fp8/base.yml +++ b/models/diffusiongemma-26b-a4b/vllm/compose/dual/fp8/base.yml @@ -2,7 +2,7 @@ # Profile (at-a-glance): # Model: DiffusionGemma 26B-A4B — discrete-diffusion LM / dLLM (RedHatAI # FP8-dynamic, compressed-tensors float-quantized) -# Topology: Dual 3090 PCIe (TP=2, no NVLink) +# Topology: Dual 3090 (TP=2, NVLink auto-detected via NVLINK_MODE — PCIe by default on this rig) # Drafter: none — diffusion LM (block-parallel denoising; AR spec-dec N/A) # KV: bf16 (pool ~408K tok @ util 0.82, 262K-len) # Vision: no (text diffusion) @@ -71,6 +71,8 @@ services: - ../../../patches/gemma-image-fixes/marlin.py:/usr/local/lib/python3.12/dist-packages/vllm/model_executor/kernels/linear/scaled_mm/marlin.py:ro - ../../../patches/gemma-image-fixes/marlin_utils_fp8.py:/usr/local/lib/python3.12/dist-packages/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py:ro - ../../../patches/gemma-image-fixes/diffusion_gemma.py:/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/diffusion_gemma.py:ro + # NVLink auto-detection — runs inside container at boot (same path as dual-max). + - ../../../../../../scripts/detect_nvlink.sh:/etc/club3090/detect_nvlink.sh:ro environment: - NVIDIA_VISIBLE_DEVICES=${ESTATE_GPUS:-${NVIDIA_VISIBLE_DEVICES:-all}} - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} @@ -80,9 +82,12 @@ services: - TORCHDYNAMO_DISABLE=1 - TORCH_COMPILE_DISABLE=1 - VLLM_WORKER_MULTIPROC_METHOD=spawn + - NVLINK_MODE=${NVLINK_MODE:-auto} - NCCL_CUMEM_ENABLE=0 - NCCL_P2P_DISABLE=1 - - VLLM_DISABLE_CUSTOM_ALL_REDUCE=1 + # custom all-reduce is now toggled by the entrypoint (OFF on PCIe via + # --disable-custom-all-reduce, ON when detect_nvlink finds NVLink/P2P) — + # replaces the former static VLLM_DISABLE_CUSTOM_ALL_REDUCE=1. - VLLM_NO_USAGE_STATS=1 - OMP_NUM_THREADS=1 - PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True,max_split_size_mb:512} @@ -98,7 +103,23 @@ services: - driver: nvidia count: all capabilities: [gpu] - # The :gemma image ENTRYPOINT is ["vllm","serve"]; command supplies the model + flags. + # Custom entrypoint overrides the :gemma image's ["vllm","serve"] so detect_nvlink + # can toggle custom all-reduce; command (the model + flags) is passed through as "$@". + entrypoint: + - bash + - -c + - | + # NVLink auto-detect: detect_nvlink.sh sets the NCCL env + _NVLINK_ENABLED; + # vLLM custom all-reduce is ON for NVLink/P2P, OFF (--disable-custom-all-reduce) + # for PCIe. $$ escapes $ so Compose v5.1+ passes the bash through verbatim — an + # un-escaped _NVLINK_ENABLED ref would be interpolated at parse time → forced to 0. + source /etc/club3090/detect_nvlink.sh + if [ "$${_NVLINK_ENABLED:-0}" = "1" ]; then + exec vllm serve "$$@" + else + exec vllm serve --disable-custom-all-reduce "$$@" + fi + - -- command: - /root/.cache/huggingface/diffusiongemma-26b-a4b-it-fp8-dynamic - --served-model-name diff --git a/models/gemma-4-26b-a4b/vllm/compose/dual/awq/mtp.yml b/models/gemma-4-26b-a4b/vllm/compose/dual/awq/mtp.yml index 09ec7e47..1b05b176 100644 --- a/models/gemma-4-26b-a4b/vllm/compose/dual/awq/mtp.yml +++ b/models/gemma-4-26b-a4b/vllm/compose/dual/awq/mtp.yml @@ -1,7 +1,7 @@ # =========================================================================== # Profile (at-a-glance): # Model: Gemma 4 26B-A4B MoE (cyankiwi AWQ-4bit, compressed-tensors) -# Topology: Dual 3090 (TP=2) +# Topology: Dual 3090 (TP=2, NVLink auto-detected via NVLINK_MODE) # Drafter: MTP n=4 — google/gemma-4-26B-A4B-it-assistant (external) # KV: bfloat16 (sidesteps Ampere fp8 dispatch issues) # Vision: off (limit-mm-per-prompt image=0 audio=0) @@ -51,12 +51,15 @@ services: - ${MODEL_DIR:-../../../../../../models-cache}:/root/.cache/huggingface - ../../../cache/torch_compile:/root/.cache/vllm/torch_compile_cache - ../../../cache/triton:/root/.triton/cache + # NVLink auto-detection — runs inside container at boot (same path as dual-max). + - ../../../../../../scripts/detect_nvlink.sh:/etc/club3090/detect_nvlink.sh:ro environment: - NVIDIA_VISIBLE_DEVICES=${ESTATE_GPUS:-${NVIDIA_VISIBLE_DEVICES:-all}} - HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-} - HF_HUB_OFFLINE=${HF_HUB_OFFLINE:-${OFFLINE:-0}} - TRANSFORMERS_OFFLINE=${TRANSFORMERS_OFFLINE:-${OFFLINE:-0}} - VLLM_WORKER_MULTIPROC_METHOD=spawn + - NVLINK_MODE=${NVLINK_MODE:-auto} - NCCL_CUMEM_ENABLE=0 - NCCL_P2P_DISABLE=1 - VLLM_NO_USAGE_STATS=1 @@ -73,6 +76,21 @@ services: - driver: nvidia count: all capabilities: [gpu] + entrypoint: + - bash + - -c + - | + # NVLink auto-detect: detect_nvlink.sh sets the NCCL env + _NVLINK_ENABLED; + # vLLM custom all-reduce is ON for NVLink/P2P, OFF (--disable-custom-all-reduce) + # for PCIe. $$ escapes $ so Compose v5.1+ passes the bash through verbatim — an + # un-escaped _NVLINK_ENABLED ref would be interpolated at parse time → forced to 0. + source /etc/club3090/detect_nvlink.sh + if [ "$${_NVLINK_ENABLED:-0}" = "1" ]; then + exec vllm serve "$$@" + else + exec vllm serve --disable-custom-all-reduce "$$@" + fi + - -- command: - --override-generation-config - '{"temperature":${TEMP:-${TEMPERATURE:-1.0}},"top_p":${TOP_P:-0.95},"top_k":${TOP_K:-64},"min_p":${MIN_P:-0.0},"repetition_penalty":${REPEAT_PENALTY:-1.0}}' diff --git a/models/qwen3-omni-30b-a3b/vllm-omni/compose/dual/autoround-int4/omni.yml b/models/qwen3-omni-30b-a3b/vllm-omni/compose/dual/autoround-int4/omni.yml index c22aacae..a8d90568 100644 --- a/models/qwen3-omni-30b-a3b/vllm-omni/compose/dual/autoround-int4/omni.yml +++ b/models/qwen3-omni-30b-a3b/vllm-omni/compose/dual/autoround-int4/omni.yml @@ -24,6 +24,10 @@ # routes through the Talker (audio) stage, which has a prefill shape bug and kills the engine. # • fp8 KV (default) → full ctx for TEXT, but breaks Code2Wav (audio). Speech out → KV_CACHE_DTYPE=auto. # • Uses BOTH cards (stage-parallel). Don't co-run another GPU-heavy service. +# • NVLink auto-config (the detect_nvlink/NVLINK_MODE custom-all-reduce toggle the other vLLM +# duals carry) is deliberately N/A here: stage-parallel (thinker→GPU0, talker+code2wav→GPU1) +# has no TP all-reduce to toggle, and the cross-stage connector uses host shared memory — so +# NCCL_P2P stays disabled by design (re-enabling P2P would regress, not help). Not missing. # Best for: Qwen3-Omni text + multimodal *understanding* on 2x 3090 (RAG/long-doc, vision/audio in). # --------------------------------------------------------------------------- # See ../../../README.md (engine folder) for the full setup walk-through (weights download,