From f19139fbef61e0b725497a3d815da7167accfa3c Mon Sep 17 00:00:00 2001 From: noonghunna <10742901+noonghunna@users.noreply.github.com> Date: Sun, 12 Jul 2026 01:03:31 +0000 Subject: [PATCH] nvfp4 composes: document the unsloth (compressed-tensors) provider variant --quantization modelopt matches NVIDIA's export only; unsloth's NVFP4 declares compressed-tensors and errors against the flag (first Discord report 2026-07-12). Inline note: delete the flag pair for unsloth, auto-detect takes over; provider A/B tied so either checkpoint is fine. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EfF565T9eSLaqGzidyJ1Pm --- models/qwen3.6-27b/vllm/compose/dual/nvfp4/mtp.yml | 5 +++++ models/qwen3.6-27b/vllm/compose/single/nvfp4/mtp.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/models/qwen3.6-27b/vllm/compose/dual/nvfp4/mtp.yml b/models/qwen3.6-27b/vllm/compose/dual/nvfp4/mtp.yml index 1c2fc02a..bc8c0045 100644 --- a/models/qwen3.6-27b/vllm/compose/dual/nvfp4/mtp.yml +++ b/models/qwen3.6-27b/vllm/compose/dual/nvfp4/mtp.yml @@ -163,6 +163,11 @@ services: - qwen3.6-27b-nvfp4 # NVIDIA model-card recipe: modelopt quantization (MIXED_PRECISION # NVFP4 + FP8 — vLLM reads hf_quant_config.json / quantization_config). + # ⚠ This flag matches NVIDIA's export (nvidia/Qwen3.6-27B-NVFP4) ONLY. + # Serving unsloth/Qwen3.6-27B-NVFP4 instead? Its config declares + # compressed-tensors → DELETE the two lines below (vLLM auto-detects; + # keeping them errors: "Quantization method ... (compressed-tensors) + # does not match ... (modelopt)"). Provider A/B 2026-07-11: quality/TPS tie. - --quantization - modelopt - --dtype diff --git a/models/qwen3.6-27b/vllm/compose/single/nvfp4/mtp.yml b/models/qwen3.6-27b/vllm/compose/single/nvfp4/mtp.yml index f68b2c9b..23fd8213 100644 --- a/models/qwen3.6-27b/vllm/compose/single/nvfp4/mtp.yml +++ b/models/qwen3.6-27b/vllm/compose/single/nvfp4/mtp.yml @@ -130,6 +130,11 @@ services: - qwen3.6-27b-autoround - qwen3.6-27b-nvfp4 # NVIDIA model-card recipe: modelopt quantization. + # ⚠ This flag matches NVIDIA's export (nvidia/Qwen3.6-27B-NVFP4) ONLY. + # Serving unsloth/Qwen3.6-27B-NVFP4 instead? Its config declares + # compressed-tensors → DELETE the two lines below (vLLM auto-detects; + # keeping them errors: "Quantization method ... (compressed-tensors) + # does not match ... (modelopt)"). Provider A/B 2026-07-11: quality/TPS tie. - --quantization - modelopt - --dtype