gemma-mtp: drop PR #41745 overlay + bump to post-merge nightly
PR #41745 (Gemma 4 MTP support, lucianommartins) merged upstream 2026-05-06 (commit 27e0057a). Today's nightly tag (1acd67a795..., pushed 2026-05-08 06:10 UTC) contains the merge. Phase 1 of a two-phase Gemma 4 cleanup: - gemma-mtp.yml: bump image SHA, drop 7 RO-mount lines for the vendored vLLM patch tree, drop entrypoint transformers==5.8.0 upgrade (verified post-merge nightly already ships transformers 5.8.0 → no need to apt-pip-upgrade at boot) - gemma-mtp-tp1.yml: same edits (TP=1 variant; still upstream-blocked on Ampere by KV format issue tracked as PR #40391, separate concern) Note: Qwen3.6 composes stay on the v7.72.2 PROD pin (01d4d1ad3...) because Genesis allowlist anchors there. Only Gemma 4 composes move to the new nightly. Genesis is Qwen3-Next-only; Gemma 4 doesn't go through the Genesis allowlist. Patches tree models/gemma-4-31b/vllm/patches/vllm-gemma4-mtp/ kept on disk as a fallback safety net pending live boot validation. Will be removed in a follow-up commit once the post-merge nightly boot is validated to produce parity 109/142 TPS narr/code on dual 3090 (the previous bench number with the overlay). gemma-dflash.yml NOT touched — it has its own separate overlay for PR #41703 (still unmerged); only the slightly-stale header comment references vllm-gemma4-mtp. Cleanup deferred until #41703 also lands or until next gemma-dflash.yml edit. Pre-Phase-2 (PR #40391 rebase for per-token-head fp8 KV) — that's the next phase, gated on this Phase 1 boot-validating cleanly. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
noonghunna
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
71e5954ea9
commit
595be8fb8e
@@ -20,12 +20,13 @@
|
||||
# target: Intel/gemma-4-31B-it-int4-AutoRound (21.2 GB, vision preserved)
|
||||
# draft : google/gemma-4-31B-it-assistant (0.5B / 927 MB BF16)
|
||||
#
|
||||
# Pre-merge dependencies (drop when both land in nightly):
|
||||
# 1. vLLM PR #41745 — gemma4_assistant model class + Gemma4Proposer.
|
||||
# Vendored at ../patches/vllm-gemma4-mtp/.
|
||||
# 2. transformers ≥ 5.8.0 — released 2026-05-05 with native gemma4_assistant
|
||||
# support. Image ships 5.7.0; entrypoint upgrades at boot.
|
||||
# See ../patches/vllm-gemma4-mtp/README.md for upgrade triggers.
|
||||
# Pre-merge dependencies — both LANDED 2026-05-08:
|
||||
# 1. vLLM PR #41745 — Gemma 4 MTP — MERGED 2026-05-06 (commit 27e0057a).
|
||||
# Nightly tag `1acd67a795...` (2026-05-08) is post-merge → overlay dropped.
|
||||
# 2. transformers ≥ 5.8.0 — released 2026-05-05. Verified: post-merge nightly
|
||||
# ships transformers 5.8.0 → entrypoint upgrade line dropped.
|
||||
# Pin uses the post-Gemma4-merge nightly. Note: Qwen3.6 composes stay on the
|
||||
# v7.72.2 PROD pin (01d4d1ad3) since Genesis allowlist anchors there.
|
||||
#
|
||||
# KV format pinned in a corner on Ampere:
|
||||
# - fp8_e5m2 → assert fail in gemma4_mm.py:1336 (vLLM allowlist excludes it)
|
||||
@@ -36,7 +37,7 @@
|
||||
# ===========================================================================
|
||||
services:
|
||||
vllm-gemma-4-31b-mtp-tp1:
|
||||
image: vllm/vllm-openai:nightly-01d4d1ad375dc5854779c593eee093bcebb0cada
|
||||
image: vllm/vllm-openai:nightly-1acd67a795ebccdf9b9db7697ae9082058301657
|
||||
container_name: vllm-gemma-4-31b-mtp-tp1
|
||||
restart: "no"
|
||||
ports:
|
||||
@@ -47,17 +48,7 @@ services:
|
||||
# subsequent boots reuse cached graphs (~3 min). Pattern from dual-turbo.yml.
|
||||
- ../cache/torch_compile:/root/.cache/vllm/torch_compile_cache
|
||||
- ../cache/triton:/root/.triton/cache
|
||||
# ---- vLLM PR #41745 overlay (lucianommartins/gemma4-mtp branch) ----
|
||||
# 6 modified files + 1 new file. RO-mount each over installed vllm paths.
|
||||
# Drop this whole block when PR merges + propagates. Source: ../patches/vllm-gemma4-mtp/
|
||||
- ../patches/vllm-gemma4-mtp/config/speculative.py:/usr/local/lib/python3.12/dist-packages/vllm/config/speculative.py:ro
|
||||
- ../patches/vllm-gemma4-mtp/model_executor/models/gemma4_mtp.py:/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/gemma4_mtp.py:ro
|
||||
- ../patches/vllm-gemma4-mtp/model_executor/models/registry.py:/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/registry.py:ro
|
||||
- ../patches/vllm-gemma4-mtp/transformers_utils/model_arch_config_convertor.py:/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/model_arch_config_convertor.py:ro
|
||||
- ../patches/vllm-gemma4-mtp/v1/spec_decode/gemma4.py:/usr/local/lib/python3.12/dist-packages/vllm/v1/spec_decode/gemma4.py:ro
|
||||
- ../patches/vllm-gemma4-mtp/v1/spec_decode/llm_base_proposer.py:/usr/local/lib/python3.12/dist-packages/vllm/v1/spec_decode/llm_base_proposer.py:ro
|
||||
- ../patches/vllm-gemma4-mtp/v1/worker/gpu_model_runner.py:/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py:ro
|
||||
# --------------------------------------------------------------------
|
||||
# PR #41745 overlay dropped 2026-05-08: merged upstream + nightly contains it.
|
||||
environment:
|
||||
- HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
|
||||
- VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
@@ -77,17 +68,9 @@ services:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
# transformers 5.8.0 (released 2026-05-05) is the first version with native
|
||||
# gemma4_assistant support. Image ships 5.7.0; bump in entrypoint until
|
||||
# vLLM nightly rebuilds against ≥5.8.0.
|
||||
entrypoint:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
pip install --quiet --upgrade transformers==5.8.0
|
||||
exec vllm serve "$@"
|
||||
- --
|
||||
# transformers 5.8.0+ ships in the post-Gemma4-merge nightly (verified
|
||||
# 2026-05-08: nightly-1acd67a795... has transformers 5.8.0). Entrypoint
|
||||
# upgrade dropped.
|
||||
command:
|
||||
- --host
|
||||
- 0.0.0.0
|
||||
|
||||
@@ -20,12 +20,13 @@
|
||||
# target: Intel/gemma-4-31B-it-int4-AutoRound (21.2 GB, vision preserved)
|
||||
# draft : google/gemma-4-31B-it-assistant (0.5B / 927 MB BF16)
|
||||
#
|
||||
# Pre-merge dependencies (drop when both land in nightly):
|
||||
# 1. vLLM PR #41745 — gemma4_assistant model class + Gemma4Proposer.
|
||||
# Vendored at ../patches/vllm-gemma4-mtp/.
|
||||
# 2. transformers ≥ 5.8.0 — released 2026-05-05 with native gemma4_assistant
|
||||
# support. Image ships 5.7.0; entrypoint upgrades at boot.
|
||||
# See ../patches/vllm-gemma4-mtp/README.md for upgrade triggers.
|
||||
# Pre-merge dependencies — both LANDED 2026-05-08:
|
||||
# 1. vLLM PR #41745 — Gemma 4 MTP — MERGED 2026-05-06 (commit 27e0057a).
|
||||
# Nightly tag `1acd67a795...` (2026-05-08) is post-merge → overlay dropped.
|
||||
# 2. transformers ≥ 5.8.0 — released 2026-05-05. Verified: post-merge nightly
|
||||
# ships transformers 5.8.0 → entrypoint upgrade line dropped.
|
||||
# Pin uses the post-Gemma4-merge nightly. Note: Qwen3.6 composes stay on the
|
||||
# v7.72.2 PROD pin (01d4d1ad3) since Genesis allowlist anchors there.
|
||||
#
|
||||
# KV format pinned in a corner on Ampere:
|
||||
# - fp8_e5m2 → assert fail in gemma4_mm.py:1336 (vLLM allowlist excludes it)
|
||||
@@ -36,7 +37,7 @@
|
||||
# ===========================================================================
|
||||
services:
|
||||
vllm-gemma-4-31b-mtp:
|
||||
image: vllm/vllm-openai:nightly-01d4d1ad375dc5854779c593eee093bcebb0cada
|
||||
image: vllm/vllm-openai:nightly-1acd67a795ebccdf9b9db7697ae9082058301657
|
||||
container_name: vllm-gemma-4-31b-mtp
|
||||
restart: "no"
|
||||
ports:
|
||||
@@ -47,17 +48,7 @@ services:
|
||||
# subsequent boots reuse cached graphs (~3 min). Pattern from dual-turbo.yml.
|
||||
- ../cache/torch_compile:/root/.cache/vllm/torch_compile_cache
|
||||
- ../cache/triton:/root/.triton/cache
|
||||
# ---- vLLM PR #41745 overlay (lucianommartins/gemma4-mtp branch) ----
|
||||
# 6 modified files + 1 new file. RO-mount each over installed vllm paths.
|
||||
# Drop this whole block when PR merges + propagates. Source: ../patches/vllm-gemma4-mtp/
|
||||
- ../patches/vllm-gemma4-mtp/config/speculative.py:/usr/local/lib/python3.12/dist-packages/vllm/config/speculative.py:ro
|
||||
- ../patches/vllm-gemma4-mtp/model_executor/models/gemma4_mtp.py:/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/gemma4_mtp.py:ro
|
||||
- ../patches/vllm-gemma4-mtp/model_executor/models/registry.py:/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/registry.py:ro
|
||||
- ../patches/vllm-gemma4-mtp/transformers_utils/model_arch_config_convertor.py:/usr/local/lib/python3.12/dist-packages/vllm/transformers_utils/model_arch_config_convertor.py:ro
|
||||
- ../patches/vllm-gemma4-mtp/v1/spec_decode/gemma4.py:/usr/local/lib/python3.12/dist-packages/vllm/v1/spec_decode/gemma4.py:ro
|
||||
- ../patches/vllm-gemma4-mtp/v1/spec_decode/llm_base_proposer.py:/usr/local/lib/python3.12/dist-packages/vllm/v1/spec_decode/llm_base_proposer.py:ro
|
||||
- ../patches/vllm-gemma4-mtp/v1/worker/gpu_model_runner.py:/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu_model_runner.py:ro
|
||||
# --------------------------------------------------------------------
|
||||
# PR #41745 overlay dropped 2026-05-08: merged upstream + nightly contains it.
|
||||
environment:
|
||||
- HUGGING_FACE_HUB_TOKEN=${HF_TOKEN:-}
|
||||
- VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
@@ -77,17 +68,9 @@ services:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
# transformers 5.8.0 (released 2026-05-05) is the first version with native
|
||||
# gemma4_assistant support. Image ships 5.7.0; bump in entrypoint until
|
||||
# vLLM nightly rebuilds against ≥5.8.0.
|
||||
entrypoint:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
pip install --quiet --upgrade transformers==5.8.0
|
||||
exec vllm serve "$@"
|
||||
- --
|
||||
# transformers 5.8.0+ ships in the post-Gemma4-merge nightly (verified
|
||||
# 2026-05-08: nightly-1acd67a795... has transformers 5.8.0). Entrypoint
|
||||
# upgrade dropped.
|
||||
command:
|
||||
- --host
|
||||
- 0.0.0.0
|
||||
|
||||
Reference in New Issue
Block a user