[Spec Decode] Add Gemma4 MTP speculative decoding support (#41745)

Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com>
Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
Luciano Martins
2026-05-06 22:39:29 +08:00
committed by GitHub
co-authored by Luciano Martins
parent f39bcf1e30
commit 27e0057aed
9 changed files with 1120 additions and 71 deletions
+6
View File
@@ -1494,6 +1494,12 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
trust_remote_code=True,
is_available_online=False,
),
"Gemma4MTPModel": _HfExamplesInfo(
"google/gemma-4-E4B-it",
speculative_model="google/gemma-4-E4B-it-assistant",
trust_remote_code=True,
min_transformers_version="5.8.0",
),
"ErnieMTPModel": _HfExamplesInfo(
"baidu/ERNIE-4.5-21B-A3B-PT",
trust_remote_code=True,
+35 -12
View File
@@ -729,15 +729,20 @@ def test_eagle_correctness_heavy(
False,
0.20,
), # hybrid + MTP, ref: ~34%-35%
(
("mtp", "google/gemma-4-E4B-it", 1, "google/gemma-4-E4B-it-assistant"),
False,
0.50,
), # gemma4 MTP with assistant model, ref: ~62%
],
ids=["mimo", "deepseek", "qwen3_5-hybrid"],
ids=["mimo", "deepseek", "qwen3_5-hybrid", "gemma4-e4b"],
)
@single_gpu_only
@large_gpu_mark(min_gb=20)
def test_mtp_correctness(
monkeypatch: pytest.MonkeyPatch,
sampling_config: SamplingParams,
model_setup: tuple[str, str, int],
model_setup: tuple[str, str, int] | tuple[str, str, int, str],
mm_enabled: bool,
expected_accuracy_threshold: float,
):
@@ -753,7 +758,11 @@ def test_mtp_correctness(
with monkeypatch.context() as m:
m.setenv("VLLM_MLA_DISABLE", "1")
method, model_name, tp_size = model_setup
if len(model_setup) == 4:
method, model_name, tp_size, draft_model = model_setup
else:
method, model_name, tp_size = model_setup
draft_model = None
_skip_if_insufficient_gpus_for_tp(tp_size)
if "Qwen3.5" in model_name and os.environ.get("VLLM_USE_V2_MODEL_RUNNER"):
@@ -764,13 +773,22 @@ def test_mtp_correctness(
attn_backend = "TRITON_ATTN" if current_platform.is_rocm() else "auto"
# Qwen3.5 is a VLM; without this, profile_run runs the ViT warmup
# and peaks well above the 18GB MIG slice used by one of the CI
# lanes. This test only exercises text generation, so the vision
# tower is never needed.
# Skip multimodal profiling for models that don't need it in this test.
extra_kwargs: dict[str, Any] = {}
if "Qwen3.5" in model_name:
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "video": 0}
elif "gemma-4" in model_name:
extra_kwargs["limit_mm_per_prompt"] = {"image": 0, "audio": 0}
if draft_model is not None and "gemma-4" in draft_model:
import transformers
from packaging.version import Version
if Version(transformers.__version__) < Version("5.8.0"):
pytest.skip(
"Gemma4 MTP assistant requires transformers>=5.8.0, "
f"got {transformers.__version__}"
)
ref_llm = LLM(
model=model_name,
@@ -788,15 +806,20 @@ def test_mtp_correctness(
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
speculative_config: dict[str, Any] = {
"method": method,
"num_speculative_tokens": 1,
"max_model_len": 2048,
}
if draft_model is not None:
speculative_config["model"] = draft_model
speculative_config["num_speculative_tokens"] = 2
spec_llm = LLM(
model=model_name,
trust_remote_code=True,
tensor_parallel_size=tp_size,
speculative_config={
"method": method,
"num_speculative_tokens": 1,
"max_model_len": 2048,
},
speculative_config=speculative_config,
max_model_len=2048,
attention_backend=attn_backend,
**extra_kwargs,