[Spec Decode] Add Gemma4 MTP speculative decoding support (#41745)

Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com>
Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
Luciano Martins
2026-05-06 22:39:29 +08:00
committed by GitHub
co-authored by Luciano Martins
parent f39bcf1e30
commit 27e0057aed
9 changed files with 1120 additions and 71 deletions
+24 -6
View File
@@ -169,6 +169,7 @@ from vllm.v1.spec_decode.dflash import DFlashProposer
from vllm.v1.spec_decode.draft_model import DraftModelProposer
from vllm.v1.spec_decode.eagle import EagleProposer
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
from vllm.v1.spec_decode.gemma4 import Gemma4Proposer
from vllm.v1.spec_decode.medusa import MedusaProposer
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
from vllm.v1.spec_decode.ngram_proposer_gpu import (
@@ -524,6 +525,7 @@ class GPUModelRunner(
| DraftModelProposer
| MedusaProposer
| ExtractHiddenStatesProposer
| Gemma4Proposer
)
if self.speculative_config.method == "ngram":
from vllm.v1.spec_decode.ngram_proposer import NgramProposer
@@ -552,6 +554,8 @@ class GPUModelRunner(
self._ngram_pinned_val_buf = torch.zeros(
self.max_num_reqs, dtype=torch.int32, pin_memory=True
)
elif self.speculative_config.use_gemma4_mtp():
self.drafter = Gemma4Proposer(self.vllm_config, self.device, self)
elif self.speculative_config.use_dflash():
self.drafter = DFlashProposer(self.vllm_config, self.device, self)
self.use_aux_hidden_state_outputs = True
@@ -2310,11 +2314,18 @@ class GPUModelRunner(
cm.slot_mapping = slot_mappings[kv_cache_gid]
if self.speculative_config and spec_decode_common_attn_metadata is None:
if isinstance(self.drafter, (EagleProposer, DFlashProposer)):
if isinstance(
self.drafter, (EagleProposer, DFlashProposer, Gemma4Proposer)
):
if self.drafter.kv_cache_gid == kv_cache_gid:
spec_decode_common_attn_metadata = cm
else:
spec_decode_common_attn_metadata = cm
# Capture per-group block tables for multi-group proposers.
if self.speculative_config and isinstance(self.drafter, Gemma4Proposer):
self.drafter.set_per_group_block_table(
kv_cache_gid, cm.block_table_tensor
)
for attn_gid in range(len(self.attn_groups[kv_cache_gid])):
if ubatch_slices is not None:
@@ -4276,7 +4287,8 @@ class GPUModelRunner(
EagleProposer
| DFlashProposer
| DraftModelProposer
| ExtractHiddenStatesProposer,
| ExtractHiddenStatesProposer
| Gemma4Proposer,
)
sampled_token_ids = sampler_output.sampled_token_ids
if input_fits_in_drafter:
@@ -4672,7 +4684,8 @@ class GPUModelRunner(
or spec_config.uses_draft_model()
):
assert isinstance(
self.drafter, EagleProposer | DFlashProposer | DraftModelProposer
self.drafter,
EagleProposer | DFlashProposer | DraftModelProposer | Gemma4Proposer,
)
if spec_config.disable_padded_drafter_batch:
@@ -5594,7 +5607,8 @@ class GPUModelRunner(
EagleProposer
| DFlashProposer
| DraftModelProposer
| ExtractHiddenStatesProposer,
| ExtractHiddenStatesProposer
| Gemma4Proposer,
)
assert self.speculative_config is not None
# Eagle currently only supports PIECEWISE cudagraphs.
@@ -6395,7 +6409,8 @@ class GPUModelRunner(
or self.speculative_config.uses_draft_model()
):
assert isinstance(
self.drafter, EagleProposer | DFlashProposer | DraftModelProposer
self.drafter,
EagleProposer | DFlashProposer | DraftModelProposer | Gemma4Proposer,
)
self.drafter.initialize_attn_backend(kv_cache_config, kernel_block_sizes)
@@ -6448,7 +6463,10 @@ class GPUModelRunner(
):
assert isinstance(
self.drafter,
EagleProposer | DFlashProposer | ExtractHiddenStatesProposer,
EagleProposer
| DFlashProposer
| ExtractHiddenStatesProposer
| Gemma4Proposer,
)
self.drafter.initialize_cudagraph_keys(cudagraph_mode)