[Spec Decode] Add Gemma4 MTP speculative decoding support (#41745)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
co-authored by
Luciano Martins
parent
f39bcf1e30
commit
27e0057aed
@@ -169,6 +169,7 @@ from vllm.v1.spec_decode.dflash import DFlashProposer
|
||||
from vllm.v1.spec_decode.draft_model import DraftModelProposer
|
||||
from vllm.v1.spec_decode.eagle import EagleProposer
|
||||
from vllm.v1.spec_decode.extract_hidden_states import ExtractHiddenStatesProposer
|
||||
from vllm.v1.spec_decode.gemma4 import Gemma4Proposer
|
||||
from vllm.v1.spec_decode.medusa import MedusaProposer
|
||||
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
|
||||
from vllm.v1.spec_decode.ngram_proposer_gpu import (
|
||||
@@ -524,6 +525,7 @@ class GPUModelRunner(
|
||||
| DraftModelProposer
|
||||
| MedusaProposer
|
||||
| ExtractHiddenStatesProposer
|
||||
| Gemma4Proposer
|
||||
)
|
||||
if self.speculative_config.method == "ngram":
|
||||
from vllm.v1.spec_decode.ngram_proposer import NgramProposer
|
||||
@@ -552,6 +554,8 @@ class GPUModelRunner(
|
||||
self._ngram_pinned_val_buf = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.int32, pin_memory=True
|
||||
)
|
||||
elif self.speculative_config.use_gemma4_mtp():
|
||||
self.drafter = Gemma4Proposer(self.vllm_config, self.device, self)
|
||||
elif self.speculative_config.use_dflash():
|
||||
self.drafter = DFlashProposer(self.vllm_config, self.device, self)
|
||||
self.use_aux_hidden_state_outputs = True
|
||||
@@ -2310,11 +2314,18 @@ class GPUModelRunner(
|
||||
cm.slot_mapping = slot_mappings[kv_cache_gid]
|
||||
|
||||
if self.speculative_config and spec_decode_common_attn_metadata is None:
|
||||
if isinstance(self.drafter, (EagleProposer, DFlashProposer)):
|
||||
if isinstance(
|
||||
self.drafter, (EagleProposer, DFlashProposer, Gemma4Proposer)
|
||||
):
|
||||
if self.drafter.kv_cache_gid == kv_cache_gid:
|
||||
spec_decode_common_attn_metadata = cm
|
||||
else:
|
||||
spec_decode_common_attn_metadata = cm
|
||||
# Capture per-group block tables for multi-group proposers.
|
||||
if self.speculative_config and isinstance(self.drafter, Gemma4Proposer):
|
||||
self.drafter.set_per_group_block_table(
|
||||
kv_cache_gid, cm.block_table_tensor
|
||||
)
|
||||
|
||||
for attn_gid in range(len(self.attn_groups[kv_cache_gid])):
|
||||
if ubatch_slices is not None:
|
||||
@@ -4276,7 +4287,8 @@ class GPUModelRunner(
|
||||
EagleProposer
|
||||
| DFlashProposer
|
||||
| DraftModelProposer
|
||||
| ExtractHiddenStatesProposer,
|
||||
| ExtractHiddenStatesProposer
|
||||
| Gemma4Proposer,
|
||||
)
|
||||
sampled_token_ids = sampler_output.sampled_token_ids
|
||||
if input_fits_in_drafter:
|
||||
@@ -4672,7 +4684,8 @@ class GPUModelRunner(
|
||||
or spec_config.uses_draft_model()
|
||||
):
|
||||
assert isinstance(
|
||||
self.drafter, EagleProposer | DFlashProposer | DraftModelProposer
|
||||
self.drafter,
|
||||
EagleProposer | DFlashProposer | DraftModelProposer | Gemma4Proposer,
|
||||
)
|
||||
|
||||
if spec_config.disable_padded_drafter_batch:
|
||||
@@ -5594,7 +5607,8 @@ class GPUModelRunner(
|
||||
EagleProposer
|
||||
| DFlashProposer
|
||||
| DraftModelProposer
|
||||
| ExtractHiddenStatesProposer,
|
||||
| ExtractHiddenStatesProposer
|
||||
| Gemma4Proposer,
|
||||
)
|
||||
assert self.speculative_config is not None
|
||||
# Eagle currently only supports PIECEWISE cudagraphs.
|
||||
@@ -6395,7 +6409,8 @@ class GPUModelRunner(
|
||||
or self.speculative_config.uses_draft_model()
|
||||
):
|
||||
assert isinstance(
|
||||
self.drafter, EagleProposer | DFlashProposer | DraftModelProposer
|
||||
self.drafter,
|
||||
EagleProposer | DFlashProposer | DraftModelProposer | Gemma4Proposer,
|
||||
)
|
||||
self.drafter.initialize_attn_backend(kv_cache_config, kernel_block_sizes)
|
||||
|
||||
@@ -6448,7 +6463,10 @@ class GPUModelRunner(
|
||||
):
|
||||
assert isinstance(
|
||||
self.drafter,
|
||||
EagleProposer | DFlashProposer | ExtractHiddenStatesProposer,
|
||||
EagleProposer
|
||||
| DFlashProposer
|
||||
| ExtractHiddenStatesProposer
|
||||
| Gemma4Proposer,
|
||||
)
|
||||
self.drafter.initialize_cudagraph_keys(cudagraph_mode)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user