[Spec Decode] Add Gemma4 MTP speculative decoding support (#41745)

Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com>
Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
Luciano Martins
2026-05-06 22:39:29 +08:00
committed by GitHub
co-authored by Luciano Martins
parent f39bcf1e30
commit 27e0057aed
9 changed files with 1120 additions and 71 deletions
@@ -512,6 +512,18 @@ class LongCatFlashMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
return getattr(self.hf_text_config, "num_nextn_predict_layers", 1)
class Gemma4MTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
def get_hidden_size(self) -> int:
# The speculator buffer must match the backbone (target) model's
# hidden dimension, not the draft model's smaller dimension.
return getattr(
self.hf_config, "backbone_hidden_size", super().get_hidden_size()
)
def get_num_hidden_layers(self) -> int:
return getattr(self.hf_text_config, "num_hidden_layers", 0)
class Gemma4ModelArchConfigConvertor(ModelArchConfigConvertorBase):
def is_mm_prefix_lm(self) -> bool:
return (
@@ -541,6 +553,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
"falcon": FalconModelArchConfigConvertor,
"gemma4": Gemma4ModelArchConfigConvertor,
"gemma4_text": Gemma4ModelArchConfigConvertor,
"gemma4_mtp": Gemma4MTPModelArchConfigConvertor,
"RefinedWeb": FalconModelArchConfigConvertor,
"RefinedWebModel": FalconModelArchConfigConvertor,
"nemotron-nas": NemotronNasModelArchConfigConvertor,