[Spec Decode] Add Gemma4 MTP speculative decoding support (#41745)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
co-authored by
Luciano Martins
parent
f39bcf1e30
commit
27e0057aed
@@ -512,6 +512,18 @@ class LongCatFlashMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
return getattr(self.hf_text_config, "num_nextn_predict_layers", 1)
|
||||
|
||||
|
||||
class Gemma4MTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
def get_hidden_size(self) -> int:
|
||||
# The speculator buffer must match the backbone (target) model's
|
||||
# hidden dimension, not the draft model's smaller dimension.
|
||||
return getattr(
|
||||
self.hf_config, "backbone_hidden_size", super().get_hidden_size()
|
||||
)
|
||||
|
||||
def get_num_hidden_layers(self) -> int:
|
||||
return getattr(self.hf_text_config, "num_hidden_layers", 0)
|
||||
|
||||
|
||||
class Gemma4ModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
def is_mm_prefix_lm(self) -> bool:
|
||||
return (
|
||||
@@ -541,6 +553,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
|
||||
"falcon": FalconModelArchConfigConvertor,
|
||||
"gemma4": Gemma4ModelArchConfigConvertor,
|
||||
"gemma4_text": Gemma4ModelArchConfigConvertor,
|
||||
"gemma4_mtp": Gemma4MTPModelArchConfigConvertor,
|
||||
"RefinedWeb": FalconModelArchConfigConvertor,
|
||||
"RefinedWebModel": FalconModelArchConfigConvertor,
|
||||
"nemotron-nas": NemotronNasModelArchConfigConvertor,
|
||||
|
||||
Reference in New Issue
Block a user