[Spec Decode] Add Gemma4 MTP speculative decoding support (#41745)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
co-authored by
Luciano Martins
parent
f39bcf1e30
commit
27e0057aed
@@ -50,6 +50,7 @@ MTPModelTypes = Literal[
|
||||
"pangu_ultra_moe_mtp",
|
||||
"step3p5_mtp",
|
||||
"hy_v3_mtp",
|
||||
"gemma4_mtp",
|
||||
]
|
||||
NgramGPUTypes = Literal["ngram_gpu"]
|
||||
DFlashModelTypes = Literal["dflash"]
|
||||
@@ -491,6 +492,17 @@ class SpeculativeConfig:
|
||||
{"n_predict": n_predict, "architectures": ["HYV3MTPModel"]}
|
||||
)
|
||||
|
||||
if hf_config.model_type == "gemma4_assistant":
|
||||
hf_config.model_type = "gemma4_mtp"
|
||||
text_config = getattr(hf_config, "text_config", hf_config)
|
||||
# The assistant runs all decoder layers in a single forward
|
||||
# call to produce one draft token, so n_predict=1.
|
||||
# num_kv_shared_layers must be 0: cross-model KV sharing is
|
||||
# set up by the proposer after model construction.
|
||||
if hasattr(text_config, "num_kv_shared_layers"):
|
||||
text_config.num_kv_shared_layers = 0
|
||||
hf_config.update({"n_predict": 1, "architectures": ["Gemma4MTPModel"]})
|
||||
|
||||
return hf_config
|
||||
|
||||
def __post_init__(self):
|
||||
@@ -1040,6 +1052,14 @@ class SpeculativeConfig:
|
||||
slots_per_req += 1
|
||||
return slots_per_req
|
||||
|
||||
def use_gemma4_mtp(self) -> bool:
|
||||
return (
|
||||
self.method == "mtp"
|
||||
and self.draft_model_config is not None
|
||||
and getattr(self.draft_model_config.hf_config, "model_type", None)
|
||||
== "gemma4_mtp"
|
||||
)
|
||||
|
||||
def use_eagle(self) -> bool:
|
||||
return self.method in ("eagle", "eagle3", "mtp", "dflash")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user