[Model]Support Step-3.7-Flash (#43859)
Signed-off-by: luotingdan <luotingdan@stepfun.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: luotingdan <luotingdan@stepfun.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Yu Huang <yuhuang@nvidia.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
co-authored by
luotingdan
Isotr0py
Yu Huang
Jee Jee Li
parent
325a1ec4fb
commit
b690b2bb67
@@ -485,7 +485,16 @@ class SpeculativeConfig:
|
||||
{"n_predict": n_predict, "architectures": ["LongCatFlashMTPModel"]}
|
||||
)
|
||||
|
||||
if hf_config.model_type == "step3p5":
|
||||
if hf_config.model_type in ("step3p5", "step3p7") or hf_config.architectures[
|
||||
0
|
||||
] in ("Step3p5ForCausalLM", "Step3p7ForConditionalGeneration"):
|
||||
quantization_config = getattr(hf_config, "quantization_config", None)
|
||||
hf_config = getattr(hf_config, "text_config", hf_config)
|
||||
if (
|
||||
quantization_config is not None
|
||||
and getattr(hf_config, "quantization_config", None) is None
|
||||
):
|
||||
hf_config.update({"quantization_config": quantization_config})
|
||||
hf_config.model_type = "step3p5_mtp"
|
||||
n_predict = getattr(hf_config, "num_nextn_predict_layers", 1)
|
||||
hf_config.update({"n_predict": n_predict, "architectures": ["Step3p5MTP"]})
|
||||
@@ -705,7 +714,11 @@ class SpeculativeConfig:
|
||||
MTPModelTypes
|
||||
):
|
||||
self.method = "mtp"
|
||||
if self.num_speculative_tokens > 1:
|
||||
if (
|
||||
self.num_speculative_tokens > 1
|
||||
and self.draft_model_config.hf_config.model_type
|
||||
!= "step3p5_mtp"
|
||||
):
|
||||
logger.warning(
|
||||
"Enabling num_speculative_tokens > 1 will run "
|
||||
"multiple times of forward on same MTP layer"
|
||||
@@ -1056,6 +1069,14 @@ class SpeculativeConfig:
|
||||
== "gemma4_mtp"
|
||||
)
|
||||
|
||||
def use_step3p5_mtp(self) -> bool:
|
||||
return (
|
||||
self.method == "mtp"
|
||||
and self.draft_model_config is not None
|
||||
and getattr(self.draft_model_config.hf_config, "model_type", None)
|
||||
== "step3p5_mtp"
|
||||
)
|
||||
|
||||
def use_eagle(self) -> bool:
|
||||
return self.method in ("eagle", "eagle3", "mtp", "dflash")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user