[Model] Add Inkling MTP=1 support [3/N] (#48869)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Bugen Zhao
Giancarlo Delfin
Isotr0py
Isotr0py
Jee Jee Li
Roger Wang
Yifan Qiao
Claude Fable 5
OpenAI Codex
parent
971dac2caa
commit
fb5ec0dc9e
@@ -55,6 +55,7 @@ MTPModelTypes = Literal[
|
||||
"step3p5_mtp",
|
||||
"hy_v3_mtp",
|
||||
"gemma4_mtp",
|
||||
"inkling_mtp",
|
||||
]
|
||||
NgramGPUTypes = Literal["ngram_gpu"]
|
||||
DFlashModelTypes = Literal["dflash"]
|
||||
@@ -553,6 +554,26 @@ class SpeculativeConfig:
|
||||
{"n_predict": n_predict, "architectures": ["HYV3MTPModel"]}
|
||||
)
|
||||
|
||||
if hf_config.model_type in ("inkling_mm_model", "inkling_model"):
|
||||
mtp_config = getattr(hf_config, "mtp_config", None) or {}
|
||||
hf_config = getattr(hf_config, "text_config", hf_config)
|
||||
checkpoint_depths = mtp_config.get("num_nextn_predict_layers", 0)
|
||||
if checkpoint_depths < 1:
|
||||
raise ValueError("The Inkling checkpoint does not contain MTP weights")
|
||||
hf_config.model_type = "inkling_mtp"
|
||||
hf_config.update(
|
||||
{
|
||||
# Inkling currently exposes only the first checkpoint depth.
|
||||
"n_predict": 1,
|
||||
"num_nextn_predict_layers": checkpoint_depths,
|
||||
"chain_hidden_post_norm": mtp_config.get(
|
||||
"chain_hidden_post_norm", False
|
||||
),
|
||||
"local_layer_ids": mtp_config.get("local_layer_ids", []),
|
||||
"architectures": ["InklingMTPModel"],
|
||||
}
|
||||
)
|
||||
|
||||
if hf_config.model_type in ("gemma4_assistant", "gemma4_unified_assistant"):
|
||||
hf_config.model_type = "gemma4_mtp"
|
||||
text_config = getattr(hf_config, "text_config", hf_config)
|
||||
@@ -874,7 +895,7 @@ class SpeculativeConfig:
|
||||
if (
|
||||
self.num_speculative_tokens > 1
|
||||
and self.draft_model_config.hf_config.model_type
|
||||
!= "step3p5_mtp"
|
||||
not in ("step3p5_mtp", "inkling_mtp")
|
||||
):
|
||||
logger.warning(
|
||||
"Enabling num_speculative_tokens > 1 will run "
|
||||
@@ -953,6 +974,14 @@ class SpeculativeConfig:
|
||||
"`num_speculative_tokens` was not provided"
|
||||
)
|
||||
|
||||
if (
|
||||
self.draft_model_config.hf_config.model_type == "inkling_mtp"
|
||||
and self.num_speculative_tokens != 1
|
||||
):
|
||||
raise ValueError(
|
||||
"Inkling MTP currently supports exactly one speculative token"
|
||||
)
|
||||
|
||||
if self.method == "dspark":
|
||||
# DSpark is a semi-autoregressive *block* drafter. A
|
||||
# speculative length smaller than the checkpoint's block
|
||||
|
||||
Reference in New Issue
Block a user