[Model] Add Inkling MTP=1 support [3/N] (#48869)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Woosuk Kwon
2026-07-16 13:27:21 -07:00
committed by GitHub
co-authored by Bugen Zhao Giancarlo Delfin Isotr0py Isotr0py Jee Jee Li Roger Wang Yifan Qiao Claude Fable 5 OpenAI Codex
parent 971dac2caa
commit fb5ec0dc9e
11 changed files with 710 additions and 6 deletions
+30 -1
View File
@@ -55,6 +55,7 @@ MTPModelTypes = Literal[
"step3p5_mtp",
"hy_v3_mtp",
"gemma4_mtp",
"inkling_mtp",
]
NgramGPUTypes = Literal["ngram_gpu"]
DFlashModelTypes = Literal["dflash"]
@@ -553,6 +554,26 @@ class SpeculativeConfig:
{"n_predict": n_predict, "architectures": ["HYV3MTPModel"]}
)
if hf_config.model_type in ("inkling_mm_model", "inkling_model"):
mtp_config = getattr(hf_config, "mtp_config", None) or {}
hf_config = getattr(hf_config, "text_config", hf_config)
checkpoint_depths = mtp_config.get("num_nextn_predict_layers", 0)
if checkpoint_depths < 1:
raise ValueError("The Inkling checkpoint does not contain MTP weights")
hf_config.model_type = "inkling_mtp"
hf_config.update(
{
# Inkling currently exposes only the first checkpoint depth.
"n_predict": 1,
"num_nextn_predict_layers": checkpoint_depths,
"chain_hidden_post_norm": mtp_config.get(
"chain_hidden_post_norm", False
),
"local_layer_ids": mtp_config.get("local_layer_ids", []),
"architectures": ["InklingMTPModel"],
}
)
if hf_config.model_type in ("gemma4_assistant", "gemma4_unified_assistant"):
hf_config.model_type = "gemma4_mtp"
text_config = getattr(hf_config, "text_config", hf_config)
@@ -874,7 +895,7 @@ class SpeculativeConfig:
if (
self.num_speculative_tokens > 1
and self.draft_model_config.hf_config.model_type
!= "step3p5_mtp"
not in ("step3p5_mtp", "inkling_mtp")
):
logger.warning(
"Enabling num_speculative_tokens > 1 will run "
@@ -953,6 +974,14 @@ class SpeculativeConfig:
"`num_speculative_tokens` was not provided"
)
if (
self.draft_model_config.hf_config.model_type == "inkling_mtp"
and self.num_speculative_tokens != 1
):
raise ValueError(
"Inkling MTP currently supports exactly one speculative token"
)
if self.method == "dspark":
# DSpark is a semi-autoregressive *block* drafter. A
# speculative length smaller than the checkpoint's block