[Model] Add Inkling MTP=1 support [3/N] (#48869)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Woosuk Kwon
2026-07-16 13:27:21 -07:00
committed by GitHub
co-authored by Bugen Zhao Giancarlo Delfin Isotr0py Isotr0py Jee Jee Li Roger Wang Yifan Qiao Claude Fable 5 OpenAI Codex
parent 971dac2caa
commit fb5ec0dc9e
11 changed files with 710 additions and 6 deletions
@@ -85,6 +85,34 @@ def test_arch_mapping_applies_before_callable_override():
assert seen_architectures == ["MiMoMTPModel"]
@pytest.mark.cpu_test
def test_inkling_override_exposes_only_first_mtp_depth():
text_config = _make_hf_config(
architectures=["InklingForCausalLM"],
model_type="inkling_model",
local_layer_ids=[1, 3],
)
config = _make_hf_config(
architectures=["InklingForConditionalGeneration"],
model_type="inkling_mm_model",
text_config=text_config,
mtp_config={
"num_nextn_predict_layers": 8,
"local_layer_ids": [0, 2, 4],
},
)
out = SpeculativeConfig.hf_config_override(config)
assert out is text_config
assert out.model_type == "inkling_mtp"
assert out.architectures == ["InklingMTPModel"]
assert out.n_predict == 1
assert out.num_nextn_predict_layers == 8
assert out.chain_hidden_post_norm is False
assert out.local_layer_ids == [0, 2, 4]
def _module_level_shrink(hf_config: PretrainedConfig) -> PretrainedConfig:
hf_config.num_hidden_layers = 1
return hf_config
@@ -8,6 +8,7 @@ from vllm.config.compilation import CompilationConfig, CUDAGraphMode
from vllm.models.inkling.common.mm_preprocess import InklingMultiModalDataParser
from vllm.models.inkling.configs import (
InklingAudioConfig,
InklingModelConfig,
InklingVisionConfig,
)
from vllm.models.inkling.nvidia.sconv_swa_attn import (
@@ -34,6 +35,10 @@ def test_inkling_raw_2d_audio_is_rejected_as_ambiguous():
parser._parse_audio_data(np.zeros((2, 100), dtype=np.float32))
def test_inkling_mtp_chain_norm_is_disabled_by_default():
assert InklingModelConfig().chain_hidden_post_norm is False
def test_inkling_supports_piecewise_cudagraphs():
support = InklingSconvMetadataBuilder.get_cudagraph_support
assert support(None, None) == AttentionCGSupport.UNIFORM_BATCH
@@ -0,0 +1,106 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Bit-exactness tests for the fused MTP depth-layer input kernel.
``embed_dual_rmsnorm_cat`` must match the unfused module sequence exactly:
each rmsnorm computes in fp32 and rounds to bf16 at the same points as the
vendored ``rmsnorm`` kernel (including the bf16 round-trip between the
chained backbone embed_norm and the depth embed_norm), and the fused row
gather matches ``F.embedding``.
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_cuda():
pytest.skip("requires CUDA", allow_module_level=True)
from vllm.models.inkling.nvidia.ops.norm import (
embed_dual_rmsnorm_cat,
embed_rmsnorm,
rmsnorm,
)
EPS = 1e-6
VOCAB = 4096
def _ref(hidden, w_h, w_e, emb, w_pre=None):
if w_pre is not None:
emb = rmsnorm(emb, w_pre, EPS)
return torch.cat([rmsnorm(hidden, w_h, EPS), rmsnorm(emb, w_e, EPS)], dim=-1)
@pytest.mark.parametrize("n", [1536, 6144])
@pytest.mark.parametrize("t", [0, 1, 7, 256])
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
def test_embed_dual_rmsnorm_cat(n: int, t: int, ids_dtype: torch.dtype) -> None:
torch.manual_seed(0)
dev = "cuda"
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
w_h = torch.randn(n, device=dev).to(torch.bfloat16)
w_e = (1 + 0.01 * torch.randn(n, device=dev)).to(torch.bfloat16)
w_pre = torch.randn(n, device=dev).to(torch.bfloat16)
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
hidden = (torch.randn(t, n, device=dev) * 2).to(torch.bfloat16)
emb = table[ids.long()]
# Fused gather + chained backbone pre-norm (the decode draft-step path).
out = embed_dual_rmsnorm_cat(
hidden,
w_h,
w_e,
EPS,
input_ids=ids,
embed_table=table,
pre_norm_weight=w_pre,
)
assert out.shape == (t, 2 * n)
assert torch.equal(out, _ref(hidden, w_h, w_e, emb, w_pre))
# Precomputed embeds, no pre-norm (draft prefill with target-merged MM
# embeddings, already backbone-normed).
out = embed_dual_rmsnorm_cat(hidden, w_h, w_e, EPS, embeds=emb)
assert torch.equal(out, _ref(hidden, w_h, w_e, emb))
# Fused gather, no pre-norm (use_embed_norm=False).
out = embed_dual_rmsnorm_cat(
hidden, w_h, w_e, EPS, input_ids=ids, embed_table=table
)
assert torch.equal(out, _ref(hidden, w_h, w_e, emb))
@pytest.mark.parametrize("n", [1536, 6144])
@pytest.mark.parametrize("t", [0, 1, 7, 256])
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
def test_embed_rmsnorm(n: int, t: int, ids_dtype: torch.dtype) -> None:
torch.manual_seed(0)
dev = "cuda"
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
w = torch.randn(n, device=dev).to(torch.bfloat16)
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
ref_emb = table[ids.long()]
# Gather + embed_norm (base model / MTP prefill embed path).
out = embed_rmsnorm(ids, table, w, EPS)
assert out.shape == (t, n)
assert torch.equal(out, rmsnorm(ref_emb, w, EPS) if t else ref_emb)
# Pure gather (use_embed_norm=False / replicated module forward).
out = embed_rmsnorm(ids, table, None, EPS)
assert torch.equal(out, ref_emb)
# Chained first-layer attn_norm (the target text-path forward): one launch
# emits both the residual and layer 0's normed attention input.
w_chain = (1 + 0.05 * torch.randn(n, device=dev)).to(torch.bfloat16)
res, attn_in = embed_rmsnorm(ids, table, w, EPS, chain_weight=w_chain)
ref_res = rmsnorm(ref_emb, w, EPS) if t else ref_emb
assert torch.equal(res, ref_res)
assert torch.equal(attn_in, rmsnorm(ref_res, w_chain, EPS) if t else ref_res)
# Chained without embed_norm (use_embed_norm=False).
res, attn_in = embed_rmsnorm(ids, table, None, EPS, chain_weight=w_chain)
assert torch.equal(res, ref_emb)
assert torch.equal(attn_in, rmsnorm(ref_emb, w_chain, EPS) if t else ref_emb)
+7
View File
@@ -1656,6 +1656,13 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"tencent/Hy3-preview",
speculative_model="tencent/Hy3-preview",
),
"InklingMTPModel": _HfExamplesInfo(
"thinkingmachines/Inkling-NVFP4",
speculative_model="thinkingmachines/Inkling-NVFP4",
tokenizer_mode="inkling",
trust_remote_code=True,
max_model_len=4096,
),
"LongCatFlashMTPModel": _HfExamplesInfo(
"meituan-longcat/LongCat-Flash-Chat",
trust_remote_code=True,