[Bugfix][Model] Honor quantization config in Inkling MTP (#61921)

Signed-off-by: Sawyer Bowerman <sbowerma@redhat.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Sawyer Bowerman
2026-10-16 15:49:26 +00:00
committed by GitHub
co-authored by Codex
parent 1a5b187066
commit 76081b9ee4
2 changed files with 58 additions and 18 deletions
+29 -9
View File
@@ -4,7 +4,7 @@
Mirrors the reference ``mtp_model.py`` shipped with the checkpoint: each MTP
depth ``i`` owns ``hidden_norm`` / ``embed_norm`` RMSNorms, an ``input_proj``
(``2H -> H``) and a full Inkling transformer block (dense bf16 MLP, with the
(``2H -> H``) and a full Inkling transformer block (dense MLP, with the
same short convolutions as the backbone; its attention is full or sliding-window
per depth, selected by ``mtp_config.local_layer_ids``). When enabled, a shared
``chain_norm`` is applied after every depth; its output is both the logits input
@@ -28,18 +28,23 @@ from torch import nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.mtp_validation import (
is_mtp_completeness_check_enabled,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
from vllm.sequence import IntermediateTensors
from ..configs import InklingModelConfig
from .layernorm import InklingRMSNorm
from .model import InklingDecoderLayer, InklingReplicatedEmbedding
from .model import (
InklingDecoderLayer,
InklingReplicatedEmbedding,
_TmlForCausalLMBase,
)
from .ops.norm import embed_dual_rmsnorm_cat, embed_rmsnorm
# Checkpoint attention projections (wq_du/wk_dv/wv_dv/wr_du) -> fused qkvr.
@@ -71,7 +76,13 @@ def _select_mtp_depth_count(n_predict: int, num_spec: int | None) -> int:
class InklingMTPDepthLayer(nn.Module):
"""One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block."""
def __init__(self, config: InklingModelConfig, prefix: str, is_local: bool) -> None:
def __init__(
self,
config: InklingModelConfig,
prefix: str,
is_local: bool,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
self.hidden_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.embed_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -80,16 +91,17 @@ class InklingMTPDepthLayer(nn.Module):
config.hidden_size,
bias=False,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.input_proj",
)
# A force-dense-MLP bf16 block; ``is_local`` selects sliding-window vs
# A force-dense-MLP block; ``is_local`` selects sliding-window vs
# full attention (the swa_* head config and sliding_window window) to
# match this depth's checkpoint transformer_block weights.
self.transformer_block = InklingDecoderLayer(
config,
layer_id=0,
is_local=is_local,
quant_config=None,
quant_config=quant_config,
prefix=f"{prefix}.transformer_block",
force_dense_mlp=True,
)
@@ -131,7 +143,10 @@ class InklingMultiTokenPredictor(nn.Module):
self.layers = nn.ModuleDict(
{
str(idx): InklingMTPDepthLayer(
config, f"{prefix}.layers.{idx}", idx in local_ids
config,
f"{prefix}.layers.{idx}",
idx in local_ids,
quant_config=vllm_config.quant_config,
)
for idx in range(self.num_mtp_layers)
}
@@ -244,6 +259,11 @@ class InklingMultiTokenPredictor(nn.Module):
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_prefix={"model.mtp.": "model."},
)
packed_modules_mapping = _TmlForCausalLMBase.packed_modules_mapping
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
@@ -418,9 +438,9 @@ def _load_inkling_mtp_weights(
continue
# Dense MLP fused gate/up + down.
if ".mlp.w13_dn.weight" in name:
if ".mlp.w13_dn." in name:
loaded_weight = _load(name.replace(".w13_dn.", ".gate_up_proj."), weight)
elif ".mlp.w2_md.weight" in name:
elif ".mlp.w2_md." in name:
loaded_weight = _load(name.replace(".w2_md.", ".down_proj."), weight)
else:
if name.endswith(".bias") and name not in params:
+29 -9
View File
@@ -4,7 +4,7 @@
Mirrors the reference ``mtp_model.py`` shipped with the checkpoint: each MTP
depth ``i`` owns ``hidden_norm`` / ``embed_norm`` RMSNorms, an ``input_proj``
(``2H -> H``) and a full Inkling transformer block (dense bf16 MLP, with the
(``2H -> H``) and a full Inkling transformer block (dense MLP, with the
same short convolutions as the backbone; its attention is full or sliding-window
per depth, selected by ``mtp_config.local_layer_ids``). When enabled, a shared
``chain_norm`` is applied after every depth; its output is both the logits input
@@ -28,18 +28,23 @@ from torch import nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.mtp_validation import (
is_mtp_completeness_check_enabled,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
from vllm.sequence import IntermediateTensors
from ..configs import InklingModelConfig
from .layernorm import InklingRMSNorm
from .model import InklingDecoderLayer, InklingReplicatedEmbedding
from .model import (
InklingDecoderLayer,
InklingReplicatedEmbedding,
_TmlForCausalLMBase,
)
from .ops.norm import embed_dual_rmsnorm_cat, embed_rmsnorm
# Checkpoint attention projections (wq_du/wk_dv/wv_dv/wr_du) -> fused qkvr.
@@ -71,7 +76,13 @@ def _select_mtp_depth_count(n_predict: int, num_spec: int | None) -> int:
class InklingMTPDepthLayer(nn.Module):
"""One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block."""
def __init__(self, config: InklingModelConfig, prefix: str, is_local: bool) -> None:
def __init__(
self,
config: InklingModelConfig,
prefix: str,
is_local: bool,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
self.hidden_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.embed_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -80,16 +91,17 @@ class InklingMTPDepthLayer(nn.Module):
config.hidden_size,
bias=False,
return_bias=False,
quant_config=quant_config,
prefix=f"{prefix}.input_proj",
)
# A force-dense-MLP bf16 block; ``is_local`` selects sliding-window vs
# A force-dense-MLP block; ``is_local`` selects sliding-window vs
# full attention (the swa_* head config and sliding_window window) to
# match this depth's checkpoint transformer_block weights.
self.transformer_block = InklingDecoderLayer(
config,
layer_id=0,
is_local=is_local,
quant_config=None,
quant_config=quant_config,
prefix=f"{prefix}.transformer_block",
force_dense_mlp=True,
)
@@ -131,7 +143,10 @@ class InklingMultiTokenPredictor(nn.Module):
self.layers = nn.ModuleDict(
{
str(idx): InklingMTPDepthLayer(
config, f"{prefix}.layers.{idx}", idx in local_ids
config,
f"{prefix}.layers.{idx}",
idx in local_ids,
quant_config=vllm_config.quant_config,
)
for idx in range(self.num_mtp_layers)
}
@@ -244,6 +259,11 @@ class InklingMultiTokenPredictor(nn.Module):
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_prefix={"model.mtp.": "model."},
)
packed_modules_mapping = _TmlForCausalLMBase.packed_modules_mapping
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
assert vllm_config.speculative_config is not None
@@ -418,9 +438,9 @@ def _load_inkling_mtp_weights(
continue
# Dense MLP fused gate/up + down.
if ".mlp.w13_dn.weight" in name:
if ".mlp.w13_dn." in name:
loaded_weight = _load(name.replace(".w13_dn.", ".gate_up_proj."), weight)
elif ".mlp.w2_md.weight" in name:
elif ".mlp.w2_md." in name:
loaded_weight = _load(name.replace(".w2_md.", ".down_proj."), weight)
else:
if name.endswith(".bias") and name not in params: