[Bugfix][Model] Honor quantization config in Inkling MTP (#61921)
Signed-off-by: Sawyer Bowerman <sbowerma@redhat.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -4,7 +4,7 @@
|
||||
|
||||
Mirrors the reference ``mtp_model.py`` shipped with the checkpoint: each MTP
|
||||
depth ``i`` owns ``hidden_norm`` / ``embed_norm`` RMSNorms, an ``input_proj``
|
||||
(``2H -> H``) and a full Inkling transformer block (dense bf16 MLP, with the
|
||||
(``2H -> H``) and a full Inkling transformer block (dense MLP, with the
|
||||
same short convolutions as the backbone; its attention is full or sliding-window
|
||||
per depth, selected by ``mtp_config.local_layer_ids``). When enabled, a shared
|
||||
``chain_norm`` is applied after every depth; its output is both the logits input
|
||||
@@ -28,18 +28,23 @@ from torch import nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
is_mtp_completeness_check_enabled,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from ..configs import InklingModelConfig
|
||||
from .layernorm import InklingRMSNorm
|
||||
from .model import InklingDecoderLayer, InklingReplicatedEmbedding
|
||||
from .model import (
|
||||
InklingDecoderLayer,
|
||||
InklingReplicatedEmbedding,
|
||||
_TmlForCausalLMBase,
|
||||
)
|
||||
from .ops.norm import embed_dual_rmsnorm_cat, embed_rmsnorm
|
||||
|
||||
# Checkpoint attention projections (wq_du/wk_dv/wv_dv/wr_du) -> fused qkvr.
|
||||
@@ -71,7 +76,13 @@ def _select_mtp_depth_count(n_predict: int, num_spec: int | None) -> int:
|
||||
class InklingMTPDepthLayer(nn.Module):
|
||||
"""One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block."""
|
||||
|
||||
def __init__(self, config: InklingModelConfig, prefix: str, is_local: bool) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
config: InklingModelConfig,
|
||||
prefix: str,
|
||||
is_local: bool,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.embed_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -80,16 +91,17 @@ class InklingMTPDepthLayer(nn.Module):
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.input_proj",
|
||||
)
|
||||
# A force-dense-MLP bf16 block; ``is_local`` selects sliding-window vs
|
||||
# A force-dense-MLP block; ``is_local`` selects sliding-window vs
|
||||
# full attention (the swa_* head config and sliding_window window) to
|
||||
# match this depth's checkpoint transformer_block weights.
|
||||
self.transformer_block = InklingDecoderLayer(
|
||||
config,
|
||||
layer_id=0,
|
||||
is_local=is_local,
|
||||
quant_config=None,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.transformer_block",
|
||||
force_dense_mlp=True,
|
||||
)
|
||||
@@ -131,7 +143,10 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
self.layers = nn.ModuleDict(
|
||||
{
|
||||
str(idx): InklingMTPDepthLayer(
|
||||
config, f"{prefix}.layers.{idx}", idx in local_ids
|
||||
config,
|
||||
f"{prefix}.layers.{idx}",
|
||||
idx in local_ids,
|
||||
quant_config=vllm_config.quant_config,
|
||||
)
|
||||
for idx in range(self.num_mtp_layers)
|
||||
}
|
||||
@@ -244,6 +259,11 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
|
||||
|
||||
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
|
||||
hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_prefix={"model.mtp.": "model."},
|
||||
)
|
||||
packed_modules_mapping = _TmlForCausalLMBase.packed_modules_mapping
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
@@ -418,9 +438,9 @@ def _load_inkling_mtp_weights(
|
||||
continue
|
||||
|
||||
# Dense MLP fused gate/up + down.
|
||||
if ".mlp.w13_dn.weight" in name:
|
||||
if ".mlp.w13_dn." in name:
|
||||
loaded_weight = _load(name.replace(".w13_dn.", ".gate_up_proj."), weight)
|
||||
elif ".mlp.w2_md.weight" in name:
|
||||
elif ".mlp.w2_md." in name:
|
||||
loaded_weight = _load(name.replace(".w2_md.", ".down_proj."), weight)
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params:
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
Mirrors the reference ``mtp_model.py`` shipped with the checkpoint: each MTP
|
||||
depth ``i`` owns ``hidden_norm`` / ``embed_norm`` RMSNorms, an ``input_proj``
|
||||
(``2H -> H``) and a full Inkling transformer block (dense bf16 MLP, with the
|
||||
(``2H -> H``) and a full Inkling transformer block (dense MLP, with the
|
||||
same short convolutions as the backbone; its attention is full or sliding-window
|
||||
per depth, selected by ``mtp_config.local_layer_ids``). When enabled, a shared
|
||||
``chain_norm`` is applied after every depth; its output is both the logits input
|
||||
@@ -28,18 +28,23 @@ from torch import nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
is_mtp_completeness_check_enabled,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModalEmbeddings
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from ..configs import InklingModelConfig
|
||||
from .layernorm import InklingRMSNorm
|
||||
from .model import InklingDecoderLayer, InklingReplicatedEmbedding
|
||||
from .model import (
|
||||
InklingDecoderLayer,
|
||||
InklingReplicatedEmbedding,
|
||||
_TmlForCausalLMBase,
|
||||
)
|
||||
from .ops.norm import embed_dual_rmsnorm_cat, embed_rmsnorm
|
||||
|
||||
# Checkpoint attention projections (wq_du/wk_dv/wv_dv/wr_du) -> fused qkvr.
|
||||
@@ -71,7 +76,13 @@ def _select_mtp_depth_count(n_predict: int, num_spec: int | None) -> int:
|
||||
class InklingMTPDepthLayer(nn.Module):
|
||||
"""One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block."""
|
||||
|
||||
def __init__(self, config: InklingModelConfig, prefix: str, is_local: bool) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
config: InklingModelConfig,
|
||||
prefix: str,
|
||||
is_local: bool,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.embed_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -80,16 +91,17 @@ class InklingMTPDepthLayer(nn.Module):
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.input_proj",
|
||||
)
|
||||
# A force-dense-MLP bf16 block; ``is_local`` selects sliding-window vs
|
||||
# A force-dense-MLP block; ``is_local`` selects sliding-window vs
|
||||
# full attention (the swa_* head config and sliding_window window) to
|
||||
# match this depth's checkpoint transformer_block weights.
|
||||
self.transformer_block = InklingDecoderLayer(
|
||||
config,
|
||||
layer_id=0,
|
||||
is_local=is_local,
|
||||
quant_config=None,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.transformer_block",
|
||||
force_dense_mlp=True,
|
||||
)
|
||||
@@ -131,7 +143,10 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
self.layers = nn.ModuleDict(
|
||||
{
|
||||
str(idx): InklingMTPDepthLayer(
|
||||
config, f"{prefix}.layers.{idx}", idx in local_ids
|
||||
config,
|
||||
f"{prefix}.layers.{idx}",
|
||||
idx in local_ids,
|
||||
quant_config=vllm_config.quant_config,
|
||||
)
|
||||
for idx in range(self.num_mtp_layers)
|
||||
}
|
||||
@@ -244,6 +259,11 @@ class InklingMultiTokenPredictor(nn.Module):
|
||||
|
||||
|
||||
class InklingMTP(nn.Module, SupportsMultiModalEmbeddings):
|
||||
hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper(
|
||||
orig_to_new_prefix={"model.mtp.": "model."},
|
||||
)
|
||||
packed_modules_mapping = _TmlForCausalLMBase.packed_modules_mapping
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
assert vllm_config.speculative_config is not None
|
||||
@@ -418,9 +438,9 @@ def _load_inkling_mtp_weights(
|
||||
continue
|
||||
|
||||
# Dense MLP fused gate/up + down.
|
||||
if ".mlp.w13_dn.weight" in name:
|
||||
if ".mlp.w13_dn." in name:
|
||||
loaded_weight = _load(name.replace(".w13_dn.", ".gate_up_proj."), weight)
|
||||
elif ".mlp.w2_md.weight" in name:
|
||||
elif ".mlp.w2_md." in name:
|
||||
loaded_weight = _load(name.replace(".w2_md.", ".down_proj."), weight)
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params:
|
||||
|
||||
Reference in New Issue
Block a user