Files
2026-10-15 20:11:17 -05:00

488 lines
19 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Qwen4Exp MTP (Multi-Token Predictor) model.
The MTP draft model reuses the Qwen4Exp backbone (PLE/HC/MoE) but:
- drops all multi-modal handling (text-only),
- forces PLE off while keeping the main model's HC stream count,
- fuses the backbone hidden and the new-token embedding via
``residual_linear_shared`` (fc_embedding + shared fc_hidden) instead of
the ``Linear(2H, H)`` + repeat used by other MTP variants,
- emits TWO hidden streams per step (scheme A): a single stream [T, H]
(final-mixer collapsed, fed to the LM head) and a pre-final-mixer
multi stream [T, hc_count*H] (fed to the next draft step).
"""
from collections.abc import Iterable
import regex as re
import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
from vllm import envs
from vllm.config import VllmConfig, replace, set_current_vllm_config
from vllm.distributed import get_pp_group
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.utils import configure_quant_config
from vllm.model_executor.models.interfaces import SupportsHCSP, SupportsPP
from vllm.model_executor.models.qwen3_5 import Qwen3_5Model
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
get_draft_quant_config,
make_empty_intermediate_tensors_factory,
maybe_fuse_shared_experts,
maybe_prefix,
)
from vllm.models.common.ops.sequence_parallel import (
sp_all_gather,
sp_padding_mask,
sp_shard,
)
from vllm.sequence import IntermediateTensors
from ..common.hyperconnection import is_hc_sequence_parallel_enabled
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
from .model import (
_EXTRA_WEIGHTS_MAPPER,
_QWEN4_EXP_IGNORED_MISSING_SUFFIXES,
Qwen4ExpDecoderLayer,
Qwen4ExpMixtureOfExperts,
Qwen4ExpSparseMoeBlock,
)
def _remap_ignored_layers(
ignored_layers: list[str],
mtp_start_layer_idx: int,
) -> list[str]:
remapped: list[str] = []
for name in ignored_layers:
if name.startswith("mtp."):
new_name = re.sub(
r"(?<=\.layers\.)\d+",
lambda m: str(mtp_start_layer_idx + int(m.group(0))),
name,
)
remapped.append(new_name)
else:
remapped.append(name)
return remapped
def _remap_quantized_layers(
quantized_layers: dict[str, dict],
mtp_start_layer_idx: int,
) -> dict[str, dict]:
"""Map checkpoint MTP layer indices to standalone draft indices."""
return {
_remap_ignored_layers([name], mtp_start_layer_idx)[0]: layer_info
for name, layer_info in quantized_layers.items()
}
def _remap_mtp_weight_name(name: str) -> str | None:
"""Map Qwen4Exp checkpoint paths into the standalone draft model."""
for checkpoint_prefix in (
"model.language_model.",
"language_model.",
):
if name.startswith(checkpoint_prefix):
name = name.removeprefix(checkpoint_prefix)
break
if name.startswith("embed_tokens."):
name = f"model.{name}"
if name.startswith("model.mtp."):
name = name.removeprefix("model.")
if name.startswith("mtp.shared_head.head."):
return name.replace("mtp.shared_head.head.", "lm_head.", 1)
if name.startswith("model.shared_head.head."):
return name.replace("model.shared_head.head.", "lm_head.", 1)
if name.startswith("shared_head.head."):
return name.replace("shared_head.head.", "lm_head.", 1)
if name.startswith("model.lm_head."):
return name.removeprefix("model.")
if name.startswith("mtp."):
return name.replace("mtp.", "model.", 1)
if name.startswith("model.embed_tokens.") or name.startswith("lm_head."):
return name
return None
def _make_draft_vllm_config(
vllm_config: VllmConfig,
mtp_start_layer_idx: int,
) -> VllmConfig:
"""Ensure that the draft model config is set in the vLLM config."""
speculative_config = vllm_config.speculative_config
if speculative_config is None or speculative_config.draft_model_config is None:
raise ValueError("speculative_config.draft_model_config must be set")
draft_quant_config = get_draft_quant_config(vllm_config)
# inject packed and ignored modules to the quantization config of draft model
if draft_quant_config is not None:
configure_quant_config(draft_quant_config, Qwen4ExpMTP)
ignored_layers = getattr(draft_quant_config, "ignored_layers", None)
if ignored_layers:
setattr( # noqa: B010
draft_quant_config,
"ignored_layers",
_remap_ignored_layers(ignored_layers, mtp_start_layer_idx),
)
exclude_modules = getattr(draft_quant_config, "exclude_modules", None)
if exclude_modules:
setattr( # noqa: B010
draft_quant_config,
"exclude_modules",
_remap_ignored_layers(exclude_modules, mtp_start_layer_idx),
)
quantized_layers = getattr(draft_quant_config, "quantized_layers", None)
if quantized_layers:
setattr( # noqa: B010
draft_quant_config,
"quantized_layers",
_remap_quantized_layers(quantized_layers, mtp_start_layer_idx),
)
draft_vllm_config = replace(
vllm_config,
model_config=speculative_config.draft_model_config,
)
# VllmConfig post-init derives the target quant config, so restore the
# independently resolved draft quant config after replacement.
draft_vllm_config.quant_config = draft_quant_config
return draft_vllm_config
class Qwen4ExpMultiTokenPredictor(nn.Module):
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper | _EXTRA_WEIGHTS_MAPPER
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
model_config = vllm_config.model_config
config: Qwen4ExpTextConfig = model_config.hf_text_config
self.config = config
# MoE SP also enables HC SP, while HC SP can be enabled independently.
self.use_hc_sequence_parallel = is_hc_sequence_parallel_enabled(vllm_config)
self.use_sequence_parallel = (
vllm_config.parallel_config.use_sequence_parallel_moe
)
self.vocab_size = config.vocab_size
self.mtp_start_layer_idx = config.num_hidden_layers
self.num_mtp_layers = getattr(config, "mtp_num_hidden_layers", 1)
self.hidden_size = config.hidden_size
self.hc_count = config.hc_count
self.embed_tokens = VocabParallelEmbedding(self.vocab_size, self.hidden_size)
draft_vllm_config = _make_draft_vllm_config(
vllm_config,
self.mtp_start_layer_idx,
)
with set_current_vllm_config(draft_vllm_config, prefix=prefix):
# residual_linear_shared fusion: fc_embedding projects the token
# embedding, fc_hidden (shared across HC branches) projects the
# backbone hidden; the embedding is added as a residual to every
# branch (see mtp_residual_linear_shared.md).
self.fc_embedding = ColumnParallelLinear(
self.hidden_size,
self.hidden_size,
gather_output=True,
bias=False,
return_bias=False,
quant_config=draft_vllm_config.quant_config,
prefix=f"{prefix}.fc_embedding",
)
self.fc_hidden = ColumnParallelLinear(
self.hidden_size,
self.hidden_size,
gather_output=True,
bias=False,
return_bias=False,
quant_config=draft_vllm_config.quant_config,
prefix=f"{prefix}.fc_hidden",
)
self.layers = nn.ModuleList(
Qwen4ExpDecoderLayer(
draft_vllm_config,
layer_type="qwen_sparse_attention",
prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}",
)
for idx in range(self.num_mtp_layers)
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
Qwen4ExpSparseMoeBlock,
"mlp",
)
self.pre_fc_norm_embedding = GemmaRMSNorm(
self.hidden_size, eps=config.rms_norm_eps
)
self.pre_fc_norm_hidden = GemmaRMSNorm(
self.hidden_size * self.hc_count, eps=config.rms_norm_eps
)
# HC final mixer collapses the multi stream into [T, H] for the LM head.
hc_config = HyperConnectionConfig(
hc_count=config.hc_count,
hidden_size=config.hidden_size,
params_dtype=torch.bfloat16,
hc_lowrank=config.hc_lowrank,
rms_norm_eps=config.rms_norm_eps,
hc_per_branch_norm=True,
)
self.hyper_connection_mixer = GatedResidual(
hc_config,
use_combine=False,
prefix=maybe_prefix(prefix, "hyper_connection_mixer"),
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], self.hidden_size * self.hc_count
)
def _iter_qsa_attentions(self):
"""Yield MTP attention modules that own a QSA indexer."""
for layer in self.layers:
attention = getattr(layer, "self_attn", None)
if (
attention is not None
and getattr(attention, "indexer", None) is not None
):
yield attention
def set_skip_topk(self, skip: bool) -> None:
"""Select on MTP step 0 and reuse its QSA indices on later steps."""
for attention in self._iter_qsa_attentions():
attention.indexer.skip_topk = skip
def compact_topk_indices(self, row_indices: torch.Tensor) -> None:
"""Keep each request's target-aligned step-0 sparse-index row."""
num_rows = row_indices.numel()
for attention in self._iter_qsa_attentions():
buffer = attention.topk_indices_buffer
selected = buffer.index_select(0, row_indices)
buffer[:num_rows].copy_(selected)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor | None = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors:
hc_count = self.hc_count
hidden_size = self.hidden_size
prev_block_output: torch.Tensor | None = None
if get_pp_group().is_first_rank:
assert hidden_states is not None
if inputs_embeds is None:
assert input_ids is not None
inputs_embeds = self.embed_input_ids(input_ids)
# Embedding branch: pre-norm -> fc_embedding -> [T, H].
inputs_embeds = self.pre_fc_norm_embedding(inputs_embeds)
inputs_embeds = self.fc_embedding(inputs_embeds)
# Backbone hidden is multi-stream [T, hc_count*H] (scheme A:
# the main model truly emits the pre-final-mixer multi stream
# on the first step; subsequent steps reuse the prior draft
# step's multi stream).
num_tokens = hidden_states.shape[0]
hidden_states = hidden_states.view(num_tokens, hc_count, hidden_size)
hidden_states = self.pre_fc_norm_hidden(hidden_states.flatten(-2)).view(
num_tokens, hc_count, hidden_size
)
hidden_states = self.fc_hidden(hidden_states)
hidden_states = hidden_states.flatten(-2)
prev_block_output = inputs_embeds
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
current_step_idx = spec_step_idx % self.num_mtp_layers
full_num_tokens = positions.shape[-1]
if self.use_hc_sequence_parallel:
# SP requires PP=1, so the first-rank branch sets prev_block_output.
assert prev_block_output is not None
if (
self.use_sequence_parallel
and envs.VLLM_MOE_SKIP_PADDING
and is_forward_context_available()
):
forward_context = get_forward_context()
forward_context.is_padding = sp_padding_mask(
forward_context.is_padding, hidden_states
)
hidden_states = sp_shard(hidden_states)
# The FC output is already complete and needs only a slice.
prev_block_output = sp_shard(prev_block_output)
layer = self.layers[current_step_idx]
hidden_states, block_output, injection = layer(
hidden_states=hidden_states,
prev_block_output=prev_block_output,
prev_injection=None,
positions=positions,
input_ids=None,
query_start_loc=None,
ngram_context=None,
)
if not get_pp_group().is_last_rank:
# As in the target model, PP carries a materialized tensor rather
# than the delayed hidden/output/injection tuple.
hidden_states = layer.mlp_hyper_connection.combine(
hidden_states, block_output, injection
)
return IntermediateTensors({"hidden_states": hidden_states})
# Last PP rank finalize. Keep both:
# (A) sample_hidden_states [T, H] -> single stream for the LM head
# (B) multi_hidden [T, hc_count*H] -> pre-final-mixer multi stream
# for the next draft step (zero extra compute, just kept).
multi_hidden, sample_hidden_states, _ = (
self.hyper_connection_mixer.combine_and_mix(
hidden_states, block_output, injection
)
)
if self.use_hc_sequence_parallel:
# Gather both outputs together, then restore their contiguous layouts.
packed_hidden_states = torch.cat(
[sample_hidden_states, multi_hidden], dim=-1
)
packed_hidden_states = sp_all_gather(packed_hidden_states)[:full_num_tokens]
sample_hidden_states, multi_hidden = packed_hidden_states.split(
[hidden_size, hc_count * hidden_size], dim=-1
)
sample_hidden_states = sample_hidden_states.contiguous()
multi_hidden = multi_hidden.contiguous()
return sample_hidden_states, multi_hidden
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = maybe_fuse_shared_experts(
weights,
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=getattr(self.config, "num_experts", 0) or 0,
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
)
mapper = self.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None}
)
loader = AutoWeightsLoader(
self,
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
)
return loader.load_weights(weights, mapper=mapper)
class Qwen4ExpMTP(nn.Module, SupportsHCSP, SupportsPP, Qwen4ExpMixtureOfExperts):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
"in_proj_qkvz": ["in_proj_qkv", "in_proj_z"],
"in_proj_ba": ["in_proj_b", "in_proj_a"],
"input_mix_weight_down_block_inject": [
"input_mix_weight_down",
"block_inject_weight",
"_input_mix_padding",
],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
self.quant_config = vllm_config.quant_config
super().__init__()
self.config = config
self.model = Qwen4ExpMultiTokenPredictor(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "mtp"),
)
if get_pp_group().is_last_rank:
if config.tie_word_embeddings:
self.lm_head = self.model.embed_tokens
else:
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "lm_head"),
)
else:
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
self.set_moe_parameters(self.model.layers)
enable_qwen4_exp_low_latency_gemm(self, vllm_config.model_config.dtype)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor | None = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor] | IntermediateTensors:
return self.model(
input_ids,
positions,
hidden_states,
intermediate_tensors,
inputs_embeds,
spec_step_idx=spec_step_idx,
)
def compute_logits(
self, hidden_states: torch.Tensor, spec_step_idx: int = 0
) -> torch.Tensor | None:
return self.logits_processor(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
def remap_weight_names():
for name, weight in weights:
remapped_name = _remap_mtp_weight_name(name)
if remapped_name is not None:
yield remapped_name, weight
mapper = WeightsMapper(
orig_to_new_substr={"hyper_connection_mixer.block_inject_weight": None}
)
loader = AutoWeightsLoader(
self,
ignore_unexpected_suffixes=_QWEN4_EXP_IGNORED_MISSING_SUFFIXES.copy(),
)
return loader.load_weights(remap_weight_names(), mapper=mapper)
__all__ = ["Qwen4ExpMTP", "Qwen4ExpMultiTokenPredictor"]