With VLLM_ABLITERATION set, boot crashed: dynamo traces abliteration.register from Qwen3NextGatedDeltaNet.__init__, and buf.copy_(r) fails on the meta device during the compile-time model construction pass. Disable tracing for register and guard the copy so an unmaterializable buffer turns projection off with a warning. Log activation at registration instead of inside project_: project_ runs in the compiled region, and a logger call there aborts dynamo. The width guard uses the same escape hatch. Also project the embedding that Qwen3NextMultiTokenPredictor builds for itself, which the first pass missed, and add tests for the 2560-value Flash-Next direction against a 5120 model and for a half-width activation. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
248 lines
8.6 KiB
Python
248 lines
8.6 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Inference-only Qwen3Next MTP model."""
|
|
|
|
from collections.abc import Iterable
|
|
|
|
import torch
|
|
from torch import nn
|
|
from transformers import Qwen3NextConfig
|
|
|
|
from vllm.compilation.decorators import support_torch_compile
|
|
from vllm.config import VllmConfig
|
|
from vllm.distributed import get_pp_group, tensor_model_parallel_all_gather
|
|
from vllm.logger import init_logger
|
|
from vllm.model_executor.layers.fused_moe.utils import (
|
|
is_model_fused_shared_expert_compatible,
|
|
)
|
|
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
|
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
|
ParallelLMHead,
|
|
VocabParallelEmbedding,
|
|
)
|
|
from vllm.model_executor.models import abliteration
|
|
from vllm.model_executor.models.qwen3_next import (
|
|
Qwen3NextDecoderLayer,
|
|
Qwen3NextModel,
|
|
Qwen3NextRMSNorm,
|
|
Qwen3NextSparseMoeBlock,
|
|
QwenNextMixtureOfExperts,
|
|
)
|
|
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
|
from vllm.sequence import IntermediateTensors
|
|
|
|
from .utils import (
|
|
AutoWeightsLoader,
|
|
make_empty_intermediate_tensors_factory,
|
|
maybe_fuse_shared_experts,
|
|
maybe_prefix,
|
|
)
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
KVCache = tuple[torch.Tensor, torch.Tensor]
|
|
|
|
|
|
@support_torch_compile
|
|
class Qwen3NextMultiTokenPredictor(nn.Module):
|
|
hf_to_vllm_mapper = Qwen3NextModel.hf_to_vllm_mapper
|
|
|
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
|
super().__init__()
|
|
|
|
model_config = vllm_config.model_config
|
|
quant_config = vllm_config.quant_config
|
|
|
|
config: Qwen3NextConfig = model_config.hf_config
|
|
|
|
self.config = config
|
|
|
|
self.vocab_size = config.vocab_size
|
|
|
|
self.mtp_start_layer_idx = config.num_hidden_layers
|
|
self.num_mtp_layers = getattr(config, "num_nextn_predict_layers", 1)
|
|
|
|
self.embed_tokens = VocabParallelEmbedding(
|
|
self.vocab_size,
|
|
config.hidden_size,
|
|
)
|
|
abliteration.register(self, config.hidden_size, model_config.dtype)
|
|
|
|
# Workaround: mtp.fc is stored as BF16 in NVFP4 checkpoints but is
|
|
# missing from the checkpoint quant exclude list (its `ignore` glob
|
|
# does not cover `mtp.fc`). Force unquantized to match the weights,
|
|
# mirroring the Qwen3.5 MTP handling (PR #38832).
|
|
fc_quant = (
|
|
None
|
|
if (quant_config and quant_config.get_name() == "modelopt_fp4")
|
|
else quant_config
|
|
)
|
|
self.fc = ColumnParallelLinear(
|
|
self.config.hidden_size * 2,
|
|
self.config.hidden_size,
|
|
gather_output=True,
|
|
bias=False,
|
|
return_bias=False,
|
|
quant_config=fc_quant,
|
|
prefix=f"{prefix}.fc",
|
|
)
|
|
|
|
self.layers = torch.nn.ModuleList(
|
|
Qwen3NextDecoderLayer(
|
|
vllm_config,
|
|
layer_type="full_attention",
|
|
prefix=f"{prefix}.layers.{idx}",
|
|
)
|
|
for idx in range(self.num_mtp_layers)
|
|
)
|
|
|
|
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
|
self.layers,
|
|
Qwen3NextSparseMoeBlock,
|
|
"mlp",
|
|
)
|
|
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
|
["hidden_states", "residual"], config.hidden_size
|
|
)
|
|
|
|
self.norm = Qwen3NextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
|
self.pre_fc_norm_hidden = Qwen3NextRMSNorm(
|
|
config.hidden_size, eps=config.rms_norm_eps
|
|
)
|
|
self.pre_fc_norm_embedding = Qwen3NextRMSNorm(
|
|
config.hidden_size, eps=config.rms_norm_eps
|
|
)
|
|
|
|
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
|
return abliteration.project_(self, self.embed_tokens(input_ids))
|
|
|
|
def forward(
|
|
self,
|
|
input_ids: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
hidden_states: torch.Tensor,
|
|
intermediate_tensors: IntermediateTensors | None = None,
|
|
inputs_embeds: torch.Tensor | None = None,
|
|
spec_step_idx: int = 0,
|
|
) -> torch.Tensor:
|
|
if get_pp_group().is_first_rank:
|
|
if inputs_embeds is None:
|
|
inputs_embeds = self.embed_input_ids(input_ids)
|
|
assert hidden_states.shape[-1] == inputs_embeds.shape[-1]
|
|
inputs_embeds = self.pre_fc_norm_embedding(inputs_embeds)
|
|
hidden_states = self.pre_fc_norm_hidden(hidden_states)
|
|
hidden_states = torch.cat([inputs_embeds, hidden_states], dim=-1)
|
|
hidden_states = self.fc(hidden_states)
|
|
residual = None
|
|
else:
|
|
assert intermediate_tensors is not None
|
|
hidden_states = intermediate_tensors["hidden_states"]
|
|
residual = intermediate_tensors["residual"]
|
|
|
|
current_step_idx = spec_step_idx % self.num_mtp_layers
|
|
mtp_layer = self.layers[current_step_idx]
|
|
if mtp_layer.use_attn_reduce_scatter_for_moe:
|
|
assert hidden_states.shape[0] == positions.shape[-1]
|
|
hidden_states = sequence_parallel_chunk(hidden_states)
|
|
assert residual is None
|
|
hidden_states, residual = mtp_layer(
|
|
positions=positions,
|
|
hidden_states=hidden_states,
|
|
residual=residual,
|
|
)
|
|
|
|
if not get_pp_group().is_last_rank:
|
|
return IntermediateTensors(
|
|
{"hidden_states": hidden_states, "residual": residual}
|
|
)
|
|
|
|
hidden_states, _ = self.norm(hidden_states, residual)
|
|
if mtp_layer.use_attn_reduce_scatter_for_moe:
|
|
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
|
|
hidden_states = hidden_states[: positions.shape[-1]]
|
|
return hidden_states
|
|
|
|
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
|
weights = maybe_fuse_shared_experts(
|
|
weights,
|
|
enabled=self.is_fused_shared_expert_enabled,
|
|
n_routed_experts=self.config.num_experts,
|
|
n_shared_experts=1,
|
|
ckpt_prefix="mlp.shared_expert",
|
|
)
|
|
loader = AutoWeightsLoader(self)
|
|
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
|
|
|
|
|
@support_torch_compile
|
|
class Qwen3NextMTP(nn.Module, QwenNextMixtureOfExperts):
|
|
packed_modules_mapping = {
|
|
"qkv_proj": [
|
|
"q_proj",
|
|
"k_proj",
|
|
"v_proj",
|
|
],
|
|
"gate_up_proj": ["gate_proj", "up_proj"],
|
|
}
|
|
|
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
|
config = vllm_config.model_config.hf_config
|
|
self.vllm_config = vllm_config
|
|
self.quant_config = vllm_config.quant_config
|
|
|
|
super().__init__()
|
|
self.config = config
|
|
self.model = Qwen3NextMultiTokenPredictor(
|
|
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "mtp")
|
|
)
|
|
|
|
self.lm_head = ParallelLMHead(
|
|
config.vocab_size,
|
|
config.hidden_size,
|
|
prefix=maybe_prefix(prefix, "lm_head"),
|
|
)
|
|
self.logits_processor = LogitsProcessor(config.vocab_size)
|
|
self.set_moe_parameters()
|
|
|
|
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
|
return self.model.embed_input_ids(input_ids)
|
|
|
|
def forward(
|
|
self,
|
|
input_ids: torch.Tensor | None,
|
|
positions: torch.Tensor,
|
|
hidden_states: torch.Tensor,
|
|
intermediate_tensors: IntermediateTensors | None = None,
|
|
inputs_embeds: torch.Tensor | None = None,
|
|
**kwargs: object,
|
|
):
|
|
hidden_states = self.model(
|
|
input_ids, positions, hidden_states, intermediate_tensors, inputs_embeds
|
|
)
|
|
return hidden_states
|
|
|
|
def compute_logits(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
spec_step_idx: int = 0,
|
|
) -> torch.Tensor | None:
|
|
return self.logits_processor(self.lm_head, hidden_states)
|
|
|
|
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
|
return not name.startswith("mtp.") and not any(
|
|
key in name for key in ["embed_tokens", "lm_head"]
|
|
)
|
|
|
|
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
|
def remap_weight_names(weights):
|
|
for name, weight in weights:
|
|
if self.is_unused_checkpoint_weight(name):
|
|
continue
|
|
if name.startswith("mtp."):
|
|
name = name.replace("mtp.", "model.")
|
|
yield name, weight
|
|
|
|
loader = AutoWeightsLoader(self)
|
|
return loader.load_weights(remap_weight_names(weights))
|