Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 21:59:03 +07:00
1012 changed files with 61873 additions and 14499 deletions
-365
View File
@@ -1,365 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2023-2025 vLLM Team
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Inference-only Arcee (AFM) model – adds support for ReLU^2 feed-forward
# activation.
from collections.abc import Iterable
from itertools import islice
from typing import Any
import torch
from torch import nn
from transformers import LlamaConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.distributed import get_pp_group
from vllm.model_executor.layers.activation import ReLUSquaredActivation
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.sequence import IntermediateTensors
from .interfaces import (
EagleModelMixin,
SupportsEagle,
SupportsEagle3,
SupportsLoRA,
SupportsPP,
)
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
)
class ArceeMLP(nn.Module):
"""Feed-forward layer for Arcee using ReLU^2 activation
(no gating as in LLaMA)."""
def __init__(
self,
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Any | None = None,
bias: bool = False,
prefix: str = "",
reduce_results: bool = True,
) -> None:
super().__init__()
# Single linear projection up to intermediate size
# (no separate gate projection)
self.up_proj = ColumnParallelLinear(
input_size=hidden_size,
output_size=intermediate_size,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.up_proj",
)
# Down projection back to hidden size
self.down_proj = RowParallelLinear(
input_size=intermediate_size,
output_size=hidden_size,
bias=bias,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=f"{prefix}.down_proj",
)
if hidden_act != "relu2":
raise ValueError(
f"Unsupported activation: {hidden_act}. "
"Only 'relu2' is supported for AFM."
)
# Define ReLU^2 activation: (ReLU(x))^2 elementwise
self.act_fn = ReLUSquaredActivation()
def forward(self, x: torch.Tensor) -> torch.Tensor:
x, _ = self.up_proj(x) # Project to intermediate size
x = self.act_fn(x) # Apply ReLU^2 activation elementwise
x, _ = self.down_proj(x) # Project back down to hidden size
return x
class ArceeDecoderLayer(nn.Module):
"""Transformer decoder block for Arcee, with self-attention and
ReLU^2 MLP."""
def __init__(
self,
config: LlamaConfig,
cache_config: Any | None = None,
quant_config: Any | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.hidden_size = config.hidden_size
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
# Determine if attention bias is needed (some variants use bias terms)
attention_bias = getattr(config, "attention_bias", False) or getattr(
config, "bias", False
)
bias_o_proj = attention_bias
if hasattr(config, "qkv_bias"):
attention_bias = config.qkv_bias
# Self-Attention (using LLaMA's attention structure)
from vllm.model_executor.models.llama import (
LlamaAttention, # import here to avoid circular import
)
self.self_attn = LlamaAttention(
config=config,
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
num_kv_heads=getattr(
config, "num_key_value_heads", config.num_attention_heads
),
max_position_embeddings=max_position_embeddings,
quant_config=quant_config,
bias=attention_bias,
bias_o_proj=bias_o_proj,
cache_config=cache_config,
prefix=f"{prefix}.self_attn",
attn_type=getattr(
config, "attn_type", "decoder"
), # assume decoder (causal) unless specified
)
# MLP with ReLU^2 activation
self.mlp = ArceeMLP(
hidden_size=self.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
bias=getattr(config, "mlp_bias", False),
prefix=f"{prefix}.mlp",
)
# Layer normalization layers (RMSNorm as in LLaMA)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self-Attention block
if residual is None:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
else:
# Fused residual add + layernorm if supported
hidden_states, residual = self.input_layernorm(hidden_states, residual)
hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states)
# Feed-forward block
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
@support_torch_compile
class ArceeModel(nn.Module, EagleModelMixin):
"""The transformer model backbone for Arcee (embedding layer + stacked
decoder blocks + final norm)."""
def __init__(
self,
*,
vllm_config,
prefix: str = "",
layer_type: type[nn.Module] = ArceeDecoderLayer,
) -> None:
super().__init__()
config: LlamaConfig = vllm_config.model_config.hf_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.quant_config = quant_config
self.config = config
self.vocab_size = config.vocab_size
# Word embeddings (parallelized if using pipeline parallel)
if get_pp_group().is_first_rank or (
config.tie_word_embeddings and get_pp_group().is_last_rank
):
self.embed_tokens = VocabParallelEmbedding(
self.vocab_size,
config.hidden_size,
quant_config=quant_config,
)
else:
self.embed_tokens = PPMissingLayer() # placeholder on non-embedding ranks
# Build decoder layers across pipeline ranks
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: layer_type(
config=config,
cache_config=cache_config,
quant_config=quant_config,
prefix=prefix,
),
prefix=f"{prefix}.layers",
)
# Final RMSNorm on the last pipeline stage
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
self.norm = PPMissingLayer()
# Prepare factory for empty intermediate tensors
# (for pipeline scheduling)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
# Embedding lookup (on first pipeline rank)
if get_pp_group().is_first_rank:
hidden_states = (
inputs_embeds
if inputs_embeds is not None
else self.embed_input_ids(input_ids)
)
residual = None
else:
assert intermediate_tensors is not None, (
"IntermediateTensors must be provided for non-first pipeline ranks"
)
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer)
):
hidden_states, residual = layer(positions, hidden_states, residual)
self._maybe_add_hidden_state(
aux_hidden_states, idx + 1, hidden_states, residual
)
if not get_pp_group().is_last_rank:
# Send intermediate results to the next pipeline stage
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
)
# On last rank: apply final layer norm
hidden_states, _ = self.norm(hidden_states, residual)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
class ArceeForCausalLM(
nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
):
"""Arcee Model for causal language modeling, integrated with vLLM
runtime."""
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
},
orig_to_new_substr={
"gate_proj": None,
},
)
# Map fused module names to their submodule components
# (for quantization and LoRA)
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
}
def __init__(self, *, vllm_config, prefix: str = "") -> None:
super().__init__()
config = vllm_config.model_config.hf_config
self.config = config
# Initialize the inner Transformer model (ArceeModel)
self.model = ArceeModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"),
)
# On the last pipeline stage, set up the LM head and logits processor
if get_pp_group().is_last_rank:
# Determine vocabulary size (including any LoRA extra tokens
# for padded LM head)
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=vllm_config.quant_config,
bias=getattr(config, "lm_head_bias", False),
prefix=maybe_prefix(prefix, "lm_head"),
)
if config.tie_word_embeddings:
# Tie output weights with input embedding matrix
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
logit_scale = getattr(config, "logit_scale", 1.0)
self.logits_processor = LogitsProcessor(
config.vocab_size, scale=logit_scale
)
else:
# Placeholder for lm_head on non-last ranks
self.lm_head = PPMissingLayer()
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
model_output = self.model(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
)
return model_output
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
# Compute final logits from hidden states (last pipeline rank only)
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights into the model (delegates to inner model and handles
tied embeddings)."""
loader = AutoWeightsLoader(self)
# AutoWeightLoader handles weight name remapping, including fusing
# separate q_proj, k_proj, v_proj into qkv_proj
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+9 -4
View File
@@ -461,6 +461,7 @@ class BertEmbeddingModel(nn.Module, SupportsQuant):
"""
is_pooling_model = True
embedding_class: type[nn.Module] = BertEmbedding
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -504,7 +505,7 @@ class BertEmbeddingModel(nn.Module, SupportsQuant):
def _build_model(self, vllm_config: VllmConfig, prefix: str = "") -> BertModel:
return BertModel(
vllm_config=vllm_config, prefix=prefix, embedding_class=BertEmbedding
vllm_config=vllm_config, prefix=prefix, embedding_class=self.embedding_class
)
def _build_pooler(self, pooler_config: PoolerConfig) -> Pooler:
@@ -781,6 +782,7 @@ class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQu
"""
is_pooling_model = True
embedding_class: type[nn.Module] = BertEmbedding
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -790,7 +792,7 @@ class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQu
self.bert = BertPoolingModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "bert"),
embedding_class=BertEmbedding,
embedding_class=self.embedding_class,
)
self.classifier = nn.Linear(
config.hidden_size,
@@ -840,6 +842,7 @@ class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQu
@default_pooling_type(tok_pooling_type="ALL")
class BertForTokenClassification(nn.Module):
is_pooling_model = True
embedding_class: type[nn.Module] = BertEmbedding
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -849,7 +852,7 @@ class BertForTokenClassification(nn.Module):
self.bert = BertModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "bert"),
embedding_class=BertEmbedding,
embedding_class=self.embedding_class,
)
self.classifier = nn.Linear(
config.hidden_size, config.num_labels, dtype=self.head_dtype
@@ -927,13 +930,15 @@ class BertForMaskedLM(nn.Module):
}
)
embedding_class: type[nn.Module] = BertEmbedding
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
self.bert = BertModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "bert"),
embedding_class=BertEmbedding,
embedding_class=self.embedding_class,
)
self.mlm_head = BertMLMHead(
hidden_size=config.hidden_size,
@@ -572,7 +572,6 @@ class GteNewModel(BertWithRope):
super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs)
# GteNewModel only gate_up_proj does not have bias.
# Hack method learned from vllm/model_executor/models/glm.py
for layer in self.encoder.layers:
layer.mlp.gate_up_proj.bias = None
layer.mlp.gate_up_proj.skip_bias_add = True
+3 -14
View File
@@ -646,20 +646,9 @@ class MambaModelConfig(VerifyAndUpdateConfig):
cache_config.mamba_cache_mode,
model_config.architecture,
)
if (
cache_config.mamba_cache_mode == "all"
and not model_config.supports_mamba_prefix_caching
):
cache_config.mamba_cache_mode = "align"
logger.warning(
"Hybrid or mamba-based model detected without support "
"for prefix caching with Mamba cache 'all' mode: "
"falling back to 'align' mode."
)
if cache_config.mamba_cache_mode == "align":
assert vllm_config.scheduler_config.enable_chunked_prefill, (
"Chunked prefill is required for mamba cache mode 'align'."
)
assert vllm_config.scheduler_config.enable_chunked_prefill, (
"Chunked prefill is required for mamba cache mode 'align'."
)
# By default, mamba block size will be set to max_model_len (see
# below). When enabling prefix caching, we align mamba block size
# to the block size as the basic granularity for prefix caching.
+2 -1
View File
@@ -658,6 +658,7 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase):
return MLAAttentionSpec(
block_size=self.cache_config.block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=self.head_dim,
dtype=self.dtype,
cache_role=SparseCacheRole.INDEXER,
@@ -715,7 +716,7 @@ class Indexer(nn.Module):
disable_tp=True,
prefix=f"{prefix}.wk_weights_proj",
)
self.k_norm = LayerNorm(self.head_dim, eps=1e-6)
self.k_norm = LayerNorm(self.head_dim, eps=1e-6, dtype=torch.float32)
self.softmax_scale = self.head_dim**-0.5
self.scale_fmt = "ue8m0"
@@ -909,9 +909,12 @@ class DiffusionGemmaModelState(ModelState):
self._req_id_to_index: dict[str, int] = {}
# Persistent buffer for per-request causal flags, updated in-place
# so FULL CUDA graph replay sees the latest values.
# so FULL CUDA graph replay sees the latest values. Must be int32:
# FlashAttentionMetadataBuilder.build() rejects other dtypes, since
# an out-of-place cast there would detach the captured graph from
# this buffer and freeze replay at the capture-time snapshot.
self._causal_buf = torch.zeros(
self.max_num_reqs, dtype=torch.bool, device=device
self.max_num_reqs, dtype=torch.int32, device=device
)
# Persistent inputs_embeds buffer — required so FULL CUDA graph
@@ -1116,11 +1119,11 @@ class DiffusionGemmaModelState(ModelState):
# Invariant: the sampler flips is_encoder_phase to False only after a
# request's FINAL prompt chunk, so a prompt spanning multiple chunks
# (longer than the token budget) stays causal for every chunk.
self._causal_buf[:actual_num_reqs] = self.diffusion_states.is_encoder_phase[
slots
]
self._causal_buf[:actual_num_reqs].copy_(
self.diffusion_states.is_encoder_phase[slots]
)
if actual_num_reqs < num_reqs:
self._causal_buf[actual_num_reqs:num_reqs] = False
self._causal_buf[actual_num_reqs:num_reqs] = 0
causal: bool | torch.Tensor = self._causal_buf[:num_reqs]
return build_attn_metadata(
-24
View File
@@ -1,24 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only HF format GLM-4 model compatible with THUDM weights."""
from vllm.config import VllmConfig
from vllm.model_executor.models.llama import LlamaForCausalLM
from .utils import PPMissingLayer
class GlmForCausalLM(LlamaForCausalLM):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
hf_config = vllm_config.model_config.hf_config
hf_config.rope_parameters["partial_rotary_factor"] = 0.5
super().__init__(vllm_config=vllm_config, prefix=prefix)
# Hack Llama model to fit HF format GLM implementation
# Attention difference between GLM and Llama:
# 1. Half partial rotary_dim and no Neox style.
# 2. There is no bias for o_proj in attention
for layer in self.model.layers:
if not isinstance(layer, PPMissingLayer):
layer.self_attn.rotary_emb.is_neox_style = False
layer.self_attn.o_proj.bias = None
layer.self_attn.o_proj.skip_bias_add = True
+16 -7
View File
@@ -61,6 +61,10 @@ from vllm.model_executor.layers.attention import (
MMEncoderAttention,
)
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
from vllm.model_executor.layers.fusion.mm_input_norm import (
IdentityInputNorm,
build_mm_input_norm,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -616,6 +620,7 @@ class Glm4vVisionTransformer(nn.Module):
vision_config: Glm4vVisionConfig,
norm_eps: float = 1e-6,
quant_config: QuantizationConfig | None = None,
input_norm: nn.Module | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -642,6 +647,7 @@ class Glm4vVisionTransformer(nn.Module):
in_channels=in_channels,
hidden_size=self.hidden_size,
)
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
norm_layer = partial(RMSNorm, eps=norm_eps)
head_dim = self.hidden_size // self.num_heads
@@ -924,7 +930,7 @@ class Glm4vVisionTransformer(nn.Module):
encoder_metadata = self.prepare_encoder_metadata(grid_thw)
# patchify
x = x.to(device=self.device, dtype=self.dtype)
x = self.input_norm(x.to(device=self.device), self.dtype)
x = self.patch_embed(x)
x = self.post_conv_layernorm(x)
@@ -1755,6 +1761,7 @@ class Glm4vForConditionalGeneration(
supports_encoder_tp_data = True
supports_tower_connector_lora = True
supports_mm_device_do_normalize = True
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
@@ -1786,6 +1793,7 @@ class Glm4vForConditionalGeneration(
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-5),
quant_config=quant_config,
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
@@ -1867,7 +1875,7 @@ class Glm4vForConditionalGeneration(
if image_input["type"] == "image_embeds":
image_embeds = image_input["image_embeds"].type(self.visual.dtype)
else:
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
pixel_values = image_input["pixel_values"]
if self.use_data_parallel:
return run_dp_sharded_mrope_vision_model(
self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d"
@@ -1888,9 +1896,7 @@ class Glm4vForConditionalGeneration(
if video_input["type"] == "video_embeds":
video_embeds = video_input["video_embeds"].type(self.visual.dtype)
else:
pixel_values_videos = video_input["pixel_values_videos"].type(
self.visual.dtype
)
pixel_values_videos = video_input["pixel_values_videos"]
if self.use_data_parallel:
return run_dp_sharded_mrope_vision_model(
self.visual,
@@ -2098,8 +2104,11 @@ class Glm4vForConditionalGeneration(
flattened_patch_size = (
in_channels * temporal_patch_size * patch_size * patch_size
)
dummy_pixel_values = torch.randn(
total_patches, flattened_patch_size, device=device, dtype=dtype
dummy_pixel_values = torch.zeros(
total_patches,
flattened_patch_size,
device=device,
dtype=self.visual.input_norm.input_dtype or dtype,
)
# Override max_seqlen with a safe upper bound for capture.
+12 -2
View File
@@ -48,6 +48,7 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
from vllm.model_executor.layers.conv import Conv2dLayer
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
@@ -254,9 +255,17 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
vision_config: "GlmOcrVisionConfig",
norm_eps: float = 1e-5,
quant_config: QuantizationConfig | None = None,
input_norm: nn.Module | None = None,
prefix: str = "",
) -> None:
super().__init__(text_config, vision_config, norm_eps, quant_config, prefix)
super().__init__(
text_config,
vision_config,
norm_eps,
quant_config,
input_norm,
prefix,
)
del self.post_conv_layernorm
del self.embeddings
@@ -334,7 +343,7 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
grid_thw = torch.tensor(grid_thw, dtype=torch.int32)
# patchify
x = x.to(device=self.device, dtype=self.dtype)
x = self.input_norm(x.to(device=self.device), self.dtype)
x = self.patch_embed(x)
# compute position embedding
@@ -393,5 +402,6 @@ class GlmOcrForConditionalGeneration(Glm4vForConditionalGeneration):
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-5),
quant_config=quant_config,
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
+4 -3
View File
@@ -35,6 +35,7 @@ from vllm.distributed.parallel_state import (
)
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -156,11 +157,11 @@ class GPT2Block(nn.Module):
hidden_size = config.hidden_size
inner_dim = config.n_inner if config.n_inner is not None else 4 * hidden_size
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.ln_1 = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.attn = GPT2Attention(
config, cache_config, quant_config, prefix=f"{prefix}.attn"
)
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.ln_2 = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.mlp = GPT2MLP(inner_dim, config, quant_config, prefix=f"{prefix}.mlp")
def forward(
@@ -212,7 +213,7 @@ class GPT2Model(nn.Module):
lambda prefix: GPT2Block(config, cache_config, quant_config, prefix=prefix),
prefix=f"{prefix}.h",
)
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.ln_f = LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.n_embd
)
+3 -2
View File
@@ -31,6 +31,7 @@ from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -163,7 +164,7 @@ class GPTJBlock(nn.Module):
):
super().__init__()
inner_dim = 4 * config.n_embd if config.n_inner is None else config.n_inner
self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
self.ln_1 = LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
self.attn = GPTJAttention(
config, cache_config, quant_config, prefix=f"{prefix}.attn"
)
@@ -206,7 +207,7 @@ class GPTJModel(nn.Module):
lambda prefix: GPTJBlock(config, cache_config, quant_config, prefix=prefix),
prefix=f"{prefix}.h",
)
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.ln_f = LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.n_embd
)
+4 -1
View File
@@ -76,6 +76,8 @@ _GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD = {
"w2_bias": "gpt_oss_w2",
"w13_weight_scale": "gpt_oss_w13",
"w2_weight_scale": "gpt_oss_w2",
"w13_weight_packed": "gpt_oss_w13",
"w2_weight_packed": "gpt_oss_w2",
}
@@ -263,7 +265,7 @@ class GptOssRoutedExperts(RoutedExperts):
) -> None:
tp_rank = self.moe_config.moe_parallel_config.tp_rank
is_w13 = shard_id == "gpt_oss_w13"
is_weight = weight_name.endswith("_weight")
is_weight = weight_name.endswith(("_weight", "_weight_packed"))
is_partitioned_scale = weight_name.endswith("_weight_scale")
if is_weight:
@@ -347,6 +349,7 @@ class GptOssRoutedExperts(RoutedExperts):
) or quant_method_name in (
"CompressedTensorsW4A4Nvfp4MoEMethod",
"Nvfp4OnlineMoEMethod",
"CompressedTensorsWNA16MoEMethod",
):
self._load_packed_expert(expert_data, loaded_weight, weight_name, shard_id)
else:
+3 -2
View File
@@ -61,6 +61,7 @@ if TYPE_CHECKING:
from vllm.multimodal.registry import _ProcessorFactories
from vllm.sequence import IntermediateTensors
from vllm.tasks import ScoreType
from vllm.tokenizers import TokenizerLike
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
@@ -1450,7 +1451,7 @@ class SupportsTranscription(Protocol):
def parse_language_detection_output(
cls,
token_ids: list[int],
tokenizer: object,
tokenizer: "TokenizerLike",
) -> str:
"""Parse the detected language from model output token IDs.
@@ -1462,7 +1463,7 @@ class SupportsTranscription(Protocol):
@classmethod
def get_language_token_ids(
cls,
tokenizer: object,
tokenizer: "TokenizerLike",
) -> list[int] | None:
"""Return token IDs that represent valid language tokens.
@@ -418,12 +418,6 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLMBase):
self.quant_config = vllm_config.quant_config
self.scheduler_config = vllm_config.scheduler_config
if vllm_config.cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Intern-S2-Mobius does not support 'all' prefix caching; "
"use '--mamba-cache-mode=align' instead."
)
nn.Module.__init__(self)
self.config = config
self.model = InternS2MobiusModel(
+237 -95
View File
@@ -71,6 +71,7 @@ from vllm.model_executor.layers.linear import (
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -119,6 +120,96 @@ def interleaved_to_split(x):
return x.reshape(*x.shape[:-1], -1, 2).transpose(-1, -2).reshape(*x.shape[:-1], -1)
def is_rope_weights_folding_supported(
qk_proj: nn.Module,
dual_chunk_attention_config: dict[str, Any] | None,
) -> bool:
"""Whether the partial-RoPE weight-fold path is safe for this attention.
Folding permutes the q/k projection's output channels at load time, which
only produces correct numerics when:
* the q/k projection is dense/unquantized and
* dual-chunk attention is disabled -- the P.RoPE equivalence has not been
validated with dual-chunk attention.
When either condition fails we fall back to the runtime channel-permutation
RoPE path instead of folding.
"""
is_dense_qk_proj = isinstance(
getattr(qk_proj, "quant_method", None), UnquantizedLinearMethod
)
return is_dense_qk_proj and dual_chunk_attention_config is None
def apply_partial_rope(
rotary_emb,
positions: torch.Tensor,
q: torch.Tensor,
k: torch.Tensor,
num_heads: int,
num_kv_heads: int,
head_dim: int,
rope_head_dim: int,
fold_rope_weights: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
if fold_rope_weights or rope_head_dim == head_dim:
return rotary_emb(positions, q, k)
q = q.reshape(*q.shape[:-1], num_heads, head_dim)
k = k.reshape(*k.shape[:-1], num_kv_heads, head_dim)
q_rope, q_nope = torch.split(
split_to_interleaved(q),
split_size_or_sections=[rope_head_dim, head_dim - rope_head_dim],
dim=-1,
)
k_rope, k_nope = torch.split(
split_to_interleaved(k),
split_size_or_sections=[rope_head_dim, head_dim - rope_head_dim],
dim=-1,
)
q_rope, k_rope = rotary_emb(
positions, interleaved_to_split(q_rope), interleaved_to_split(k_rope)
)
q = interleaved_to_split(
torch.cat([split_to_interleaved(q_rope), q_nope], dim=-1)
).reshape(*q.shape[:-2], -1)
k = interleaved_to_split(
torch.cat([split_to_interleaved(k_rope), k_nope], dim=-1)
).reshape(*k.shape[:-2], -1)
return q, k
def _rope_weight_perm(head_dim: int, rope_head_dim: int) -> torch.Tensor:
"""Per-head gather index that folds K2Horizon's partial-RoPE channel reordering.
The K2Horizon checkpoint stores each head's channels in a GPT-J-interleaved
convention with rope/nope channels interleaved. Applying this fixed
(position-independent) permutation to the q/k weight rows lets the runtime
use vLLM's *native* NeoX partial-RoPE path (``head_size = head_dim``,
``rotary_dim = rope_head_dim``) with no per-forward permutes/all-gather.
"""
D, R = head_dim, rope_head_dim
assert R % 2 == 0 and R <= D, (
f"partial NeoX RoPE requires an even rope_head_dim <= head_dim, "
f"got R={R}, D={D}"
)
h = D // 2
idx = torch.cat(
[
torch.arange(0, R // 2), # rope first-half channels
torch.arange(h, h + R // 2), # rope second-half channels
torch.arange(R // 2, h), # nope remainder (first half)
torch.arange(h + R // 2, D), # nope remainder (second half)
]
)
return idx
def calc_router_weights(
router_logits: torch.Tensor,
e_score_correction_bias: torch.Tensor | None,
@@ -411,6 +502,7 @@ class K2HorizonSparseMoeBlock(nn.Module):
self.gate.bias = nn.Parameter(self.gate.bias.float(), requires_grad=False)
self.num_shared_experts = config.num_shared_experts
self.shared_experts: K2HorizonMLP | None
if config.num_shared_experts > 0:
self.shared_experts = K2HorizonMLP(
hidden_size=config.hidden_size,
@@ -529,11 +621,37 @@ class K2HorizonAttention(nn.Module):
)
self.rope_head_dim = rope_head_dim or self.head_dim
self.rotary_emb = get_rope(
self.rope_head_dim,
max_position=max_position_embeddings,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
# Fold the partial-RoPE channel permutation into the q/k weights only
# when supported (dense projections + no dual-chunk attention);
# otherwise keep the runtime channel-permutation path
# (see ``is_rope_weights_folding_supported``).
self.fold_rope_weights = is_rope_weights_folding_supported(
self.qkv_proj, dual_chunk_attention_config
)
if self.fold_rope_weights:
rope_parameters = dict(rope_parameters)
rope_parameters["rope_dim"] = self.rope_head_dim
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position_embeddings,
is_neox_style=True,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
)
else:
self.rotary_emb = get_rope(
self.rope_head_dim,
max_position=max_position_embeddings,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
)
attn_kwargs: dict[str, Any] = (
{
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
if dual_chunk_attention_config
else {}
)
self.attn = Attention(
self.num_heads,
@@ -543,12 +661,7 @@ class K2HorizonAttention(nn.Module):
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
**{
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
if dual_chunk_attention_config
else {},
**attn_kwargs,
)
self.query_key_norm = query_key_norm
@@ -584,40 +697,17 @@ class K2HorizonAttention(nn.Module):
q = self.q_norm(q)
k = self.k_norm(k)
if self.rope_head_dim == self.head_dim:
q, k = self.rotary_emb(positions, q, k)
else:
q = q.reshape(*q.shape[:-1], self.num_heads, self.head_dim)
k = k.reshape(*k.shape[:-1], self.num_kv_heads, self.head_dim)
q_rope, q_nope = torch.split(
split_to_interleaved(q),
split_size_or_sections=[
self.rope_head_dim,
self.head_dim - self.rope_head_dim,
],
dim=-1,
)
k_rope, k_nope = torch.split(
split_to_interleaved(k),
split_size_or_sections=[
self.rope_head_dim,
self.head_dim - self.rope_head_dim,
],
dim=-1,
)
q_rope, k_rope = self.rotary_emb(
positions, interleaved_to_split(q_rope), interleaved_to_split(k_rope)
)
q = interleaved_to_split(
torch.cat([split_to_interleaved(q_rope), q_nope], dim=-1)
).reshape(*q.shape[:-2], -1)
k = interleaved_to_split(
torch.cat([split_to_interleaved(k_rope), k_nope], dim=-1)
).reshape(*k.shape[:-2], -1)
q, k = apply_partial_rope(
self.rotary_emb,
positions,
q,
k,
self.num_heads,
self.num_kv_heads,
self.head_dim,
self.rope_head_dim,
self.fold_rope_weights,
)
attn_output = self.attn(q, k, v)
@@ -732,11 +822,37 @@ class K2HorizonMoVAAttention(nn.Module):
)
self.rope_head_dim = rope_head_dim or self.head_dim
self.rotary_emb = get_rope(
self.rope_head_dim,
max_position=max_position_embeddings,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
# Fold the partial-RoPE channel permutation into the q/k weights only
# when supported (dense projections + no dual-chunk attention);
# otherwise keep the runtime channel-permutation path
# (see ``is_rope_weights_folding_supported``).
self.fold_rope_weights = is_rope_weights_folding_supported(
self.qk_proj, dual_chunk_attention_config
)
if self.fold_rope_weights:
rope_parameters = dict(rope_parameters)
rope_parameters["rope_dim"] = self.rope_head_dim
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position_embeddings,
is_neox_style=True,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
)
else:
self.rotary_emb = get_rope(
self.rope_head_dim,
max_position=max_position_embeddings,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
)
attn_kwargs: dict[str, Any] = (
{
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
if dual_chunk_attention_config
else {}
)
self.attn = Attention(
self.num_heads,
@@ -746,12 +862,7 @@ class K2HorizonMoVAAttention(nn.Module):
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
**{
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
if dual_chunk_attention_config
else {},
**attn_kwargs,
)
self.query_key_norm = query_key_norm
@@ -838,40 +949,17 @@ class K2HorizonMoVAAttention(nn.Module):
q = self.q_norm(q)
k = self.k_norm(k)
if self.rope_head_dim == self.head_dim:
q, k = self.rotary_emb(positions, q, k)
else:
q = q.reshape(*q.shape[:-1], self.num_heads, self.head_dim)
k = k.reshape(*k.shape[:-1], self.num_kv_heads, self.head_dim)
q_rope, q_nope = torch.split(
split_to_interleaved(q),
split_size_or_sections=[
self.rope_head_dim,
self.head_dim - self.rope_head_dim,
],
dim=-1,
)
k_rope, k_nope = torch.split(
split_to_interleaved(k),
split_size_or_sections=[
self.rope_head_dim,
self.head_dim - self.rope_head_dim,
],
dim=-1,
)
q_rope, k_rope = self.rotary_emb(
positions, interleaved_to_split(q_rope), interleaved_to_split(k_rope)
)
q = interleaved_to_split(
torch.cat([split_to_interleaved(q_rope), q_nope], dim=-1)
).reshape(*q.shape[:-2], -1)
k = interleaved_to_split(
torch.cat([split_to_interleaved(k_rope), k_nope], dim=-1)
).reshape(*k.shape[:-2], -1)
q, k = apply_partial_rope(
self.rotary_emb,
positions,
q,
k,
self.num_heads,
self.num_kv_heads,
self.head_dim,
self.rope_head_dim,
self.fold_rope_weights,
)
attn_output = self.attn(q, k, v)
@@ -1109,6 +1197,27 @@ class K2HorizonModel(nn.Module, EagleModelMixin):
("gate_up_proj", "up_proj", 1),
]
# Per-head gather index used to fold K2Horizon's partial-RoPE channel
# reordering into the q_proj/k_proj rows and the q_norm/k_norm scales,
# so the runtime can use vLLM's native partial NeoX RoPE.
head_dim = self.config.head_dim or (
self.config.hidden_size // self.config.num_attention_heads
)
rope_head_dim = self.config.rope_head_dim or head_dim
rope_perm_idx = _rope_weight_perm(head_dim, rope_head_dim)
# Attention modules whose q/k projections are dense: only these fold the
# RoPE channel permutation into their q/k weights, biases and q/k norm
# scales at load time. Attentions with packed/quantized q/k projections
# keep the runtime channel-permutation path, so their weights must be
# loaded verbatim (folding would misalign per-channel scales).
rope_fold_prefixes = tuple(
f"{module_name}."
for module_name, module in self.named_modules()
if isinstance(module, (K2HorizonAttention, K2HorizonMoVAAttention))
and getattr(module, "fold_rope_weights", False)
)
# Skip loading extra parameters for GPTQ/modelopt models.
# ignore_suffixes = (
# ".bias",
@@ -1126,9 +1235,10 @@ class K2HorizonModel(nn.Module, EagleModelMixin):
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
if "scale" in name or "zero_point" in name:
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue
name = remapped_name
# QK norm weights
if name.endswith(".self_attn.q_norm.weight") or name.endswith(
@@ -1139,6 +1249,14 @@ class K2HorizonModel(nn.Module, EagleModelMixin):
if name not in params_dict:
continue
# Fold the per-head RoPE channel permutation into the norm scales.
if self.config.query_key_norm and name.startswith(rope_fold_prefixes):
loaded_weight = (
loaded_weight.view(-1, head_dim)[:, rope_perm_idx]
.reshape(-1)
.contiguous()
)
param = params_dict[name]
tp_rank = get_tensor_model_parallel_rank()
tp_size = get_tensor_model_parallel_world_size()
@@ -1157,7 +1275,9 @@ class K2HorizonModel(nn.Module, EagleModelMixin):
kv_rank
]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader: Callable[..., Any] = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
continue
@@ -1203,6 +1323,27 @@ class K2HorizonModel(nn.Module, EagleModelMixin):
if ".self_attn." in name and param_name == "gate_up_proj":
assert weight_name == "gate_proj"
continue
# Fold the per-head RoPE channel permutation into the dense
# q_proj / k_proj rows (W' = P W), and biases, before TP
# sharding.
if shard_id in ("q", "k") and name.startswith(rope_fold_prefixes):
if name.endswith(".weight"):
hidden = loaded_weight.shape[-1]
loaded_weight = (
loaded_weight.view(-1, head_dim, hidden)[
:, rope_perm_idx, :
]
.reshape(-1, hidden)
.contiguous()
)
elif self.config.attention_bias and name.endswith(".bias"):
loaded_weight = (
loaded_weight.view(-1, head_dim)[:, rope_perm_idx]
.reshape(-1)
.contiguous()
)
if (
".self_attn." in name
and param_name == "qkv_proj"
@@ -1249,9 +1390,10 @@ class K2HorizonModel(nn.Module, EagleModelMixin):
continue
if name.endswith("scale"):
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue
name = remapped_name
if name not in params_dict:
continue
+16 -15
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping, Sequence
from functools import partial
from typing import Annotated, Literal, TypeAlias
from typing import Annotated, Any, Literal, TypeAlias, cast
import numpy as np
import regex as re
@@ -288,7 +288,7 @@ class CustomQwen2VLVE(Qwen2VisionTransformer):
"""Drop-in replacement for the HF `_from_config` constructor."""
return cls(config)
def forward(
def forward( # type: ignore[override]
self,
pixel_values: torch.Tensor,
grid_thw: torch.Tensor,
@@ -331,12 +331,12 @@ class CustomQwen2VLVE(Qwen2VisionTransformer):
# Pre-compute seqlens for attention backend.
max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens)
encoder_states = () if output_hidden_states else None
encoder_states: tuple[torch.Tensor, ...] = ()
for blk in self.blocks:
if output_hidden_states:
# Store patch-level states (S, D).
encoder_states = encoder_states + (x.squeeze(1),)
encoder_states += (x.squeeze(1),)
x = blk(
x,
@@ -348,14 +348,15 @@ class CustomQwen2VLVE(Qwen2VisionTransformer):
# Final hidden state at patch level (S, D).
hidden_states = x.squeeze(1)
all_hidden_states = None
if output_hidden_states:
encoder_states = encoder_states + (hidden_states,)
all_hidden_states = encoder_states + (hidden_states,)
if not return_dict:
return tuple(v for v in [hidden_states, encoder_states] if v is not None)
return tuple(v for v in [hidden_states, all_hidden_states] if v is not None)
return BaseModelOutput(
last_hidden_state=hidden_states,
hidden_states=encoder_states,
hidden_states=all_hidden_states,
)
def get_num_tokens(self) -> int:
@@ -479,7 +480,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
assert isinstance(prompt_text, str)
# Images
image_inputs = hf_data.get("images", [])
image_inputs = cast(list[Any], hf_data.get("images", []))
pixel_sizes = []
if not isinstance(image_inputs[0], Image.Image):
image_inputs = [Image.fromarray(image) for image in image_inputs]
@@ -487,9 +488,9 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
image_processor = self.info.get_hf_processor().image_processor
processor_output = [image_processor(image) for image in image_inputs]
pixel_values = [o["pixel_values"] for o in processor_output]
image_meta = [o["image_meta"] for o in processor_output]
image_meta_list = [o["image_meta"] for o in processor_output]
# list of dict -> dict of list
image_meta = {k: [d[k] for d in image_meta] for k in image_meta[0]}
image_meta = {k: [d[k] for d in image_meta_list] for k in image_meta_list[0]}
for pixel_value in pixel_values:
pixel_sizes.append(pixel_value.shape[0])
@@ -545,7 +546,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
) -> Sequence[PromptUpdate]:
tokenizer = self.info.get_tokenizer()
def get_replacement(idx: int) -> Sequence[int]:
def get_replacement(idx: int) -> list[int]:
out_item = out_mm_kwargs["image"][idx]
image_token_thw = out_item["image_token_thw"].data
assert isinstance(image_token_thw, torch.Tensor)
@@ -683,7 +684,7 @@ class KananaVForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
def _get_visual_feature_at(
self,
v_output: Sequence[torch.Tensor],
layer_index: int | Sequence[int],
layer_index: int | list[int] | tuple[int, ...],
) -> torch.Tensor:
if isinstance(layer_index, (list, tuple)):
visual_features = torch.stack(v_output, dim=1)[
@@ -696,7 +697,7 @@ class KananaVForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
def forward_vision(
self,
pixel_values: torch.Tensor,
image_metas: dict | None = None,
image_metas: dict,
) -> torch.Tensor:
vision_model_args = {
"pixel_values": pixel_values,
@@ -714,7 +715,7 @@ class KananaVForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
def forward_projector(
self,
visual_features: torch.Tensor,
image_metas: dict | None = None,
image_metas: dict,
) -> torch.Tensor:
visual_embeds = self.abstractor(
visual_features,
@@ -725,7 +726,7 @@ class KananaVForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP)
def forward_and_project_vision(
self,
pixel_values: torch.Tensor,
image_metas: dict | None = None,
image_metas: dict,
) -> torch.Tensor:
assert pixel_values is not None
visual_features = self.forward_vision(pixel_values, image_metas=image_metas)
+29 -34
View File
@@ -208,8 +208,8 @@ class KeyeVisionEmbeddings(nn.Module):
self.num_patches = (self.image_size // self.patch_size) ** 2
self.num_positions = self.num_patches
self.cache_position_embedding = dict()
self.cache_position_count = dict()
self.cache_position_embedding: dict[tuple[int, int], torch.Tensor] = {}
self.cache_position_count: dict[tuple[int, int], int] = {}
self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
self.packing_position_embedding = nn.Embedding(32768, self.embed_dim)
@@ -264,7 +264,7 @@ class KeyeVisionEmbeddings(nn.Module):
if len(self.cache_position_embedding) >= max_cache:
min_hit_grid = min(
self.cache_position_count,
key=self.cache_position_count.get,
key=self.cache_position_count.__getitem__,
)
self.cache_position_count.pop(min_hit_grid)
self.cache_position_embedding.pop(min_hit_grid)
@@ -278,8 +278,7 @@ class KeyeVisionEmbeddings(nn.Module):
self,
pixel_values: torch.FloatTensor,
position_ids: torch.Tensor | None = None,
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
| None = None,
image_grid_thw: list[tuple[int, int, int]] | None = None,
interpolate_pos_encoding=False,
) -> torch.Tensor:
if pixel_values.dim() == 4:
@@ -408,7 +407,7 @@ class KeyeSiglipAttention(nn.Module):
hidden_states: torch.Tensor,
attention_mask: torch.Tensor | None = None,
output_attentions: bool | None = False,
cu_seqlens: list[torch.Tensor] | None = None,
cu_seqlens: torch.Tensor | None = None,
rope_emb: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
@@ -417,6 +416,8 @@ class KeyeSiglipAttention(nn.Module):
dim=-1,
)
if cu_seqlens is None:
raise ValueError("cu_seqlens cannot be None.")
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
if rope_emb is None:
@@ -432,8 +433,6 @@ class KeyeSiglipAttention(nn.Module):
self.head_dim,
)
else:
if cu_seqlens is None:
raise ValueError("cu_seqlens cannot be None when rope_emb is not None.")
cos, sin = rope_emb
q = q.view(*q.shape[:-1], self.num_heads, self.head_dim)
k = k.view(
@@ -511,7 +510,7 @@ class KeyeSiglipEncoderLayer(nn.Module):
hidden_states: torch.Tensor,
attention_mask: torch.Tensor,
output_attentions: bool | None = False,
cu_seqlens: list[torch.Tensor] | None = None,
cu_seqlens: torch.Tensor | None = None,
rope_emb: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> tuple[torch.FloatTensor]:
residual = hidden_states
@@ -576,13 +575,12 @@ class KeyeSiglipEncoder(nn.Module):
attention_mask: torch.Tensor | None = None,
output_attentions: bool | None = None,
output_hidden_states: bool | None = None,
cu_seqlens: list[torch.Tensor] | None = None,
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
| None = None,
cu_seqlens: torch.Tensor | None = None,
image_grid_thw: list[tuple[int, int, int]] | None = None,
height_position_ids: torch.Tensor | None = None,
width_position_ids: torch.Tensor | None = None,
use_rope: bool | None = False,
window_size: bool | None = -1,
window_size: int = -1,
vision_or_text: str = "vision",
) -> BaseModelOutput:
device = inputs_embeds.device
@@ -662,14 +660,13 @@ class KeyeSiglipVisionTransformer(nn.Module):
position_ids: torch.Tensor | None = None,
height_position_ids: torch.Tensor | None = None,
width_position_ids: torch.Tensor | None = None,
cu_seqlens: list[torch.Tensor] | None = None,
cu_seqlens: torch.Tensor | None = None,
padding_mask: torch.Tensor | None = None,
vision_return_embed_list: bool | None = False,
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
| None = None,
image_grid_thw: list[tuple[int, int, int]] | None = None,
return_pooler_output: bool | None = True,
use_rope: bool | None = False,
window_size: bool | None = -1,
window_size: int = -1,
) -> BaseModelOutputWithPooling:
hidden_states = self.embeddings(
pixel_values,
@@ -757,12 +754,11 @@ class KeyeSiglipVisionModel(nn.Module):
interpolate_pos_encoding: bool = False,
position_ids: torch.Tensor | None = None,
vision_return_embed_list: bool | None = False,
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
| None = None,
cu_seqlens: list[torch.Tensor] | None = None,
image_grid_thw: list[tuple[int, int, int]] | None = None,
cu_seqlens: torch.Tensor | None = None,
return_pooler_output: bool | None = True,
use_rope: bool | None = False,
window_size: bool | None = -1,
window_size: int = -1,
) -> BaseModelOutputWithPooling:
return self.vision_model(
pixel_values=pixel_values,
@@ -1337,7 +1333,7 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
def _process_video_embeds(
self,
video_type: Literal["video_embeds", "pixel_values_videos"],
video_grid_thw: list[torch.Tensor],
video_grid_thw: torch.Tensor,
pixel_values_videos: torch.Tensor | None = None,
) -> torch.Tensor | list[torch.Tensor]:
siglip_position_ids = list()
@@ -1361,6 +1357,7 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
"Video embeddings are not supported for this processing path."
)
else:
assert pixel_values_videos is not None
pixel_values_videos = pixel_values_videos.type(self.visual.dtype)
# These are host-built; concat straight into pinned buffers so
# the H2D copies stay non-blocking.
@@ -1535,12 +1532,11 @@ class KeyeForConditionalGeneration(
image_grid_thw=image_grid_thw,
)
if image_embeds is not None:
return KeyeImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
image_grid_thw=image_grid_thw,
)
return KeyeImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
image_grid_thw=image_grid_thw,
)
def _parse_and_validate_video_input(
self, **kwargs: object
@@ -1559,12 +1555,11 @@ class KeyeForConditionalGeneration(
video_grid_thw=video_grid_thw,
)
if video_embeds is not None:
return KeyeVideoEmbeddingInputs(
type="video_embeds",
video_embeds=video_embeds,
video_grid_thw=video_grid_thw,
)
return KeyeVideoEmbeddingInputs(
type="video_embeds",
video_embeds=video_embeds,
video_grid_thw=video_grid_thw,
)
def _process_video_input(
self, video_input: KeyeVideoInputs
+13 -15
View File
@@ -75,7 +75,7 @@ def split_thw(grid_thw: torch.Tensor) -> torch.Tensor:
def get_num_patches(
grid_thw: torch.Tensor, num_frames: list[int] | torch.Tensor
) -> list[int]:
) -> torch.Tensor:
"""Return num_patches per video.
Args:
@@ -84,7 +84,7 @@ def get_num_patches(
num_frames: List or tensor indicating the number of frames per video
Returns:
List of ints representing the number of patches for each video
Tensor of the number of patches for each video
Examples:
>>> # Suppose there are 2 videos with a total of 3 grids
@@ -538,12 +538,11 @@ class KeyeVL1_5ForConditionalGeneration(
image_grid_thw=image_grid_thw,
)
if image_embeds is not None:
return KeyeVL1_5ImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
image_grid_thw=image_grid_thw,
)
return KeyeVL1_5ImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
image_grid_thw=image_grid_thw,
)
def _parse_and_validate_video_input(
self, **kwargs: object
@@ -564,13 +563,12 @@ class KeyeVL1_5ForConditionalGeneration(
num_frames=num_frames,
)
if video_embeds is not None:
return KeyeVL1_5VideoEmbeddingInputs(
type="video_embeds",
video_embeds=video_embeds,
video_grid_thw=video_grid_thw,
num_frames=num_frames,
)
return KeyeVL1_5VideoEmbeddingInputs(
type="video_embeds",
video_embeds=video_embeds,
video_grid_thw=video_grid_thw,
num_frames=num_frames,
)
def _process_video_input(
self, video_input: KeyeVL1_5VideoInputs
+3 -1
View File
@@ -18,6 +18,7 @@ from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import PromptType, TokensPrompt
from vllm.model_executor.model_loader import DefaultModelLoader
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsMultiModal,
SupportsPP,
SupportsTranscription,
@@ -240,6 +241,7 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn
# KimiAudioProcessor expects raw numpy arrays
mm_data = hf_inputs.hf_data
if audios := mm_data.pop("audio", []):
assert isinstance(audios, Sequence)
audio_arrays = []
for aud in audios:
if isinstance(aud, (tuple, list)) and len(aud) == 2:
@@ -485,7 +487,7 @@ class KimiAudioForConditionalGeneration(
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: tuple[torch.Tensor, ...] | None = None,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
+12 -6
View File
@@ -136,9 +136,9 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
# may remap token IDs vs config.json.
config_token_id = hf_config.media_placeholder_token_id
resolved_token_id = tokenizer.convert_tokens_to_ids("<|media_pad|>")
unk_token_id = getattr(tokenizer, "unk_token_id", None)
is_valid_resolved = isinstance(resolved_token_id, int) and (
tokenizer.unk_token_id is None
or resolved_token_id != tokenizer.unk_token_id
unk_token_id is None or resolved_token_id != unk_token_id
)
if is_valid_resolved and resolved_token_id != config_token_id:
logger.warning_once(
@@ -165,7 +165,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
)
self.media_tokens_calculator = image_processor.media_tokens_calculator
def get_hf_processor(self):
def get_hf_processor(self, **kwargs: object):
return self.hf_processor
def get_hf_config(self):
@@ -191,7 +191,11 @@ class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]):
)
video_chunk_dummy_item = VisionChunkVideo(
type="video_chunk", video_chunk=dummy_videos
type="video_chunk",
video_chunk=dummy_videos,
uuid=None,
prompt="",
video_idx=0,
)
video_chunk_num_tokens = self.info.media_tokens_calculator(
video_chunk_dummy_item
@@ -204,6 +208,7 @@ class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]):
width=MaxImageTokenMeta.width,
num_images=1,
)[0],
uuid=None,
)
image_num_tokens = self.info.media_tokens_calculator(image_dummy_item)
# return the larger one
@@ -377,7 +382,7 @@ class KimiK25ForConditionalGeneration(
)
self.media_placeholder: int = self.config.media_placeholder_token_id
def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig | None):
if isinstance(quant_config, compressed_tensors.CompressedTensorsConfig):
return None
return quant_config
@@ -392,6 +397,7 @@ class KimiK25ForConditionalGeneration(
if isinstance(pixel_values, list):
pixel_values = torch.cat(pixel_values, dim=0)
assert isinstance(pixel_values, torch.Tensor)
if len(pixel_values.shape) == 5 or len(pixel_values.shape) == 3:
pixel_values = pixel_values.reshape(
@@ -612,7 +618,7 @@ class KimiK25ForConditionalGeneration(
grid_thw_list = [[1, ho * kh, wo * kw] for _ in range(max_batch_size)]
ps = self.vision_tower.patch_size
ps: int | tuple[int, int] = self.vision_tower.patch_size
if isinstance(ps, int):
ps = (ps, ps)
total_patches = max_batch_size * ho * kh * wo * kw
+1 -1
View File
@@ -804,7 +804,7 @@ class MoonViT3dPretrainedModel(nn.Module):
self,
grid_thw_list: list[list[int]],
*,
max_batch_size: int,
max_batch_size: int | None,
max_seqlen_override: int | None = None,
device: torch.device,
) -> dict[str, torch.Tensor | None]:
+3 -2
View File
@@ -268,6 +268,7 @@ class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
if isinstance(images, ImageEmbeddingItems):
num_image_tokens = images.get_feature_size(item_idx)
else:
assert isinstance(images, ImageProcessorItems)
image_size = images.get_image_size(item_idx)
num_image_tokens = self.info.get_num_image_tokens(
image_width=image_size.width,
@@ -317,7 +318,7 @@ class KimiVLForConditionalGeneration(
assert isinstance(config.vision_config, MoonViTConfig)
self.use_data_parallel = (
model_config.multimodal_config.mm_encoder_tp_mode == "data"
model_config.get_multimodal_config().mm_encoder_tp_mode == "data"
)
self.hidden_size = config.text_config.hidden_size
@@ -469,7 +470,7 @@ class KimiVLForConditionalGeneration(
)
grid_hws_list = [(ho * kh, wo * kw) for _ in range(max_batch_size)]
patch_size = self.config.vision_config.patch_size
patch_size: int | tuple[int, int] = self.config.vision_config.patch_size
if isinstance(patch_size, int):
patch_size = (patch_size, patch_size)
-7
View File
@@ -458,13 +458,6 @@ class Lfm2ForCausalLM(
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Lfm2 currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
super().__init__()
self.config = config
self.model = Lfm2Model(
-7
View File
@@ -573,13 +573,6 @@ class Lfm2MoeForCausalLM(
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Lfm2Moe currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
super().__init__()
self.config = config
+2
View File
@@ -115,6 +115,8 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessing
dummy_inputs=Mistral3DummyInputsBuilder,
)
class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration):
supports_mm_device_do_normalize = False
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"model.vision_encoder.": "vision_tower.",
-253
View File
@@ -1,253 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from .qwen3_moe import (
Qwen3MoeAttention,
Qwen3MoeDecoderLayer,
Qwen3MoeForCausalLM,
Qwen3MoeMLP,
Qwen3MoeModel,
Qwen3MoeSparseMoeBlock,
)
from .utils import PPMissingLayer, extract_layer_index, maybe_prefix
class MellumAttention(Qwen3MoeAttention):
"""Differences from `Qwen3MoeAttention`:
- Supports `per_layer_sliding_window` for `Attention`.
"""
def __init__(
self,
hidden_size: int,
num_heads: int,
num_kv_heads: int,
rope_parameters: dict[str, Any],
max_position_embeddings: int = 8192,
head_dim: int | None = None,
rms_norm_eps: float = 1e-06,
qkv_bias: bool = False,
cache_config: Any | None = None,
quant_config: Any | None = None,
prefix: str = "",
dual_chunk_attention_config: dict[str, Any] | None = None,
per_layer_sliding_window: int | None = None,
) -> None:
nn.Module.__init__(self)
self.hidden_size = hidden_size
tp_size = get_tensor_model_parallel_world_size()
self.total_num_heads = num_heads
assert self.total_num_heads % tp_size == 0
self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = num_kv_heads
if self.total_num_kv_heads >= tp_size:
assert self.total_num_kv_heads % tp_size == 0
else:
assert tp_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
self.head_dim = head_dim or (hidden_size // self.total_num_heads)
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
self.scaling = self.head_dim**-0.5
self.max_position_embeddings = max_position_embeddings
self.dual_chunk_attention_config = dual_chunk_attention_config
self.qkv_proj = QKVParallelLinear(
hidden_size,
self.head_dim,
self.total_num_heads,
self.total_num_kv_heads,
bias=qkv_bias,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.o_proj = RowParallelLinear(
self.total_num_heads * self.head_dim,
hidden_size,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
)
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position_embeddings,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
)
if dual_chunk_attention_config:
self.attn = Attention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
per_layer_sliding_window=per_layer_sliding_window,
prefix=f"{prefix}.attn",
layer_idx=extract_layer_index(prefix),
dual_chunk_attention_config=dual_chunk_attention_config,
)
else:
self.attn = Attention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
per_layer_sliding_window=per_layer_sliding_window,
prefix=f"{prefix}.attn",
)
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
class MellumDecoderLayer(Qwen3MoeDecoderLayer):
"""Differences from `Qwen3MoeDecoderLayer`:
- Supports interleaved SWA and per-layer RoPE scaling.
"""
def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
nn.Module.__init__(self)
config = vllm_config.model_config.hf_text_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.hidden_size = config.hidden_size
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
dual_chunk_attention_config = getattr(
config, "dual_chunk_attention_config", None
)
layer_idx = extract_layer_index(prefix)
layer_type = config.layer_types[layer_idx]
if layer_type == "sliding_attention":
sliding_window = getattr(config, "sliding_window", None)
else:
sliding_window = None
rope_parameters = config.rope_parameters[layer_type]
self.self_attn = MellumAttention(
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
num_kv_heads=config.num_key_value_heads,
rope_parameters=rope_parameters,
max_position_embeddings=max_position_embeddings,
rms_norm_eps=config.rms_norm_eps,
qkv_bias=getattr(config, "attention_bias", False),
head_dim=getattr(config, "head_dim", None),
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
dual_chunk_attention_config=dual_chunk_attention_config,
per_layer_sliding_window=sliding_window,
)
if config.mlp_layer_types[layer_idx] == "sparse":
self.mlp = Qwen3MoeSparseMoeBlock(
vllm_config=vllm_config, prefix=f"{prefix}.mlp"
)
else:
self.mlp = Qwen3MoeMLP(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
@support_torch_compile
class MellumModel(Qwen3MoeModel):
"""Differences from `Qwen3MoeModel`:
- Uses `MellumDecoderLayer`.
"""
def __init__(
self,
*,
vllm_config: VllmConfig,
prefix: str = "",
):
super().__init__(
vllm_config=vllm_config,
prefix=prefix,
decoder_layer_type=MellumDecoderLayer,
)
class MellumForCausalLM(Qwen3MoeForCausalLM):
"""Differences from `Qwen3MoeForCausalLM`:
- Uses `MellumModel`.
"""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self)
config = vllm_config.model_config.hf_text_config
quant_config = vllm_config.quant_config
self.config = config
self.quant_config = quant_config
if "dense" in getattr(config, "mlp_layer_types", []):
self.packed_modules_mapping["gate_up_proj"] = ["gate_proj", "up_proj"]
self.model = MellumModel(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
if self.config.tie_word_embeddings:
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
self.moe_layers = []
example_layer = None
for layer in self.model.layers:
if isinstance(layer, PPMissingLayer):
continue
assert isinstance(layer, Qwen3MoeDecoderLayer)
if isinstance(layer.mlp, Qwen3MoeSparseMoeBlock):
example_layer = layer.mlp
self.moe_layers.append(layer.mlp.experts)
if example_layer is None:
raise RuntimeError("No MoE layer found in the model.layers.")
self.num_moe_layers = len(self.moe_layers)
self.num_expert_groups = 1
self.num_shared_experts = 0
self.num_logical_experts = example_layer.n_logical_experts
self.num_physical_experts = example_layer.n_physical_experts
self.num_local_physical_experts = example_layer.n_local_physical_experts
self.num_routed_experts = example_layer.n_routed_experts
self.num_redundant_experts = example_layer.n_redundant_experts
+6 -2
View File
@@ -742,8 +742,12 @@ class MiMoV2Model(nn.Module, EagleModelMixin):
expert_params_mapping = self.get_expert_mapping()
# Pro-format fused qkv_proj arrives as two tensors (weight and
# weight_scale_inv). Store them per-layer so that they can be
# sharded together.
pending_fp8_qkv_proj: dict[str, dict[str, torch.Tensor]] = {}
# sharded together. The state must outlive this call: AutoWeightsLoader
# delegates per contiguous group of names, so a pair can straddle two
# calls and would otherwise be dropped silently.
pending_fp8_qkv_proj = getattr(self, "_pending_fp8_qkv_proj", None)
if pending_fp8_qkv_proj is None:
self._pending_fp8_qkv_proj = pending_fp8_qkv_proj = {}
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
+6 -1
View File
@@ -266,7 +266,12 @@ class MiMoV2MTP(nn.Module):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
pending_qkv_proj: dict[str, dict[str, torch.Tensor]] = {}
# The pairing state must outlive this call: AutoWeightsLoader delegates
# per contiguous group of names, so a pair can straddle two calls and
# would otherwise be dropped silently.
pending_qkv_proj = getattr(self, "_pending_qkv_proj", None)
if pending_qkv_proj is None:
self._pending_qkv_proj = pending_qkv_proj = {}
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
+20 -4
View File
@@ -69,7 +69,10 @@ from .qwen2_5_vl import (
Qwen2_5_VLVideoInputs,
Qwen2_5_VLVideoPixelInputs,
)
from .qwen2_vl import _create_qwen2vl_field_factory
from .qwen2_vl import (
Qwen2VLMultiModalDataParser,
_create_qwen2vl_field_factory,
)
from .utils import AutoWeightsLoader, IntermediateTensors, WeightsMapper, maybe_prefix
@@ -670,6 +673,14 @@ class MiMoVisionTransformer(nn.Module):
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class MiMoV2OmniMultiModalDataParser(Qwen2VLMultiModalDataParser):
# Video stays raw: get_video_replacement also reads second_per_grid_ts and
# video_start_times, which a metadata-only rewrite would not carry.
embedding_fields = {
"image": Qwen2VLMultiModalDataParser.embedding_fields["image"],
}
class MiMoV2OmniProcessingInfo(BaseProcessingInfo):
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": None, "image": None, "video": None}
@@ -689,9 +700,14 @@ class MiMoV2OmniProcessingInfo(BaseProcessingInfo):
return self.get_hf_processor(**kwargs).image_processor
def get_data_parser(self):
from vllm.multimodal.parse import MultiModalDataParser
return MultiModalDataParser(target_sr=24000.0)
# Without embedding_fields the EC producer publishes no metadata and
# the EPD proxy never rewrites the media item.
return MiMoV2OmniMultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
target_sr=24000.0,
expected_hidden_size=self._get_expected_hidden_size(),
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_mm_max_tokens_per_item(
self,
+19 -3
View File
@@ -7,6 +7,8 @@ from typing import Annotated, Any, Literal, cast
import torch
import torch.nn as nn
import transformers
from packaging.version import Version
from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig
from transformers.models.pixtral import PixtralProcessor
@@ -14,6 +16,7 @@ from vllm.config import VllmConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -56,6 +59,10 @@ from .utils import (
maybe_prefix,
)
TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY = Version(transformers.__version__) >= Version(
"5.15.0"
)
class Mistral3ImagePixelInputs(TensorSchema):
"""Dimensions:
@@ -225,7 +232,7 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):
image_size = size["longest_edge"]
return Mistral3HFEncoderInfo(self.get_hf_config(), image_size)
def get_hf_processor(self, **kwargs: object):
def get_hf_processor(self, **kwargs: object) -> PixtralProcessor:
return self.ctx.get_hf_processor(PixtralProcessor, **kwargs)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
@@ -277,8 +284,12 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder):
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None:
# PixtralProcessor supports image-only calls starting in transformers 5.15.
# Older releases need dummy text and take the slower tokenization path.
if not TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY:
return self.dummy_inputs.get_dummy_text(mm_counts)
return None
def _postprocess_hf_mm_data(
self,
@@ -378,6 +389,7 @@ def init_vision_tower_for_mistral3(
hf_config: Mistral3Config,
quant_config: QuantizationConfig | None,
*,
input_norm: nn.Module | None = None,
require_post_norm: bool | None = None,
prefix: str = "",
) -> PixtralHFVisionModel:
@@ -391,6 +403,7 @@ def init_vision_tower_for_mistral3(
return PixtralHFVisionModel(
vision_config,
quant_config=quant_config,
input_norm=input_norm,
num_hidden_layers_override=num_hidden_layers,
require_post_norm=require_post_norm,
prefix=prefix,
@@ -410,6 +423,8 @@ class Mistral3ForConditionalGeneration(
SupportsEagle,
SupportsEagle3,
):
supports_mm_device_do_normalize = True
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
@@ -464,6 +479,7 @@ class Mistral3ForConditionalGeneration(
self.vision_tower = init_vision_tower_for_mistral3(
config,
quant_config=quant_config,
input_norm=build_mm_input_norm(vllm_config.model_config),
require_post_norm=False,
prefix=maybe_prefix(prefix, "vision_tower"),
)
+42 -13
View File
@@ -33,6 +33,7 @@ from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.linear import (
@@ -77,23 +78,22 @@ def _cast_if_autocast_enabled(device_type: str, *args):
)
class NemotronLayerNorm1P(nn.LayerNorm):
def __init__(
self,
normalized_shape: int | list[int] | torch.Size,
eps: float = 1e-5,
elementwise_affine: bool = True,
bias: bool = True,
device=None,
dtype=None,
):
super().__init__(normalized_shape, eps, elementwise_affine, bias, device, dtype)
@CustomOp.register("nemotron_layer_norm")
class NemotronLayerNorm1P(CustomOp):
"""LayerNorm variant used by Nemotron: `x * (1 + w)` instead of `x * w`."""
def forward(
def __init__(self, hidden_size: int, eps: float = 1e-5) -> None:
super().__init__()
self.normalized_shape = (hidden_size,)
self.eps = eps
self.weight = nn.Parameter(torch.zeros(hidden_size))
self.bias = nn.Parameter(torch.zeros(hidden_size))
def forward_native(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if residual is not None:
x = x + residual
residual = x
@@ -105,6 +105,35 @@ class NemotronLayerNorm1P(nn.LayerNorm):
x = torch.nn.functional.layer_norm(*args)
return x if residual is None else (x, residual)
def forward_cuda(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
return self.forward_native(x, residual)
def forward_xpu(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_nemotron_layer_norm
if not hasattr(torch.ops._C, "nemotron_layer_norm"):
return self.forward_native(x, residual)
if residual is not None:
if not hasattr(torch.ops._C, "fused_add_nemotron_layer_norm"):
return self.forward_native(x, residual)
torch.ops.vllm.xpu_fused_add_nemotron_layer_norm(
x, residual, self.weight, self.bias, self.eps
)
return x, residual
# empty_like preserves x's strides, but the kernel requires a
# contiguous out (unlike x, which it can handle non-contiguous).
out = torch.empty(x.shape, device=x.device, dtype=x.dtype)
torch.ops.vllm.xpu_nemotron_layer_norm(out, x, self.weight, self.bias, self.eps)
return out
class NemotronMLP(nn.Module):
def __init__(
+1
View File
@@ -815,6 +815,7 @@ class NemotronHForCausalLM(
tp_world_size=parallel_config.tensor_parallel_size,
logical_window=cache_config.replayssm_buffer_len,
backend=vllm_config.mamba_config.backend,
num_speculative_tokens=vllm_config.num_speculative_tokens,
)
return base_shape
@@ -568,6 +568,15 @@ class NemotronParseForConditionalGeneration(nn.Module, SupportsMultiModal):
self.lm_head = ParallelLMHead(
config.decoder.vocab_size, config.decoder.d_model, quant_config=quant_config
)
# Some checkpoints (e.g. compact exports) tie the output head to the
# decoder's input embeddings instead of materializing a separate
# lm_head.weight tensor.
tie_word_embeddings = bool(
getattr(config, "tie_word_embeddings", False)
or getattr(config.decoder, "tie_word_embeddings", False)
)
if tie_word_embeddings:
self.lm_head = self.lm_head.tie_weights(self.decoder.embed_tokens)
self.logits_processor = LogitsProcessor(
self.vocab_size, config.decoder.vocab_size
)
+20
View File
@@ -8,6 +8,10 @@ import torch.nn as nn
from transformers import AutoModel, PreTrainedConfig
from vllm.config import VllmConfig
from vllm.model_executor.layers.fusion.mm_input_norm import (
FusedMMInputNorm,
IdentityInputNorm,
)
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.pooler import DispatchPooler
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -26,6 +30,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import cached_image_processor_from_config
from vllm.transformers_utils.processors.nemotron_vl import (
SIGLIP_MEAN,
SIGLIP_STD,
LlamaNemotronNanoVLImageProcessor,
LlamaNemotronNanoVLProcessor,
LlamaNemotronVLEmbedImageProcessor,
@@ -474,6 +480,7 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling)
"""
is_pooling_model = True
supports_mm_device_do_normalize = True
# Weight mapping from checkpoint format to vLLM format
# Different from parent class due to different vision model structure
@@ -498,6 +505,15 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling)
# Override: get img_context_token_id from config (parent sets None)
self.img_context_token_id = getattr(config, "img_context_token_id", None)
if self.model_config.get_multimodal_config().mm_device_do_normalize:
self.input_norm = FusedMMInputNorm(
image_mean=list(SIGLIP_MEAN),
image_std=list(SIGLIP_STD),
rescale_factor=1 / 255,
)
else:
self.input_norm = IdentityInputNorm()
# Initialize pooler for embedding output
pooler_config = vllm_config.model_config.pooler_config
assert pooler_config is not None
@@ -537,6 +553,10 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling)
def _call_vision_model(self, pixel_values: torch.Tensor) -> torch.Tensor:
"""Override to handle SigLIP interface."""
original_shape = pixel_values.shape
pixel_values = self.input_norm(
pixel_values.flatten(start_dim=1), self.vision_model.dtype
).view(original_shape)
return self.vision_model(pixel_values)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+5 -3
View File
@@ -32,6 +32,7 @@ from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -143,7 +144,7 @@ class OPTDecoderLayer(nn.Module):
)
self.do_layer_norm_before = config.do_layer_norm_before
self.self_attn_layer_norm = nn.LayerNorm(
self.self_attn_layer_norm = LayerNorm(
self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine
)
self.fc1 = ColumnParallelLinear(
@@ -161,7 +162,7 @@ class OPTDecoderLayer(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.fc2",
)
self.final_layer_norm = nn.LayerNorm(
self.final_layer_norm = LayerNorm(
self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine
)
@@ -246,8 +247,9 @@ class OPTDecoder(nn.Module):
# keep backward compatibility with checkpoints that have been fine-tuned
# before transformers v4.20.1
# see https://github.com/facebookresearch/metaseq/pull/164
self.final_layer_norm: LayerNorm | None
if config.do_layer_norm_before and not config._remove_final_layer_norm:
self.final_layer_norm = nn.LayerNorm(
self.final_layer_norm = LayerNorm(
config.hidden_size,
elementwise_affine=config.layer_norm_elementwise_affine,
)
+95 -14
View File
@@ -10,6 +10,7 @@ from typing import Annotated, Literal
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
from mistral_common.protocol.instruct.messages import UserMessage
from mistral_common.protocol.instruct.request import ChatCompletionRequest
@@ -20,6 +21,10 @@ from transformers.models.pixtral.image_processing_pixtral import (
)
from transformers.models.pixtral.modeling_pixtral import apply_rotary_pos_emb
from vllm.compilation.decorators import (
should_torch_compile_mm_encoder,
support_torch_compile,
)
from vllm.config import VllmConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import divide, get_tensor_model_parallel_world_size
@@ -828,6 +833,16 @@ class Attention(nn.Module):
return out
@support_torch_compile(
dynamic_arg_dims={
"x": 1,
"freqs_cis": 0,
"cu_seqlens": 0,
"sequence_lengths": 0,
},
enable_if=should_torch_compile_mm_encoder,
is_encoder=True,
)
class TransformerBlock(nn.Module):
def __init__(
self,
@@ -1297,7 +1312,7 @@ class PixtralHFAttention(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
position_embeddings: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
sequence_lengths: torch.Tensor | None,
@@ -1328,6 +1343,17 @@ class PixtralHFAttention(nn.Module):
return attn_output, None
@support_torch_compile(
dynamic_arg_dims={
"hidden_states": 1,
"rotary_pos_emb_cos": 0,
"rotary_pos_emb_sin": 0,
"cu_seqlens": 0,
"sequence_lengths": 0,
},
enable_if=should_torch_compile_mm_encoder,
is_encoder=True,
)
class PixtralHFTransformerBlock(nn.Module):
def __init__(
self,
@@ -1354,14 +1380,15 @@ class PixtralHFTransformerBlock(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
position_embeddings: torch.Tensor,
rotary_pos_emb_cos: torch.Tensor,
rotary_pos_emb_sin: torch.Tensor,
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
sequence_lengths: torch.Tensor | None,
) -> torch.Tensor:
r, _ = self.attention.forward(
self.attention_norm(hidden_states),
position_embeddings=position_embeddings,
position_embeddings=(rotary_pos_emb_cos, rotary_pos_emb_sin),
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
@@ -1402,18 +1429,20 @@ class PixtralHFTransformer(nn.Module):
def forward(
self,
x: torch.Tensor,
position_embeddings: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
sequence_lengths: torch.Tensor | None,
return_all_hidden_states: bool,
) -> torch.Tensor:
hidden_states_pool = [x]
rotary_pos_emb_cos, rotary_pos_emb_sin = position_embeddings
for layer in self.layers:
x = layer(
x,
position_embeddings,
rotary_pos_emb_cos,
rotary_pos_emb_sin,
cu_seqlens,
max_seqlen,
sequence_lengths,
@@ -1427,12 +1456,55 @@ class PixtralHFTransformer(nn.Module):
return x
def pixtral_patch_embed(
images: list[torch.Tensor],
weight: torch.Tensor,
input_norm: nn.Module,
) -> tuple[torch.Tensor, list[torch.Tensor]]:
"""Normalize and project all CHW images in one matrix multiplication."""
out_channels, in_channels, patch_height, patch_width = weight.shape
grid_sizes = []
for image in images:
assert image.ndim == 3 and image.shape[0] == in_channels
_, height, width = image.shape
patch_rows = height // patch_height
patch_cols = width // patch_width
grid_sizes.append((patch_rows, patch_cols))
patch_counts = [rows * cols for rows, cols in grid_sizes]
packed = torch.empty(
(sum(patch_counts), in_channels * patch_height * patch_width),
dtype=images[0].dtype,
device=images[0].device,
)
offset = 0
for image, (rows, cols), count in zip(images, grid_sizes, patch_counts):
source = (
image[:, : rows * patch_height, : cols * patch_width]
.reshape(in_channels, rows, patch_height, cols, patch_width)
.permute(1, 3, 0, 2, 4)
)
packed[offset : offset + count].view(
rows, cols, in_channels, patch_height, patch_width
).copy_(source)
offset += count
normalized = input_norm(packed, weight.dtype)
projected = F.linear(normalized, weight.reshape(out_channels, -1))
embeddings = [
tokens.transpose(0, 1).reshape(1, out_channels, rows, cols)
for tokens, (rows, cols) in zip(projected.split(patch_counts), grid_sizes)
]
return projected.unsqueeze(0), embeddings
class PixtralHFVisionModel(nn.Module):
def __init__(
self,
config: PixtralVisionConfig,
quant_config: QuantizationConfig | None = None,
*,
input_norm: nn.Module | None = None,
num_hidden_layers_override: int | None = None,
require_post_norm: bool | None = None,
prefix: str = "",
@@ -1440,6 +1512,7 @@ class PixtralHFVisionModel(nn.Module):
super().__init__()
self.config = config
self.input_norm = input_norm
self.patch_conv = Conv2dLayer(
in_channels=config.num_channels,
@@ -1495,16 +1568,24 @@ class PixtralHFVisionModel(nn.Module):
all tokens of all images of shape (N_toks, D)
"""
# pass images through initial convolution independently
patch_embeds_list = [
self.patch_conv(img.unsqueeze(0).to(self.dtype)) for img in pixel_values
]
images = [image.to(device=self.device) for image in pixel_values]
if self.input_norm is not None:
patch_embeds, patch_embeds_list = pixtral_patch_embed(
images,
self.patch_conv.weight,
self.input_norm,
)
else:
patch_embeds_list = []
for image in images:
patch_embeds_list.append(
self.patch_conv(image.unsqueeze(0).to(self.dtype))
)
patch_embeds = torch.cat(
[p.flatten(2).permute(0, 2, 1) for p in patch_embeds_list], dim=1
)
patch_embeds = [p.flatten(2).permute(0, 2, 1) for p in patch_embeds_list]
embed_sizes = [p.shape[1] for p in patch_embeds]
# flatten to a single sequence
patch_embeds = torch.cat(patch_embeds, dim=1)
embed_sizes = [p.shape[-2] * p.shape[-1] for p in patch_embeds_list]
patch_embeds = self.ln_pre(patch_embeds)
# positional embeddings
-6
View File
@@ -331,14 +331,8 @@ class Qwen3_5ForCausalLMBase(
config = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
scheduler_config = vllm_config.scheduler_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3.5 currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
@@ -303,13 +303,6 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
config = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3_5MTP currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
-6
View File
@@ -828,14 +828,8 @@ class Qwen3NextForCausalLM(
config = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
scheduler_config = vllm_config.scheduler_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3Next currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
@@ -187,13 +187,6 @@ class Qwen3NextMTP(nn.Module, QwenNextMixtureOfExperts):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
config = vllm_config.model_config.hf_config
self.vllm_config = vllm_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3NextMTP currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
+4 -5
View File
@@ -72,7 +72,6 @@ _TEXT_GENERATION_MODELS = {
# [Decoder-only]
"AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
"ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
"ArceeForCausalLM": ("arcee", "ArceeForCausalLM"),
"AXK1ForCausalLM": ("AXK1", "AXK1ForCausalLM"),
"BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
"BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"),
@@ -84,7 +83,6 @@ _TEXT_GENERATION_MODELS = {
"CohereForCausalLM": ("commandr", "CohereForCausalLM"),
"Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"),
"Cohere2MoeForCausalLM": ("cohere2_moe", "Cohere2MoeForCausalLM"),
"CwmForCausalLM": ("llama", "LlamaForCausalLM"),
"DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
"DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
"DeepseekForCausalLM": ("deepseek_v2", "DeepseekForCausalLM"),
@@ -114,7 +112,6 @@ _TEXT_GENERATION_MODELS = {
"vllm.models.qwen4_exp",
"Qwen4ExpForCausalLM",
),
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
"Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
"Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
@@ -160,7 +157,6 @@ _TEXT_GENERATION_MODELS = {
),
"MambaForCausalLM": ("mamba", "MambaForCausalLM"),
"Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"),
"MellumForCausalLM": ("mellum", "MellumForCausalLM"),
"MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
"MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
"MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"),
@@ -226,7 +222,6 @@ _EMBEDDING_MODELS = {
"DeepseekV3BidirectionalModel": ("deepseek_v2", "DeepseekV3ForCausalLM"),
"Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
"Gemma3TextModel": ("gemma3", "Gemma3Model"),
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
"GteModel": ("bert_with_rope", "SnowflakeGteNewModel"),
"GteNewModel": ("bert_with_rope", "GteNewModel"),
"JinaEmbeddingsV5Model": ("jina", "JinaEmbeddingsV5Model"),
@@ -709,10 +704,14 @@ _SPECULATIVE_DECODING_MODELS = {
_TRANSFORMERS_SUPPORTED_MODELS = {
# Text generation models
"ArceeForCausalLM": ("transformers", "TransformersForCausalLM"),
"CwmForCausalLM": ("transformers", "TransformersForCausalLM"),
"FlexOlmoForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"GlmForCausalLM": ("transformers", "TransformersForCausalLM"),
"GPTBigCodeForCausalLM": ("transformers", "TransformersForCausalLM"),
"HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"),
"HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"),
"OlmoForCausalLM": ("transformers", "TransformersForCausalLM"),
"Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"),
+8 -3
View File
@@ -117,6 +117,8 @@ class RobertaClassificationHead(nn.Module):
class RobertaEmbeddingModel(BertEmbeddingModel):
"""A model that uses Roberta to provide embedding functionalities."""
embedding_class: type[nn.Module] = RobertaEmbedding
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
self.padding_idx: int = vllm_config.model_config.hf_config.pad_token_id
@@ -143,7 +145,7 @@ class RobertaEmbeddingModel(BertEmbeddingModel):
return BertModel(
vllm_config=vllm_config,
prefix=prefix,
embedding_class=RobertaEmbedding,
embedding_class=self.embedding_class,
)
else:
return JinaRobertaModel(vllm_config=vllm_config, prefix=prefix)
@@ -293,6 +295,8 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding, Support
}
)
embedding_class: type[nn.Module] = RobertaEmbedding
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
@@ -303,7 +307,7 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding, Support
self.roberta = BertModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "bert"),
embedding_class=RobertaEmbedding,
embedding_class=self.embedding_class,
)
self.classifier = RobertaClassificationHead(vllm_config.model_config)
@@ -355,6 +359,7 @@ class RobertaForTokenClassification(nn.Module):
"""
is_pooling_model = True
embedding_class: type[nn.Module] = RobertaEmbedding
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -364,7 +369,7 @@ class RobertaForTokenClassification(nn.Module):
self.roberta = BertModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "roberta"),
embedding_class=RobertaEmbedding,
embedding_class=self.embedding_class,
)
self.classifier = nn.Linear(
config.hidden_size, config.num_labels, dtype=self.head_dtype
+23 -13
View File
@@ -73,6 +73,7 @@ from .utils import (
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
spec_decode_needs_target_embed,
)
@@ -460,7 +461,9 @@ class SarvamMLABlock(nn.Module):
}
)
class SarvamMLAModel(nn.Module, EagleModelMixin):
"""Sarvam MLA backbone with stage-local EAGLE3 auxiliary capture."""
"""Sarvam MLA backbone with EAGLE3 auxiliary capture across pipeline stages."""
supports_aux_hidden_states_over_pp = True
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
@@ -487,8 +490,10 @@ class SarvamMLAModel(nn.Module, EagleModelMixin):
self.vocab_size = config.vocab_size
self.embed_dim = config.hidden_size
self.tie_word_embeddings = getattr(config, "tie_word_embeddings", False)
if get_pp_group().is_first_rank or (
self.tie_word_embeddings and get_pp_group().is_last_rank
if (
get_pp_group().is_first_rank
or (self.tie_word_embeddings and get_pp_group().is_last_rank)
or spec_decode_needs_target_embed(vllm_config)
):
self.embed_tokens = VocabParallelEmbedding(
self.vocab_size,
@@ -530,13 +535,10 @@ class SarvamMLAModel(nn.Module, EagleModelMixin):
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
"""Run this stage and optionally return its auxiliary hidden states.
Auxiliary captures are local to this stage, matching Qwen3 MoE.
Pipeline transport carries only hidden states and residual; EAGLE3
capture across pipeline stages is not supported by this model.
Returns:
Intermediate tensors on non-final stages. On the final stage,
normalized hidden states, paired with auxiliary states if captured.
Intermediate tensors with local auxiliary states on non-final stages.
On the final stage, normalized hidden states paired with the ordered
auxiliary states from all stages if captured.
"""
if get_pp_group().is_first_rank:
@@ -551,9 +553,12 @@ class SarvamMLAModel(nn.Module, EagleModelMixin):
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
aux_hidden_states = self._maybe_add_hidden_state(
[], self.start_layer, hidden_states, residual
)
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
aux_hidden_states: list[torch.Tensor] = []
if get_pp_group().is_first_rank:
self._maybe_add_hidden_state(
aux_hidden_states, self.start_layer, hidden_states, residual
)
for layer_idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer),
start=self.start_layer,
@@ -569,13 +574,18 @@ class SarvamMLAModel(nn.Module, EagleModelMixin):
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
{
"hidden_states": hidden_states,
"residual": residual,
**self.pack_local_aux_hidden_states(aux_hidden_states),
}
)
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
aux_hidden_states = remote_aux + aux_hidden_states
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
+7 -5
View File
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.attention import (
MMEncoderAttention,
)
from vllm.model_executor.layers.conv import Conv2dLayer
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -467,13 +468,13 @@ class SiglipEncoderLayer(nn.Module):
prefix=f"{prefix}.self_attn",
attn_cls=attn_cls,
)
self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
self.layer_norm1 = LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
self.mlp = SiglipMLP(
config,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
self.layer_norm2 = LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
def forward(
self,
@@ -573,7 +574,7 @@ class SiglipTextTransformer(nn.Module):
attn_cls=EncoderOnlyAttention,
)
self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
self.final_layer_norm = LayerNorm(embed_dim, eps=config.layer_norm_eps)
self.head = nn.Linear(embed_dim, config.projection_size)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -616,7 +617,7 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
self.attention = torch.nn.MultiheadAttention(
config.hidden_size, config.num_attention_heads, batch_first=True
)
self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
self.layernorm = LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
self.mlp = SiglipMLP(
config=config,
quant_config=quant_config,
@@ -685,8 +686,9 @@ class SiglipVisionTransformer(nn.Module):
if require_post_norm is None:
require_post_norm = len(self.encoder.layers) == num_hidden_layers
self.post_layernorm: LayerNorm | None
if require_post_norm:
self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
self.post_layernorm = LayerNorm(embed_dim, eps=config.layer_norm_eps)
else:
self.post_layernorm = None
+16 -1
View File
@@ -22,7 +22,11 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .step3p5 import Step3p5DecoderLayer
from .utils import get_spec_layer_idx_from_weight_name, maybe_prefix
from .utils import (
get_draft_quant_config,
get_spec_layer_idx_from_weight_name,
maybe_prefix,
)
logger = init_logger(__name__)
@@ -152,8 +156,19 @@ class Step3p5AMultiTokenPredictor(nn.Module):
class Step3p5MTP(nn.Module):
# Each MTP layer's shared_head carries its own LM head weights.
has_own_lm_head = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
# The target config may be a multimodal wrapper (Step3.7); build the
# draft from its own text and quant configs.
assert vllm_config.speculative_config is not None
draft_quant_config = get_draft_quant_config(vllm_config)
vllm_config = vllm_config.with_hf_config(
vllm_config.speculative_config.draft_model_config.hf_config
)
vllm_config.quant_config = draft_quant_config
self.config = vllm_config.model_config.hf_config
self.vllm_config = vllm_config
self.model = Step3p5AMultiTokenPredictor(
@@ -71,6 +71,7 @@ from vllm.model_executor.models.transformers.utils import (
named_state,
replace_conv_class,
replace_embedding_class,
replace_layernorm_class,
replace_linear_class,
)
from vllm.model_executor.models.utils import (
@@ -466,6 +467,7 @@ class Base(
- `nn.Conv2d` / `nn.Conv3d` with vLLM's `Conv2d` / `Conv3d`
- Vocab `nn.Embedding`s with vLLM's `VocabParallelEmbedding`
- RMSNorm (detected from their dataflow) with vLLM's `RMSNorm`or `GemmaRMSNorm`
- `nn.LayerNorm` with vLLM's `LayerNorm`
"""
tp_plan = self.model.tp_plan or {}
@@ -556,6 +558,8 @@ class Base(
new_module = replace_embedding_class(
child_module, self.quant_config, prefix=qual_name
)
elif isinstance(child_module, nn.LayerNorm):
new_module = replace_layernorm_class(child_module)
elif child_module_fusers := fusers[child_module]:
for fuser in child_module_fusers:
register_fusion(fuser, qual_name, child_module)
@@ -28,6 +28,7 @@ from torch import nn
from vllm.logger import init_logger
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
ReplicatedLinear,
@@ -187,6 +188,34 @@ def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv:
)
def replace_layernorm_class(layernorm: nn.LayerNorm) -> nn.Module:
"""Replace a standard (mean-centered) `nn.LayerNorm` with vLLM's
`LayerNorm`.
Args:
layernorm: `nn.LayerNorm` to be replaced.
Returns:
The new `LayerNorm`. If the layernorm is not supported (a subclass with
its own behavior, multi-dim `normalized_shape`, or not both
elementwise-affine and biased), returns the original module unchanged.
"""
if (
type(layernorm) is not nn.LayerNorm
or len(layernorm.normalized_shape) != 1
or not layernorm.elementwise_affine
or layernorm.bias is None
):
return layernorm
return LayerNorm(
layernorm.normalized_shape[0],
eps=layernorm.eps,
dtype=layernorm.weight.dtype,
)
def attrsetter(attr: str) -> Callable[[object, object], None]:
"""Set a possibly nested attribute, like the inverse of attrgetter."""
parent, _, name = attr.rpartition(".")
+19 -4
View File
@@ -321,6 +321,14 @@ class AutoWeightsLoader:
f"into a single parameter {base_prefix!r}"
)
if param.dtype != weight_data.dtype:
logger.warning(
"Attempted to load weight %s with dtype %s into "
"parameter with dtype %s",
weight_qualname,
weight_data.dtype,
param.dtype,
)
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, weight_data)
@@ -844,17 +852,24 @@ class PPMissingLayer(torch.nn.Identity):
return args[0] if args else next(iter(kwargs.values()))
def spec_decode_needs_target_embed(vllm_config: VllmConfig) -> bool:
"""Whether the last PP rank needs the target input embedding."""
def spec_decode_needs_target_embed(
vllm_config: VllmConfig, *, include_mtp: bool = False
) -> bool:
"""Whether the last PP rank needs the target input embedding.
MTP targets opt in when their drafter supports sharing this embedding.
"""
from vllm.distributed.parallel_state import get_pp_group
speculative_config = vllm_config.speculative_config
if speculative_config is None or speculative_config.method not in (
if speculative_config is None:
return False
if speculative_config.method not in (
"eagle",
"eagle3",
"dflash",
"dspark",
):
) and not (include_mtp and speculative_config.method == "mtp"):
return False
pp = get_pp_group()
return pp.world_size > 1 and pp.is_last_rank
+52 -12
View File
@@ -63,6 +63,7 @@ from vllm.multimodal.processing import (
)
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.renderers import TokenizeParams
from vllm.tokenizers import TokenizerLike
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.jsontree import json_map_leaves
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -325,7 +326,7 @@ class WhisperCrossAttention(WhisperAttention):
prefix=f"{prefix}.kv_proj",
)
def forward(
def forward( # type: ignore[override]
self,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor | None,
@@ -872,7 +873,7 @@ class WhisperForConditionalGeneration(
@classmethod
def get_language_token_ids(
cls,
tokenizer: object,
tokenizer: TokenizerLike,
) -> list[int]:
"""Return token IDs for all supported language tokens.
@@ -908,8 +909,8 @@ class WhisperForConditionalGeneration(
def parse_language_detection_output(
cls,
token_ids: list[int],
tokenizer: object,
) -> str | None:
tokenizer: TokenizerLike,
) -> str:
"""Parse the language token predicted by Whisper.
Decodes the first token ID and extracts the language code from the
@@ -1004,6 +1005,7 @@ class WhisperForConditionalGeneration(
audio_input = self._parse_and_validate_audio_input(**kwargs)
# Split concatenated encoder outputs into one tensor per audio input
enc_output = self.model.get_encoder_outputs(audio_input["input_features"])
assert enc_output is not None
# The assumption is we can only process whole mm items (audios)
return enc_output.unbind(dim=0)
@@ -1021,8 +1023,12 @@ class WhisperForConditionalGeneration(
def _parse_and_validate_audio_input(self, **kwargs: object) -> WhisperAudioInputs:
input_features = kwargs.pop("input_features", None)
def to_dtype(value: object) -> torch.Tensor:
assert isinstance(value, torch.Tensor)
return value.to(self.dtype)
if input_features is not None:
input_features = json_map_leaves(lambda x: x.to(self.dtype), input_features)
input_features = json_map_leaves(to_dtype, input_features)
return WhisperAudioInputs(input_features=input_features)
@@ -1033,20 +1039,54 @@ class WhisperForConditionalGeneration(
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
# add fake zeros bias for k_proj to state_dict
weights = _create_fake_bias_for_k_proj(weights, ".k_proj.weight")
# add fake zeros bias for k_proj to state_dict. Covers the
# ``.k_proj.weight`` and the compressed-tensors packed
# ``.k_proj.weight_packed`` checkpoints.
k_proj_weight_suffixes = (".k_proj.weight", ".k_proj.weight_packed")
for suffix in k_proj_weight_suffixes:
weights = _create_fake_bias_for_k_proj(
weights, suffix, out_features=self.config.d_model
)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def _create_fake_bias_for_k_proj(
weights: Iterable[tuple[str, torch.Tensor]], fake_bias_key_name: str
weights: Iterable[tuple[str, torch.Tensor]],
fake_bias_key_name: str,
out_features: int | None = None,
) -> Iterable[tuple[str, torch.Tensor]]:
"""Create full zeros bias for k_proj weight in self-attn and x-attn layers.
So that the bias for k_proj in qkv_proj can be initialized with zeros.
So that the bias for k_proj in qkv_proj or kv_proj can be initialized with
zeros.
If the checkpoint already provides a real ``.bias`` entry for the given
weight, it is forwarded as-is and no fake bias is injected for that layer.
"""
# Map the weight-name suffix to the corresponding bias-name suffix, e.g.
# ".k_proj.weight_packed" / ".k_proj.weight" -> ".k_proj.bias",
# ".wk.weight" -> ".wk.bias". ``.weight_packed`` is stripped first so the
# longer suffix wins.
bias_key_name = (
fake_bias_key_name.removesuffix(".weight_packed").removesuffix(".weight")
+ ".bias"
)
real_bias_names: set[str] = set()
pending: dict[str, torch.Tensor] = {}
for name, weight in weights:
yield name, weight
if name.endswith(bias_key_name):
real_bias_names.add(name)
pending.pop(name, None)
continue
if name.endswith(fake_bias_key_name):
bias = torch.zeros(weight.size(0))
bias_name = name.replace("weight", "bias")
yield bias_name, bias
bias_name = name[: -len(fake_bias_key_name)] + bias_key_name
if bias_name not in real_bias_names:
pending[bias_name] = torch.zeros(
out_features if out_features is not None else weight.size(0)
)
for bias_name, bias in pending.items():
yield bias_name, bias
+16 -8
View File
@@ -6,7 +6,7 @@ import math
from dataclasses import replace
from fractions import Fraction
from functools import partial
from typing import ClassVar
from typing import ClassVar, Protocol
import torch
import torch.nn.functional as F
@@ -35,11 +35,6 @@ from vllm.v1.attention.backend import (
)
from vllm.v1.attention.backends.cpu_attn import CPUAttentionBackend
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
try:
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
except ImportError:
AiterFlashAttentionBackend = None
from vllm.v1.attention.backends.rocm_attn import RocmAttentionBackend
from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend
from vllm.v1.attention.selector import get_attn_backend
@@ -47,11 +42,24 @@ from vllm.v1.kv_cache_interface import AttentionSpec, SlidingWindowSpec
from .utils import make_layers
AiterFlashAttentionBackend: type[AttentionBackend] | None = None
try:
from vllm.v1.attention.backends import rocm_aiter_fa
AiterFlashAttentionBackend = rocm_aiter_fa.AiterFlashAttentionBackend
except ImportError:
pass
logger = logging.getLogger(__name__)
CausalRMSNorm = partial(RMSNorm, eps=1e-5)
class _BlockPoolingAttentionMetadata(Protocol):
@property
def slot_mapping(self) -> torch.Tensor: ...
def _pad1d(
x: torch.Tensor,
paddings: tuple[int, int],
@@ -115,7 +123,7 @@ class WhisperCausalConv1d(nn.Conv1d):
@functools.lru_cache
def create_whisper_attention_backend_with_block_pooling(
underlying_attn_backend: AttentionBackend,
underlying_attn_backend: type[AttentionBackend],
block_pool_size: int,
sliding_window: int | None = None,
) -> type[AttentionBackend]:
@@ -270,7 +278,7 @@ def create_whisper_attention_backend_with_block_pooling(
key: torch.Tensor,
value: torch.Tensor,
kv_cache: torch.Tensor,
attn_metadata: AttentionMetadata,
attn_metadata: _BlockPoolingAttentionMetadata | None,
output: torch.Tensor,
output_scale: torch.Tensor | None = None,
output_block_scale: torch.Tensor | None = None,
+15 -10
View File
@@ -86,16 +86,21 @@ class Zamba2LoRA(nn.Module):
)
if isinstance(output_dim, list):
B_class = MergedColumnParallelLinear
self.B: ColumnParallelLinear = MergedColumnParallelLinear(
rank,
output_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.B",
)
else:
B_class = ColumnParallelLinear
self.B = B_class(
rank,
output_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.B",
)
self.B = ColumnParallelLinear(
rank,
output_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.B",
)
def forward(
self,
@@ -304,7 +309,7 @@ class Zamba2MLP(nn.Module):
self,
config: Zamba2Config,
bare_block_idx: int,
num_hybrid_layers: dict[int, int],
num_hybrid_layers: int,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None: