Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 17:41:13 +00:00
1122 changed files with 60114 additions and 20956 deletions
+2 -1
View File
@@ -29,6 +29,7 @@ from itertools import islice
import torch
from torch import nn
from transformers import AXK1Config
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
@@ -78,7 +79,6 @@ from vllm.model_executor.models.deepseek_v2 import (
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.AXK1 import AXK1Config
from .interfaces import MixtureOfExperts, SupportsEagle, SupportsLoRA, SupportsPP
from .utils import (
@@ -199,6 +199,7 @@ class AXK1MoE(nn.Module):
if self.is_fused_shared_expert_enabled
else None,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
shared_expert_prefix=f"{prefix}.shared_experts",
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+10 -10
View File
@@ -7,6 +7,7 @@ from collections.abc import Iterable
import torch
import torch.nn as nn
from transformers import Aimv2VisionConfig
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.distributed.utils import divide
@@ -21,20 +22,19 @@ from vllm.model_executor.layers.linear import (
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
from vllm.transformers_utils.configs.ovis import AIMv2Config
class AIMv2SwiGLUFFN(nn.Module):
def __init__(
self,
config: AIMv2Config,
config: Aimv2VisionConfig,
quant_config: QuantizationConfig | None,
prefix: str,
):
super().__init__()
hidden_features = config.intermediate_size
in_features = config.hidden_size
bias = config.use_bias
bias = config.mlp_bias
self.fc13 = MergedColumnParallelLinear(
in_features,
@@ -60,7 +60,7 @@ class AIMv2SwiGLUFFN(nn.Module):
class AIMv2PatchEmbed(nn.Module):
def __init__(self, config: AIMv2Config):
def __init__(self, config: Aimv2VisionConfig):
super().__init__()
self.proj = Conv2dLayer(
config.num_channels,
@@ -77,7 +77,7 @@ class AIMv2PatchEmbed(nn.Module):
class AIMv2ViTPreprocessor(nn.Module):
def __init__(self, config: AIMv2Config):
def __init__(self, config: Aimv2VisionConfig):
super().__init__()
num_patches = (config.image_size // config.patch_size) ** 2
@@ -95,7 +95,7 @@ class AIMv2ViTPreprocessor(nn.Module):
class AIMv2Attention(nn.Module):
def __init__(
self,
config: AIMv2Config,
config: Aimv2VisionConfig,
quant_config: QuantizationConfig | None,
prefix: str,
):
@@ -124,7 +124,7 @@ class AIMv2Attention(nn.Module):
self.proj = RowParallelLinear(
input_size=self.embed_dim,
output_size=self.embed_dim,
bias=config.use_bias,
bias=config.mlp_bias,
quant_config=quant_config,
prefix=f"{prefix}.proj",
)
@@ -151,7 +151,7 @@ class AIMv2Attention(nn.Module):
class AIMv2Block(nn.Module):
def __init__(
self,
config: AIMv2Config,
config: Aimv2VisionConfig,
quant_config: QuantizationConfig | None,
prefix: str,
):
@@ -174,7 +174,7 @@ class AIMv2Block(nn.Module):
class AIMv2Transformer(nn.Module):
def __init__(
self,
config: AIMv2Config,
config: Aimv2VisionConfig,
quant_config: QuantizationConfig | None,
*,
require_post_norm: bool | None = None,
@@ -214,7 +214,7 @@ class AIMv2Model(torch.nn.Module):
def __init__(
self,
config: AIMv2Config,
config: Aimv2VisionConfig,
quant_config: QuantizationConfig | None,
*,
require_post_norm: bool | None = None,
@@ -26,6 +26,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import MultiModalDataItems
@@ -520,8 +521,12 @@ class BailingMoeV3VLForConditionalGeneration(
tower_model="visual.",
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
return num_image_tokens * self.visual.spatial_merge_size**2
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
return num_vision_tokens // self.visual.spatial_merge_size**2
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
return num_mm_embeds * self.visual.spatial_merge_size**2, num_mm_embeds
-14
View File
@@ -403,20 +403,6 @@ class Cohere2MoeModel(nn.Module, EagleModelMixin):
config.vocab_size, config.hidden_size
)
# Decoder layers read per-layer MLP layout from config.mlp_layer_types
# (dense MLP vs MoE) and use it for weight loading. Transformers >=5.10
# populates this field; older versions only expose first_k_dense_replace.
# Normalize here so layer construction below sees a consistent layout.
if getattr(config, "mlp_layer_types", None) is None:
first_k_dense_replace = getattr(config, "first_k_dense_replace", None)
n = config.num_hidden_layers
if first_k_dense_replace is not None:
config.mlp_layer_types = ["dense"] * first_k_dense_replace + [
"sparse"
] * (n - first_k_dense_replace)
else:
config.mlp_layer_types = ["sparse"] * n
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: Cohere2MoeDecoderLayer(
+10 -17
View File
@@ -82,6 +82,7 @@ from vllm.multimodal.inputs import (
ImageItem,
MultiModalFeatureSpec,
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import (
@@ -2085,24 +2086,16 @@ class CohereCompassForConditionalGeneration(
tower_model="visual.",
)
def get_num_mm_encoder_tokens(
def get_mm_lora_token_counts(
self,
num_image_tokens: int,
) -> int:
hf_config = self.config
vision_config = hf_config.vision_config
merge_size = vision_config.spatial_merge_size
return num_image_tokens * merge_size**2
def get_num_mm_connector_tokens(
self,
num_vision_tokens: int,
) -> int:
hf_config = self.config
vision_config = hf_config.vision_config
merge_size = vision_config.spatial_merge_size
return num_vision_tokens // merge_size**2
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
merge_size = self.config.vision_config.spatial_merge_size
return num_mm_embeds * merge_size**2, num_mm_embeds
@lru_cache
+46
View File
@@ -43,6 +43,26 @@ class DeepseekV32ForCausalLM(VerifyAndUpdateConfig):
class GlmMoeDsaForCausalLM(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
from vllm.platforms import current_platform
cache_config = vllm_config.cache_config
if (
cache_config.cache_dtype == "auto"
and current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
):
cache_config.cache_dtype = "fp8_e4m3"
logger.info_once("Using fp8 kv-cache for GlmMoeDsaForCausalLM on SM10x")
# MTP shares the target's KV cache layout; a separate draft model
# must not inherit a default it was never validated with.
spec_config = vllm_config.speculative_config
if (
spec_config is not None
and spec_config.method != "mtp"
and spec_config.kv_cache_dtype is None
):
spec_config.kv_cache_dtype = "auto"
# For Glm-Moe-DSA, qrep + a2a is better than the default all-gather + ag-rs
# in most cases.
vllm_config.parallel_config.set_dcp_defaults(
@@ -270,6 +290,31 @@ class Gemma4Config(VerifyAndUpdateConfig):
)
class EmbeddingGemma2ModelConfig(Gemma4Config):
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
Gemma4Config.verify_and_update_config(vllm_config)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
attn_config = vllm_config.attention_config
if attn_config.backend is None:
attn_config.backend = AttentionBackendEnum.TRITON_ATTN
logger.info(
"EmbeddingGemma2: attention backend not specified; defaulting to "
"TRITON_ATTN (validated backend for heterogeneous head dimensions)."
)
else:
logger.info(
"EmbeddingGemma2: using attention backend %s", attn_config.backend
)
model_config = vllm_config.model_config
if model_config.max_model_len > 8192 and getattr(
model_config, "original_max_model_len", None
) in (None, -1):
model_config.max_model_len = 8192
class DiffusionGemmaModelForBlockDiffusionConfig(VerifyAndUpdateConfig):
@classmethod
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
@@ -1013,6 +1058,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"DeepseekV41ForCausalLM": DeepseekV4ForCausalLMConfig,
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
"DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501
"EmbeddingGemma2Model": EmbeddingGemma2ModelConfig,
"Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501
"FalconMambaForCausalLM": MambaModelConfig,
"Gemma3TextModel": Gemma3TextModelConfig,
+18 -1
View File
@@ -2,11 +2,28 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import regex
from transformers import Cosmos3OmniConfig
from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration
from vllm.model_executor.models.qwen3_vl import (
Qwen3VLDummyInputsBuilder,
Qwen3VLForConditionalGeneration,
Qwen3VLMultiModalProcessor,
Qwen3VLProcessingInfo,
)
from vllm.model_executor.models.utils import WeightsMapper
from vllm.multimodal import MULTIMODAL_REGISTRY
class Cosmos3ProcessingInfo(Qwen3VLProcessingInfo):
def get_hf_config(self) -> Cosmos3OmniConfig:
return self.ctx.get_hf_config(Cosmos3OmniConfig)
@MULTIMODAL_REGISTRY.register_processor(
Qwen3VLMultiModalProcessor,
info=Cosmos3ProcessingInfo,
dummy_inputs=Qwen3VLDummyInputsBuilder,
)
class Cosmos3ForConditionalGeneration(Qwen3VLForConditionalGeneration):
# Cosmos3 unified checkpoints store a Qwen3-VL understanding tower
# alongside a generation tower in a flat key layout. This mapper drops
+1 -2
View File
@@ -5,7 +5,7 @@ from collections.abc import Iterable
import torch
import torch.nn as nn
from transformers import ProcessorMixin
from transformers import NemotronHConfig, ProcessorMixin
from vllm.config import VllmConfig
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -20,7 +20,6 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
from vllm.utils.torch_utils import async_tensor_h2d
from .interfaces import (
@@ -396,6 +396,7 @@ class DeepseekV2MoE(nn.Module):
if self.is_fused_shared_expert_enabled
else None,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
shared_expert_prefix=f"{prefix}.shared_experts",
router_logits_dtype=self.gate.out_dtype,
)
+13 -25
View File
@@ -16,7 +16,7 @@ via Gemma4MultimodalEmbedder.
from __future__ import annotations
from collections.abc import Iterable, Mapping, Sequence
from collections.abc import Iterable, Sequence
from types import SimpleNamespace
from typing import Any
@@ -25,7 +25,7 @@ import torch
import torch._dynamo
from torch import nn
from torch.nn import functional as F
from transformers import AutoModel
from transformers import AutoModel, DiffusionGemmaConfig
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
@@ -113,26 +113,13 @@ class DiffusionGemmaSelfConditioning(nn.Module):
class DiffusionGemmaProcessingInfo(Gemma4ProcessingInfo):
"""Processing info for DiffusionGemma.
"""Processing info for DiffusionGemma, which has no audio tower."""
Overrides ``get_hf_config`` to accept ``DiffusionGemmaConfig``
(which inherits from ``PreTrainedConfig``, not ``Gemma4Config``).
Supports image and video modalities.
"""
def get_hf_config(self) -> DiffusionGemmaConfig:
return self.ctx.get_hf_config(DiffusionGemmaConfig)
def get_hf_config(self):
# DiffusionGemmaConfig doesn't inherit from Gemma4Config, so we
# accept any PreTrainedConfig here.
return self.ctx.get_hf_config()
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
# DiffusionGemma supports image and video inputs.
return {"image": None, "video": None}
def get_mm_max_tokens_per_item(
self, seq_len: int, mm_counts: Mapping[str, int]
) -> Mapping[str, int] | None:
return super().get_mm_max_tokens_per_item(seq_len, mm_counts)
def has_audio(self) -> bool:
return False
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
@@ -445,11 +432,6 @@ def _mask_rows_to_allowed(
return logits if out is None else out
# Tiles specialize the graph on batch size 1, on a tile as wide as the canvas,
# on compute_sc and on sizes that coincide with the state buffers. That set
# is small but passes Dynamo's default of 8, after which every step would run
# eager: about twice as slow for a self-conditioned step.
@torch._dynamo.config.patch(recompile_limit=64)
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
def _denoise_temperature(
step_tensor: torch.Tensor,
@@ -464,6 +446,12 @@ def _denoise_temperature(
return t_min + (t_max - t_min) * (remaining / max_denoising_steps)
# Tiles specialize the graph on batch size 1, on a tile as wide as the canvas,
# on compute_sc and on sizes that coincide with the state buffers. That set
# is small but passes Dynamo's default of 8, after which every step would run
# eager: about twice as slow for a self-conditioned step.
@torch._dynamo.config.patch(recompile_limit=64)
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
def _compiled_sample_step(
# Per-position statistics of the temperature-scaled logits, from
# sample_row_stats: [num_decode, CL] each, and the softmax
File diff suppressed because it is too large Load Diff
+1
View File
@@ -569,6 +569,7 @@ class FalconH1ForCausalLM(
head_dim=hf_config.mamba_d_head,
state_size=hf_config.mamba_d_state,
conv_kernel=hf_config.mamba_d_conv,
num_spec=vllm_config.num_speculative_tokens,
)
@classmethod
+1 -14
View File
@@ -154,20 +154,7 @@ class Gemma3Attention(nn.Module):
self.is_sliding = layer_type == "sliding_attention"
sliding_window = config.sliding_window if self.is_sliding else None
# Initialize the rotary embedding.
if layer_type in config.rope_parameters:
# Transformers v5 rope config.
rope_parameters = config.rope_parameters[layer_type]
else:
# Transformers v4 rope config.
# Global attention. Use the values in config.json.
rope_parameters = config.rope_parameters
# Local attention. Override the values in config.json.
if self.is_sliding:
rope_parameters = dict(
rope_type="default", rope_theta=config.rope_local_base_freq
)
rope_parameters = config.rope_parameters[layer_type]
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position_embeddings,
+1 -11
View File
@@ -350,17 +350,7 @@ class Gemma3nAttention(nn.Module):
is_sliding = layer_type == "sliding_attention"
self.sliding_window = config.sliding_window if is_sliding else None
# Initialize the rotary embedding.
if layer_type in config.rope_parameters:
# Transformers v5 rope config.
rope_parameters = config.rope_parameters[layer_type]
else:
# Transformers v4 rope config.
# Global attention. Use the values in config.json.
rope_parameters = config.rope_parameters.copy()
# Local attention. Override the values in config.json.
if is_sliding:
rope_parameters["rope_theta"] = config.rope_local_base_freq
rope_parameters = config.rope_parameters[layer_type]
kv_sharing_target_layer_name = None
if self.is_kv_shared_layer:
+2 -5
View File
@@ -59,7 +59,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import KVSharingFastPrefillMetadata
@@ -576,7 +575,7 @@ class Gemma4DecoderLayer(nn.Module):
self.layer_idx = layer_idx
# Gemma4 uses different head dimensions for sliding vs full attention
layer_config = gemma4_layer_config(config, layer_idx)
layer_config = config.per_layer_config[layer_idx]
head_dim = layer_config.head_dim
num_kv_heads = layer_config.num_key_value_heads
@@ -628,9 +627,7 @@ class Gemma4DecoderLayer(nn.Module):
)
# MoE (Mixture of Experts) — router + expert block parallel to MLP
self.enable_moe_block = getattr(config, "enable_moe_block", False) or getattr(
config, "use_second_mlp_block", False
)
self.enable_moe_block = getattr(config, "num_experts", None) is not None
self.router: Gemma4Router | None
self.experts: MoERunner | None
self.post_feedforward_layernorm_1: RMSNorm | None
+1 -2
View File
@@ -20,7 +20,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
from .gemma4_mtp import Gemma4MTPAttention, Gemma4MTPDecoderLayer
from .qwen3_dflash import DFlashQwen3Model, _dflash_layer_causal
@@ -48,7 +47,7 @@ class Gemma4DSparkAttention(Gemma4MTPAttention):
config, "attention_k_eq_v", False
)
layer_config = gemma4_layer_config(config, layer_idx)
layer_config = config.per_layer_config[layer_idx]
head_dim = layer_config.head_dim
num_kv_heads = layer_config.num_key_value_heads
super().__init__(
+7 -9
View File
@@ -222,12 +222,11 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
**kwargs,
)
def has_audio(self) -> bool:
return self.get_hf_config().audio_config is not None
def validate_num_items(self, modality: str, num_items: int) -> None:
if (
modality == "audio"
and num_items > 0
and self.get_hf_config().audio_config is None
):
if modality == "audio" and num_items > 0 and not self.has_audio():
model_config = self.ctx.model_config
model = get_served_model_name(
model_config.model, model_config.served_model_name
@@ -243,7 +242,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
limits: dict[str, int | None] = {"image": None}
if self.get_hf_config().audio_config is not None:
if self.has_audio():
limits["audio"] = None
limits["video"] = None
return limits
@@ -261,7 +260,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
if isinstance(val, int) and val in _SUPPORTED_SOFT_TOKENS:
tokens_per_image = val
tokens: dict[str, int] = {"image": tokens_per_image}
if config.audio_config is not None:
if self.has_audio():
# Audio max tokens from the processor's audio_seq_length.
processor = self.get_hf_processor()
tokens["audio"] = processor.audio_seq_length
@@ -275,9 +274,8 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
return tokens
def get_data_parser(self) -> MultiModalDataParser:
config = self.get_hf_config()
kwargs: dict[str, Any] = {"video_needs_metadata": True}
if getattr(config, "audio_config", None) is not None:
if self.has_audio():
processor = self.get_hf_processor()
kwargs["target_sr"] = processor.feature_extractor.sampling_rate
return MultiModalDataParser(**kwargs)
+1 -2
View File
@@ -45,7 +45,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
from .gemma4 import Gemma4MLP, _get_text_config
from .utils import (
@@ -265,7 +264,7 @@ class Gemma4MTPDecoderLayer(nn.Module):
self.hidden_size = config.hidden_size
layer_idx = extract_layer_index(prefix)
layer_config = gemma4_layer_config(config, layer_idx)
layer_config = config.per_layer_config[layer_idx]
head_dim = layer_config.head_dim
num_kv_heads = layer_config.num_key_value_heads
+1
View File
@@ -204,6 +204,7 @@ class Glm4MoE(nn.Module):
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
),
fuse_shared_experts=self.is_fused_shared_expert_enabled,
shared_expert_prefix=f"{prefix}.shared_experts",
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+42 -54
View File
@@ -493,13 +493,9 @@ class Granite4VisionForConditionalGeneration(
prefix=maybe_prefix(prefix, "vision_tower"),
)
# image_newline parameter
if config.use_image_newline_parameter:
self.image_newline = nn.Parameter(
torch.empty(config.text_config.hidden_size)
)
else:
self.image_newline = None
self.image_newline = nn.Parameter(
torch.empty(config.text_config.hidden_size)
)
cache_config = vllm_config.cache_config
@@ -517,20 +513,18 @@ class Granite4VisionForConditionalGeneration(
)
# Spatial projectors: 4 offset groups
self.spatial_projectors = None
if config.use_spatial_sampling:
self.spatial_projectors = nn.ModuleList(
[
WindowQFormerDownsampler(
config,
quant_config=quant_config,
cache_config=cache_config,
spatial_offset=i,
prefix=maybe_prefix(prefix, f"spatial_projectors.{i}"),
)
for i in range(4)
]
)
self.spatial_projectors = nn.ModuleList(
[
WindowQFormerDownsampler(
config,
quant_config=quant_config,
cache_config=cache_config,
spatial_offset=i,
prefix=maybe_prefix(prefix, f"spatial_projectors.{i}"),
)
for i in range(4)
]
)
# ----- Language model (marked as LM) -----
with self._mark_language_model(vllm_config):
@@ -545,12 +539,9 @@ class Granite4VisionForConditionalGeneration(
# Store config values we need
self._deepstack_layer_map = config.deepstack_layer_map # [[-19, 9], ...]
self._use_spatial_sampling = getattr(config, "use_spatial_sampling", False)
self._spatial_vision_layer = getattr(config, "spatial_vision_layer", -1)
self._spatial_target_layers = getattr(config, "spatial_target_layers", [])
self._vision_feature_select_strategy = getattr(
config, "vision_feature_select_strategy", "full"
)
self._spatial_vision_layer = config.spatial_vision_layer
self._spatial_target_layers = config.spatial_target_layers
self._vision_feature_select_strategy = config.vision_feature_select_strategy
self._downsample_rate = Fraction(config.downsample_rate)
# Ordered list of LLM layer indices for each deepstack level.
@@ -558,7 +549,7 @@ class Granite4VisionForConditionalGeneration(
# (before any embed_multimodal call).
self._ds_layer_indices: list[int] = [
llm_layer for _, llm_layer in config.deepstack_layer_map
] + list(getattr(config, "spatial_target_layers", []))
] + list(config.spatial_target_layers)
# Share ds_layer_indices with the LLM causal model so
# make_empty_intermediate_tensors includes the correct keys
@@ -571,7 +562,7 @@ class Granite4VisionForConditionalGeneration(
# the updated values written just before each prefill.
# Shape: (max_num_batched_tokens, lm_hidden_size) per level.
n_layerwise = len(config.deepstack_layer_map)
n_spatial = len(getattr(config, "spatial_target_layers", []))
n_spatial = len(config.spatial_target_layers)
num_ds_levels = n_layerwise + n_spatial
lm_hidden = config.text_config.hidden_size
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
@@ -644,26 +635,24 @@ class Granite4VisionForConditionalGeneration(
)
image_feature = unpad_image(image_feature, image_sizes[image_idx])
if self.image_newline is not None:
image_feature = torch.cat(
(
image_feature,
self.image_newline[:, None, None]
.expand(*image_feature.shape[:-1], 1)
.to(image_feature.device, image_feature.dtype),
),
dim=-1,
)
image_feature = torch.cat(
(
image_feature,
self.image_newline[:, None, None]
.expand(*image_feature.shape[:-1], 1)
.to(image_feature.device, image_feature.dtype),
),
dim=-1,
)
image_feature = image_feature.flatten(1, 2).transpose(0, 1)
image_feature = torch.cat((base_image_feature, image_feature), dim=0)
else:
image_feature = image_feature[0]
if self.image_newline is not None:
image_feature = torch.cat(
(image_feature, self.image_newline[None].to(image_feature)),
dim=0,
)
image_feature = torch.cat(
(image_feature, self.image_newline[None].to(image_feature)),
dim=0,
)
new_image_features.append(image_feature)
@@ -718,16 +707,15 @@ class Granite4VisionForConditionalGeneration(
)
levels.append((llm_layer, per_image))
if self._use_spatial_sampling and self.spatial_projectors is not None:
spatial_hidden = all_hidden_states[self._spatial_vision_layer]
if select_strategy == "default":
spatial_hidden = spatial_hidden[:, 1:]
for group_idx, llm_layer in enumerate(self._spatial_target_layers):
projected = self.spatial_projectors[group_idx](spatial_hidden)
per_image = self._pack_and_unpad_image_features(
torch.split(projected, image_num_patches, dim=0), image_sizes
)
levels.append((llm_layer, per_image))
spatial_hidden = all_hidden_states[self._spatial_vision_layer]
if select_strategy == "default":
spatial_hidden = spatial_hidden[:, 1:]
for group_idx, llm_layer in enumerate(self._spatial_target_layers):
projected = self.spatial_projectors[group_idx](spatial_hidden)
per_image = self._pack_and_unpad_image_features(
torch.split(projected, image_num_patches, dim=0), image_sizes
)
levels.append((llm_layer, per_image))
llm_layer_indices = [llm_layer for llm_layer, _ in levels]
num_images = len(image_sizes)
+2 -5
View File
@@ -158,9 +158,7 @@ class GraniteSpeechMultiModalProcessor(
feature_extractor = processor.audio_processor
vocab = tokenizer.get_vocab()
# Use getattr with default to be compatible with transformers<4.48
audio_token = getattr(processor, "audio_token", "<|audio|>")
audio_token_id = vocab[audio_token]
audio_token_id = vocab[processor.audio_token]
def get_replacement(item_idx: int):
audios = mm_items.get_items("audio", AudioProcessorItems)
@@ -221,8 +219,7 @@ class GraniteSpeechDummyInputsBuilder(
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_audios = mm_counts.get("audio", 0)
hf_processor = self.info.get_hf_processor()
audio_token = getattr(hf_processor, "audio_token", "<|audio|>")
return audio_token * num_audios
return hf_processor.audio_token * num_audios
### QFormer Projector
@@ -320,10 +320,6 @@ class GraniteMoeHybridAttention(nn.Module):
ALL_DECODER_LAYER_TYPES = {
# Transformers < 5.13.0
"attention": GraniteMoeHybridAttentionDecoderLayer,
"mamba": GraniteMoeHybridMambaDecoderLayer,
# Transformers >= 5.13.0
"full_attention": GraniteMoeHybridAttentionDecoderLayer,
"linear_attention": GraniteMoeHybridMambaDecoderLayer,
}
@@ -647,6 +643,7 @@ class GraniteMoeHybridForCausalLM(
head_dim=hf_config.mamba_d_head,
state_size=hf_config.mamba_d_state,
conv_kernel=hf_config.mamba_d_conv,
num_spec=vllm_config.num_speculative_tokens,
)
@classmethod
+1 -2
View File
@@ -31,7 +31,7 @@ from typing import Any
import torch
from torch import nn
from transformers import PreTrainedConfig
from transformers import HYV3Config, PreTrainedConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
@@ -67,7 +67,6 @@ from vllm.model_executor.model_loader.weight_utils import (
maybe_remap_kv_scale_name,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.hy_v3 import HYV3Config
from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
from .utils import (
+6 -11
View File
@@ -31,6 +31,7 @@ from itertools import islice
import torch
from torch import nn
from transformers import HyperCLOVAXConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
@@ -51,7 +52,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.hyperclovax import HyperCLOVAXConfig
from .interfaces import SupportsLoRA, SupportsPP
from .utils import (
@@ -137,9 +137,7 @@ class HyperCLOVAXAttention(nn.Module):
# the KV heads across multiple tensor parallel GPUs.
assert tp_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
self.head_dim = getattr(
config, "head_dim", self.hidden_size // self.total_num_heads
)
self.head_dim = config.head_dim
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
self.scaling = config.attention_multiplier
@@ -166,7 +164,7 @@ class HyperCLOVAXAttention(nn.Module):
self.head_dim,
max_position=max_position_embeddings,
is_neox_style=True,
rope_parameters=getattr(config, "rope_parameters", None),
rope_parameters=config.rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
)
@@ -217,18 +215,15 @@ class HyperCLOVAXDecoderLayer(nn.Module):
"dual_chunk_attention_config",
None,
)
attention_bias = getattr(config, "attention_bias", False)
self.self_attn = HyperCLOVAXAttention(
config=config,
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
num_kv_heads=getattr(
config, "num_key_value_heads", config.num_attention_heads
),
num_kv_heads=config.num_key_value_heads,
max_position_embeddings=max_position_embeddings,
quant_config=quant_config,
bias=attention_bias,
bias=config.attention_bias,
cache_config=cache_config,
prefix=f"{prefix}.self_attn",
dual_chunk_attention_config=dual_chunk_attention_config,
@@ -238,7 +233,7 @@ class HyperCLOVAXDecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
bias=getattr(config, "mlp_bias", False),
bias=config.mlp_bias,
prefix=f"{prefix}.mlp",
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+5 -18
View File
@@ -21,6 +21,7 @@ from typing import (
ClassVar,
Literal,
Protocol,
Self,
TypeAlias,
overload,
runtime_checkable,
@@ -31,7 +32,7 @@ import torch
import torch.nn as nn
from torch import Tensor
from transformers.models.whisper.tokenization_whisper import LANGUAGES
from typing_extensions import Self, TypeIs, deprecated
from typing_extensions import TypeIs
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -423,17 +424,6 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
yield
@deprecated(
"get_num_mm_encoder_tokens is deprecated; use get_mm_lora_token_counts instead."
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: ...
@deprecated(
"get_num_mm_connector_tokens is deprecated; use "
"get_mm_lora_token_counts instead."
)
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: ...
def get_mm_lora_token_counts(
self,
*,
@@ -447,12 +437,9 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
connector forwards. Models with multiple modalities can override this
when each modality has different encoder padding or pooling behavior.
"""
del modality, mm_kwargs
num_encoder_tokens = self.get_num_mm_encoder_tokens(num_mm_embeds)
num_connector_tokens = self.get_num_mm_connector_tokens(num_encoder_tokens)
return (
num_encoder_tokens,
num_connector_tokens if isinstance(num_connector_tokens, int) else None,
raise NotImplementedError(
f"{type(self).__name__} must implement get_mm_lora_token_counts "
"to support tower/connector LoRA."
)
def _embed_text_input_ids(
+25 -39
View File
@@ -11,12 +11,13 @@ from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Literal
import torch
from torch import nn
from transformers import BatchFeature
from transformers import BatchFeature, Kimi_K25Config
from vllm.config import VllmConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.compressed_tensors import (
compressed_tensors,
@@ -62,7 +63,6 @@ from vllm.multimodal.processing import (
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.kimi_k25 import KimiK25Config
from vllm.transformers_utils.processor import cached_get_image_processor
from vllm.transformers_utils.processors.kimi_k25 import KimiK25Processor
from vllm.transformers_utils.processors.kimi_k25_vision_fused import (
@@ -134,7 +134,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
# Resolve token ID from the tokenizer because transformers v5
# may remap token IDs vs config.json.
config_token_id = hf_config.media_placeholder_token_id
config_token_id = hf_config.image_token_id
resolved_token_id = tokenizer.convert_tokens_to_ids("<|media_pad|>")
unk_token_id = getattr(tokenizer, "unk_token_id", None)
is_valid_resolved = isinstance(resolved_token_id, int) and (
@@ -142,7 +142,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
)
if is_valid_resolved and resolved_token_id != config_token_id:
logger.warning_once(
"Kimi-K2.5 config.media_placeholder_token_id (%d) disagrees "
"Kimi-K2.5 config.image_token_id (%d) disagrees "
"with tokenizer mapping for <|media_pad|> (%d). "
"Using tokenizer value.",
config_token_id,
@@ -150,7 +150,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
)
media_token_id = resolved_token_id
# Patch config so downstream code also sees the correct ID.
hf_config.media_placeholder_token_id = resolved_token_id
hf_config.image_token_id = resolved_token_id
else:
media_token_id = config_token_id
@@ -169,7 +169,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
return self.hf_processor
def get_hf_config(self):
return self.ctx.get_hf_config(KimiK25Config)
return self.ctx.get_hf_config(Kimi_K25Config)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
# None means unlimited
@@ -306,6 +306,7 @@ class KimiK25ForConditionalGeneration(
supports_encoder_tp_data = True
supports_encoder_cudagraph: ClassVar[Literal[True]] = True
supports_mm_device_do_normalize = True
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
@@ -336,7 +337,7 @@ class KimiK25ForConditionalGeneration(
) -> None:
super().__init__()
model_config = vllm_config.model_config
config: KimiK25Config = model_config.hf_config
config: Kimi_K25Config = model_config.hf_config
self.config = config
quant_config = vllm_config.quant_config
@@ -345,11 +346,11 @@ class KimiK25ForConditionalGeneration(
)
self.hidden_size = config.text_config.hidden_size
self.device = current_platform.current_device()
# Build vision tower directly with KimiK25VisionConfig
with self._mark_tower_model(vllm_config, "vision_chunk"):
self.vision_tower = MoonViT3dPretrainedModel(
config.vision_config,
quant_config=self._maybe_ignore_quant_config(quant_config),
input_norm=build_mm_input_norm(vllm_config.model_config),
prefix=maybe_prefix(prefix, "vision_tower"),
)
if self._maybe_ignore_quant_config(quant_config) is not None:
@@ -361,6 +362,7 @@ class KimiK25ForConditionalGeneration(
self.mm_projector = KimiK25MultiModalProjector(
config=config.vision_config,
out_hidden_size=config.text_config.hidden_size,
use_data_parallel=self.use_data_parallel,
quant_config=self._maybe_ignore_quant_config(quant_config),
prefix=maybe_prefix(prefix, "mm_projector"),
@@ -380,7 +382,7 @@ class KimiK25ForConditionalGeneration(
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors
)
self.media_placeholder: int = self.config.media_placeholder_token_id
self.media_placeholder: int = self.config.image_token_id
def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig | None):
if isinstance(quant_config, compressed_tensors.CompressedTensorsConfig):
@@ -404,9 +406,6 @@ class KimiK25ForConditionalGeneration(
pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
)
# The batch dimension of pixel_values has been flattened into shape[0]
target_dtype = next(self.vision_tower.parameters()).dtype
pixel_values = pixel_values.to(target_dtype)
assert isinstance(grid_thws, torch.Tensor), (
f"expect grid_thws to be a tensor, got {type(grid_thws)}"
)
@@ -479,30 +478,16 @@ class KimiK25ForConditionalGeneration(
# Image-only (t == 1). Video chunks (t > 1) fall back to eager. #
# ------------------------------------------------------------------ #
@property
def _encoder_cudagraph_pad_totals(self) -> dict[int, int]:
"""Row count of each captured buffer set, keyed by cu_seqlens ptr."""
totals = self.__dict__.get("_encoder_cg_pad_totals")
if totals is None:
totals = {}
self.__dict__["_encoder_cg_pad_totals"] = totals
return totals
def get_encoder_cudagraph_config(self) -> "EncoderCudaGraphConfig":
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
pad_totals = self._encoder_cudagraph_pad_totals
def pad_cu_seqlens(dst: torch.Tensor, src: torch.Tensor) -> None:
# Varlen attention requires cu_seqlens[-1] to equal the number of
# rows actually passed in. The captured buffers are sized for the
# full token budget, so a smaller real batch has to be completed
# with one trailing padding sequence; declaring fewer rows than the
# buffer holds is undefined behaviour and returns NaN on FlashAttn.
total = pad_totals.get(dst.data_ptr())
# Zero-length padding (same as Qwen): tail slots repeat the last
# real offset, so FlashAttn schedules no work for padding rows.
n = min(src.shape[0], dst.shape[0])
dst[:n].copy_(src[:n])
dst[n:] = total if total is not None else src[-1]
if n < dst.shape[0]:
dst[n:] = src[-1] if n else 0
return EncoderCudaGraphConfig(
modalities=["vision_chunk"],
@@ -623,13 +608,18 @@ class KimiK25ForConditionalGeneration(
ps = (ps, ps)
total_patches = max_batch_size * ho * kh * wo * kw
dummy_pixel_values = torch.zeros(
total_patches, 3, ps[0], ps[1], device=device, dtype=dtype
total_patches,
3,
ps[0],
ps[1],
device=device,
dtype=self.vision_tower.patch_embed.input_norm.input_dtype or dtype,
)
# max_seqlen must cover the worst case: one item consuming the full
# budget, i.e. token_budget * kh * kw patches.
# max_batch_size + 1 leaves a spare cu_seqlens slot so replay can append
# a padding sequence covering rows the real batch does not fill.
# max_batch_size + 1 leaves a spare cu_seqlens slot so replay can
# pad the tail with zero-length sequences.
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
grid_thw_list,
max_batch_size=max_batch_size + 1,
@@ -640,10 +630,6 @@ class KimiK25ForConditionalGeneration(
values: dict[str, torch.Tensor] = {"pixel_values": dummy_pixel_values}
values.update({k: v for k, v in metadata.items() if v is not None})
cu_seqlens = values.get("cu_seqlens")
if cu_seqlens is not None:
self._encoder_cudagraph_pad_totals[cu_seqlens.data_ptr()] = total_patches
return EncoderCudaGraphCaptureInputs(values=values)
def prepare_encoder_cudagraph_replay_buffers(
@@ -658,8 +644,8 @@ class KimiK25ForConditionalGeneration(
grid_thw_list = self._get_grid_thw_list(mm_kwargs)
pixel_values = mm_kwargs["pixel_values"]
# Unpadded: pad_cu_seqlens completes the tail with the padding sequence
# so cu_seqlens[-1] matches the captured buffer's row count.
# Unpadded: pad_cu_seqlens pads the tail with zero-length sequences
# whose entries repeat the real total.
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
grid_thw_list,
max_batch_size=None,
+32 -18
View File
@@ -15,6 +15,7 @@ import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import Kimi_K25VisionConfig
from transformers.activations import GELUActivation
from vllm.distributed import divide, get_tensor_model_parallel_world_size
@@ -22,6 +23,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
from vllm.model_executor.layers.conv import Conv2dLayer
from vllm.model_executor.layers.fusion.mm_input_norm import IdentityInputNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -39,13 +41,21 @@ from vllm.model_executor.models.vision import (
run_dp_sharded_mrope_vision_model,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig
from vllm.transformers_utils.configs.kimi_k3 import KimiK3VisionConfig
from vllm.triton_utils import HAS_TRITON
from vllm.utils.torch_utils import async_tensor_h2d
logger = init_logger(__name__)
def _get_pos_emb_size(
config: Kimi_K25VisionConfig | KimiK3VisionConfig, dim: str
) -> int:
# Legacy checkpoints and Kimi-K3 use `init_pos_emb_*`
legacy = getattr(config, f"init_pos_emb_{dim}", None)
return legacy if legacy is not None else getattr(config, f"pos_emb_{dim}")
def _apply_rope_input_validation(x, freqs_cis):
assert x.ndim == freqs_cis.ndim + 1, (x.shape, freqs_cis.shape)
assert x.shape[:-2] == freqs_cis.shape[:-1], (x.shape, freqs_cis.shape)
@@ -187,6 +197,7 @@ class MoonVision3dPatchEmbed(nn.Module):
pos_emb_type: str = "divided_fixed",
patch_embed_proj_bias: bool = True,
pos_emb_interpolation_mode: str = "bicubic",
input_norm: nn.Module | None = None,
):
super().__init__()
assert isinstance(patch_size, int | Sequence), (
@@ -206,6 +217,7 @@ class MoonVision3dPatchEmbed(nn.Module):
stride=patch_size,
bias=patch_embed_proj_bias,
)
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
if pos_emb_type == "divided_fixed":
self.pos_emb = Learnable2DInterpPosEmbDivided_fixed(
@@ -225,6 +237,7 @@ class MoonVision3dPatchEmbed(nn.Module):
*,
pos_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
x = self.input_norm(x.flatten(1), self.proj.weight.dtype).view_as(x)
# forward_native dispatches this non-overlapping patch projection to GEMM.
x = self.proj.forward_native(x).view(x.size(0), self.proj.out_channels)
if pos_embeds is not None:
@@ -694,35 +707,35 @@ def tpool_patch_merger_packed(
class MoonViT3dPretrainedModel(nn.Module):
"""Main vision tower model.
Uses KimiK25VisionConfig directly from transformers_utils/configs/kimi_k25.py.
"""
"""Main vision tower model."""
def __init__(
self,
config: KimiK25VisionConfig,
config: Kimi_K25VisionConfig | KimiK3VisionConfig,
quant_config: QuantizationConfig | None = None,
input_norm: nn.Module | None = None,
prefix: str = "",
):
super().__init__()
config = deepcopy(config)
self.config = config # Required for run_dp_sharded_mrope_vision_model
self.merge_kernel_size = config.merge_kernel_size
merge_h, merge_w = config.merge_kernel_size
self.merge_kernel_size = (merge_h, merge_w)
self.patch_size = config.patch_size
self.merge_type = config.merge_type
self.merge_type = getattr(config, "merge_type", "sd2_tpool")
self.patch_embed = MoonVision3dPatchEmbed(
out_dim=config.hidden_size,
patch_size=config.patch_size,
pos_emb_height=config.init_pos_emb_height,
pos_emb_width=config.init_pos_emb_width,
pos_emb_time=config.init_pos_emb_time,
pos_emb_type=config.pos_emb_type,
pos_emb_height=_get_pos_emb_size(config, "height"),
pos_emb_width=_get_pos_emb_size(config, "width"),
pos_emb_time=_get_pos_emb_size(config, "time"),
pos_emb_type=getattr(config, "pos_emb_type", "divided_fixed"),
patch_embed_proj_bias=getattr(config, "patch_embed_proj_bias", True),
pos_emb_interpolation_mode=getattr(
config, "pos_emb_interpolation_mode", "bicubic"
),
input_norm=input_norm,
)
self.encoder = MoonViT3dEncoder(
@@ -734,14 +747,14 @@ class MoonViT3dPretrainedModel(nn.Module):
"qkv_hidden_size": getattr(config, "qkv_hidden_size", None),
"mlp_dim": config.intermediate_size,
"activation": get_act_fn(
getattr(config, "activation_func", "gelu_pytorch_tanh")
getattr(config, "activation_func", None) or config.hidden_act
),
"attn_bias": getattr(config, "attn_bias", True),
"norm_type": getattr(config, "norm_type", "layernorm"),
"mlp_type": getattr(config, "mlp_type", "mlp2"),
"linear_bias": getattr(config, "linear_bias", True),
},
video_attn_type=config.video_attn_type,
video_attn_type=getattr(config, "video_attn_type", "spatial_temporal"),
quant_config=quant_config,
prefix=maybe_prefix(prefix, "encoder"),
)
@@ -880,7 +893,8 @@ class KimiK25MultiModalProjector(nn.Module):
def __init__(
self,
config: KimiK25VisionConfig,
config: Kimi_K25VisionConfig | KimiK3VisionConfig,
out_hidden_size: int,
use_data_parallel: bool = False,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
@@ -903,13 +917,13 @@ class KimiK25MultiModalProjector(nn.Module):
)
self.linear_2 = ReplicatedLinear(
self.hidden_size,
getattr(config, "text_hidden_size", config.mm_hidden_size),
out_hidden_size,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.linear_2",
)
self.post_norm = torch.nn.RMSNorm(
getattr(config, "text_hidden_size", config.mm_hidden_size),
out_hidden_size,
eps=config.projector_ln_eps,
)
self.act = GELUActivation()
@@ -925,7 +939,7 @@ class KimiK25MultiModalProjector(nn.Module):
)
self.linear_2 = ReplicatedLinear(
self.hidden_size,
config.mm_hidden_size,
out_hidden_size,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.linear_2",
+1 -1
View File
@@ -5,6 +5,7 @@ from itertools import islice
import torch
import torch.nn as nn
from transformers import Lfm2MoeConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
@@ -40,7 +41,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.lfm2_moe import Lfm2MoeConfig
from .interfaces import (
HasInnerState,
+21 -2
View File
@@ -36,10 +36,15 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
from vllm.model_executor.models.llama4 import Llama4DecoderLayer, Llama4ForCausalLM
from vllm.model_executor.models.utils import extract_layer_index
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
from .interfaces import (
MultiModalEmbeddings,
SupportsMultiModalEmbeddings,
_require_is_multimodal,
)
from .utils import (
AutoWeightsLoader,
WeightsMapper,
_merge_multimodal_embeddings,
maybe_prefix,
process_eagle_weight,
)
@@ -204,7 +209,21 @@ class EagleLlama4ForCausalLM(_EagleLlama4ForCausalLMBase, SupportsMultiModalEmbe
def get_language_model(self) -> torch.nn.Module:
return self.model
embed_input_ids = SupportsMultiModal.embed_input_ids # type: ignore
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
inputs_embeds = self.model.embed_input_ids(input_ids)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
return inputs_embeds
return _merge_multimodal_embeddings(
inputs_embeds=inputs_embeds,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=_require_is_multimodal(is_multimodal),
)
def forward(
self,
+28 -59
View File
@@ -58,13 +58,17 @@ from vllm.model_executor.layers.linear import (
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.int8_utils import block_dequant
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.model_loader.weight_utils import (
LoaderFunction,
composed_weight_loader,
default_weight_loader,
)
from vllm.model_executor.models.deepseek_v2 import DeepseekV2MLAAttention
from vllm.model_executor.utils import set_weight_attrs
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsLoRA, SupportsPP
@@ -80,6 +84,23 @@ from .utils import (
logger = init_logger(__name__)
def _scaled_weight_loader(scale: float) -> LoaderFunction:
return composed_weight_loader(default_weight_loader, lambda x: x * scale)
def scale_mla_lora_norms_on_load(self_attn: nn.Module, config) -> None:
"""Fold LongCat's MLA LoRA scaling into the q/kv_a_layernorm weight loaders."""
for norm, rank, enabled in (
(self_attn.q_a_layernorm, config.q_lora_rank, config.mla_scale_q_lora),
(self_attn.kv_a_layernorm, config.kv_lora_rank, config.mla_scale_kv_lora),
):
if enabled:
scale = (config.hidden_size / rank) ** 0.5
set_weight_attrs(
norm.weight, {"weight_loader": _scaled_weight_loader(scale)}
)
class FlashMLP(nn.Module):
"""Flash MLP layer."""
@@ -265,6 +286,8 @@ class FlashDecoderLayer(nn.Module):
for i in range(2)
]
)
for self_attn in self.self_attn:
scale_mla_lora_norms_on_load(self_attn, config)
self.input_layernorm = nn.ModuleList(
[RMSNorm(config.hidden_size, eps=config.rms_norm_eps) for i in range(2)]
)
@@ -450,6 +473,9 @@ class FlashModel(nn.Module):
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
# The MTP layers are loaded by LongCatFlashMTP.
if name.startswith("mtp."):
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
@@ -461,9 +487,6 @@ class FlashModel(nn.Module):
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
# Skip mtp
if ".mtp." in name:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
@@ -478,9 +501,6 @@ class FlashModel(nn.Module):
continue
is_expert_weight = True
name_mapped = name.replace(weight_name, param_name)
# Skip mtp
if ".mtp." in name_mapped:
continue
if (
name_mapped.endswith(".bias") or name_mapped.endswith("_bias")
) and name not in params_dict:
@@ -515,9 +535,6 @@ class FlashModel(nn.Module):
# Skip loading kv_scale from ckpts towards new design.
if name.endswith(".kv_scale") and name not in params_dict:
continue
# Skip mtp
if ".mtp." in name:
continue
if name is None:
continue
if is_pp_missing_parameter(name, self):
@@ -528,54 +545,6 @@ class FlashModel(nn.Module):
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
for layer_id in range(self.config.num_layers):
for i in range(2):
if isinstance(self.layers[layer_id], PPMissingLayer):
continue
self_attn = self.layers[layer_id].self_attn[i]
if (
self.quant_config is not None
and hasattr(self.quant_config, "weight_block_size")
and self_attn.kv_b_proj.weight.dtype
in (
torch.float8_e4m3fn,
torch.float8_e4m3fnuz,
)
):
weight_block_size = self.quant_config.weight_block_size
if weight_block_size is not None:
assert hasattr(self_attn.kv_b_proj, "weight_scale_inv")
dtype = torch.get_default_dtype()
w = block_dequant(
self_attn.kv_b_proj.weight,
self_attn.kv_b_proj.weight_scale_inv,
weight_block_size,
).to(dtype)
else:
w = self_attn.kv_b_proj.weight
w_kc, w_vc = w.unflatten(
0, (-1, self_attn.qk_nope_head_dim + self_attn.v_head_dim)
).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1)
self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2)
self_attn.w_vc = w_vc.contiguous().transpose(1, 2)
# Guard against compounding on incremental load_weights calls:
# the in-place ``*=`` would otherwise re-apply the MLA LoRA
# scaling to the layernorm weights on each pass.
if self.config.mla_scale_q_lora and not getattr(
self_attn, "_mla_q_lora_scaled", False
):
self_attn.q_a_layernorm.weight.data *= (
self.config.hidden_size / self.config.q_lora_rank
) ** 0.5
self_attn._mla_q_lora_scaled = True
if self.config.mla_scale_kv_lora and not getattr(
self_attn, "_mla_kv_lora_scaled", False
):
self_attn.kv_a_layernorm.weight.data *= (
self.config.hidden_size / self.config.kv_lora_rank
) ** 0.5
self_attn._mla_kv_lora_scaled = True
return loaded_params
@@ -14,7 +14,6 @@ from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.int8_utils import block_dequant
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
@@ -23,6 +22,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .deepseek_v2 import DeepseekV2DecoderLayer
from .longcat_flash import scale_mla_lora_norms_on_load
from .utils import maybe_prefix
@@ -45,6 +45,7 @@ class LongCatMultiTokenPredictorLayer(nn.Module):
prefix="eh_proj",
)
self.mtp_block = DeepseekV2DecoderLayer(vllm_config, prefix)
scale_mla_lora_norms_on_load(self.mtp_block.self_attn, config)
self.final_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def forward(
@@ -263,50 +264,6 @@ class LongCatFlashMTP(nn.Module):
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
spec_layer_id = self.model.mtp_start_layer_idx
self_attn = self.model.layers[str(spec_layer_id)].mtp_block.self_attn
if (
self.quant_config is not None
and hasattr(self.quant_config, "weight_block_size")
and self_attn.kv_b_proj.weight.dtype
in (
torch.float8_e4m3fn,
torch.float8_e4m3fnuz,
)
):
weight_block_size = self.quant_config.weight_block_size
if weight_block_size is not None:
dtype = torch.get_default_dtype()
w = block_dequant(
self_attn.kv_b_proj.weight,
self_attn.kv_b_proj.weight_scale_inv,
weight_block_size,
).to(dtype)
else:
w = self_attn.kv_b_proj.weight
else:
w = self_attn.kv_b_proj.weight
w_kc, w_vc = w.unflatten(
0, (-1, self_attn.qk_nope_head_dim + self_attn.v_head_dim)
).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1)
self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2)
self_attn.w_vc = w_vc.contiguous().transpose(1, 2)
# Guard against compounding on incremental load_weights calls (the
# in-place *= would otherwise double-apply the LoRA scaling).
if self.config.mla_scale_q_lora and not getattr(
self_attn, "_mla_q_lora_scaled", False
):
self_attn.q_a_layernorm.weight.data *= (
self.config.hidden_size / self.config.q_lora_rank
) ** 0.5
self_attn._mla_q_lora_scaled = True
if self.config.mla_scale_kv_lora and not getattr(
self_attn, "_mla_kv_lora_scaled", False
):
self_attn.kv_a_layernorm.weight.data *= (
self.config.hidden_size / self.config.kv_lora_rank
) ** 0.5
self_attn._mla_kv_lora_scaled = True
return loaded_params
def _rewrite_spec_layer_name(
+1 -1
View File
@@ -73,7 +73,7 @@ class MiMoMultiTokenPredictorLayer(nn.Module):
) -> torch.Tensor:
assert inputs_embeds is not None
# masking inputs at position 0, as not needed by MTP
inputs_embeds[positions == 0] = 0
inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
inputs_embeds = self.token_layernorm(inputs_embeds)
previous_hidden_states = self.hidden_layernorm(previous_hidden_states)
+6
View File
@@ -372,6 +372,10 @@ class MiMoV2FlashDecoderLayer(nn.Module):
max_position_embeddings = getattr(config, "max_position_embeddings", 32768)
v_scale = getattr(config, "attention_value_scale", None)
# cache_config.sliding_window holds MiMo's SWA window; global layers
# would otherwise fall back to it in Attention.
cache_config = vllm_config.cache_config
cache_config.sliding_window = None
if self.is_compressed_softmax_layer():
self.self_attn = MiMoV2Attention(
@@ -389,6 +393,7 @@ class MiMoV2FlashDecoderLayer(nn.Module):
layer_id=layer_id,
rope_theta=getattr(config, "swa_rope_theta", rope_theta),
max_position_embeddings=max_position_embeddings,
cache_config=cache_config,
quant_config=quant_config,
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
prefix=f"{prefix}.self_attn",
@@ -406,6 +411,7 @@ class MiMoV2FlashDecoderLayer(nn.Module):
layer_id=layer_id,
rope_theta=rope_theta,
max_position_embeddings=max_position_embeddings,
cache_config=cache_config,
quant_config=quant_config,
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
prefix=f"{prefix}.self_attn",
+6 -1
View File
@@ -24,7 +24,7 @@ import torch
import torch.nn as nn
from transformers import PreTrainedConfig
from vllm.config import VllmConfig
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -64,6 +64,7 @@ class MiMoV2MTPLayer(nn.Module):
self,
config: PreTrainedConfig,
prefix: str,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
@@ -100,6 +101,7 @@ class MiMoV2MTPLayer(nn.Module):
layer_id=0,
rope_theta=swa_rope_theta,
max_position_embeddings=getattr(config, "max_position_embeddings", 32768),
cache_config=cache_config,
quant_config=quant_config,
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
prefix=f"{prefix}.self_attn",
@@ -147,6 +149,7 @@ class _MiMoV2MTPLayers(nn.Module):
self,
config: PreTrainedConfig,
num_mtp_layers: int,
cache_config: CacheConfig | None,
quant_config: QuantizationConfig | None,
prefix: str,
) -> None:
@@ -156,6 +159,7 @@ class _MiMoV2MTPLayers(nn.Module):
str(i): MiMoV2MTPLayer(
config=config,
prefix=f"{prefix}.{i}",
cache_config=cache_config,
quant_config=quant_config,
)
for i in range(num_mtp_layers)
@@ -182,6 +186,7 @@ class MiMoV2MultiTokenPredictor(nn.Module):
self.mtp = _MiMoV2MTPLayers(
config=config,
num_mtp_layers=num_mtp_layers,
cache_config=vllm_config.cache_config,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "mtp.layers"),
)
@@ -23,8 +23,17 @@ from vllm.model_executor.models.deepseek_v2 import (
)
from vllm.model_executor.models.mistral_large_3 import MistralLarge3ForCausalLM
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
from .utils import WeightsMapper, make_empty_intermediate_tensors_factory, maybe_prefix
from .interfaces import (
MultiModalEmbeddings,
SupportsMultiModalEmbeddings,
_require_is_multimodal,
)
from .utils import (
WeightsMapper,
_merge_multimodal_embeddings,
make_empty_intermediate_tensors_factory,
maybe_prefix,
)
logger = init_logger(__name__)
@@ -148,7 +157,21 @@ class EagleMistralLarge3ForCausalLM(
def get_language_model(self) -> torch.nn.Module:
return self.model
embed_input_ids = SupportsMultiModal.embed_input_ids # type: ignore
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
inputs_embeds = self.model.embed_input_ids(input_ids)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
return inputs_embeds
return _merge_multimodal_embeddings(
inputs_embeds=inputs_embeds,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=_require_is_multimodal(is_multimodal),
)
def forward(
self,
+6 -22
View File
@@ -94,28 +94,12 @@ class ModernBertAttention(nn.Module):
prefix=f"{prefix}.Wqkv",
)
if layer_types := getattr(config, "layer_types", None):
# Transformers v5
layer_type = layer_types[layer_id]
rope_parameters = config.rope_parameters[layer_type]
sliding_window: int | None = None
if layer_type == "sliding_attention":
# Treats the local attention boundary as inclusive
sliding_window = config.sliding_window + 1
else:
# Transformers v4
sliding_window = None
if layer_id % config.global_attn_every_n_layers != 0:
# ModernBertConfig does not expose sliding_window
sliding_window = config.local_attention // 2
rope_theta = (
config.local_rope_theta
if config.local_rope_theta is not None
else config.global_rope_theta
)
else:
rope_theta = config.global_rope_theta
rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
layer_type = config.layer_types[layer_id]
rope_parameters = config.rope_parameters[layer_type]
sliding_window: int | None = None
if layer_type == "sliding_attention":
# Treats the local attention boundary as inclusive
sliding_window = config.sliding_window + 1
self.rotary_emb = get_rope(
head_size=self.head_dim,
+1 -1
View File
@@ -29,6 +29,7 @@ from itertools import islice
import torch
from torch import nn
from transformers import NemotronConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
@@ -49,7 +50,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.nemotron import NemotronConfig
from .interfaces import SupportsLoRA, SupportsPP
from .utils import (
+3 -3
View File
@@ -23,6 +23,7 @@ from itertools import islice
import torch
from torch import nn
from transformers import NemotronHConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import (
@@ -87,7 +88,6 @@ from vllm.model_executor.models.utils import (
sequence_parallel_chunk,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
class NemotronHMLP(nn.Module):
@@ -235,7 +235,7 @@ class NemotronHMoE(nn.Module):
intermediate_size=config.moe_intermediate_size,
renormalize=config.norm_topk_prob,
quant_config=quant_config,
ckpt_names=("up_proj", "down_proj", ""),
ckpt_names=("up_proj", "down_proj", None),
use_grouped_topk=True,
num_expert_group=config.n_group,
topk_group=config.topk_group,
@@ -707,7 +707,7 @@ class NemotronHModel(nn.Module, EagleModelMixin):
self,
ckpt_gate_proj_name="up_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="",
ckpt_up_proj_name=None,
num_experts=self._get_max_n_routed_experts(),
num_redundant_experts=getattr(self, "num_redundant_experts", 0),
)
+9 -10
View File
@@ -7,6 +7,7 @@ from collections.abc import Callable, Iterable
import torch
import torch.nn as nn
from transformers import NemotronHConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, VllmConfig
@@ -33,7 +34,6 @@ from vllm.model_executor.models.utils import (
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
from .interfaces import SupportsPP, SupportsQuant
from .nemotron_h import (
@@ -426,6 +426,11 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant):
)
return self.logits_processor(self.lm_head, hidden_states)
def is_unused_checkpoint_weight(self, name: str) -> bool:
"""Only MTP, embedding and LM head weights are loaded."""
name = name.removeprefix("language_model.")
return not name.startswith(("mtp.", "lm_head.")) and "embeddings" not in name
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load MTP weights with proper name remapping."""
stacked_params_mapping = [
@@ -444,7 +449,7 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant):
self,
ckpt_gate_proj_name="up_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="", # Empty - non-gated MoE
ckpt_up_proj_name=None, # non-gated MoE
num_experts=num_experts,
num_redundant_experts=self.num_redundant_experts,
)
@@ -456,17 +461,11 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant):
# MTP weights are nested in "language_model."
# in Multimodal Nemotron-H checkpoints.
name = name.removeprefix("language_model.")
if self.is_unused_checkpoint_weight(name):
continue
is_lm_head_weight = name.startswith("lm_head.")
if is_lm_head_weight:
self.has_own_lm_head = True
# Only process MTP and LM head weights -
# skip all non-MTP and non-LM head weights
if (
not name.startswith("mtp.")
and "embeddings" not in name
and not is_lm_head_weight
):
continue
# Skip rotary embeddings (computed, not loaded)
if "rotary_emb.inv_freq" in name:
continue
@@ -568,6 +568,15 @@ class NemotronParseForConditionalGeneration(nn.Module, SupportsMultiModal):
self.lm_head = ParallelLMHead(
config.decoder.vocab_size, config.decoder.d_model, quant_config=quant_config
)
# Some checkpoints (e.g. compact exports) tie the output head to the
# decoder's input embeddings instead of materializing a separate
# lm_head.weight tensor.
tie_word_embeddings = bool(
getattr(config, "tie_word_embeddings", False)
or getattr(config.decoder, "tie_word_embeddings", False)
)
if tie_word_embeddings:
self.lm_head = self.lm_head.tie_weights(self.decoder.embed_tokens)
self.logits_processor = LogitsProcessor(
self.vocab_size, config.decoder.vocab_size
)
+1 -1
View File
@@ -125,7 +125,7 @@ class VisualTokenizer(torch.nn.Module):
prefix: str = "",
) -> nn.Module:
model_type = config.backbone_config.model_type
if model_type == "aimv2":
if model_type == "aimv2_vision_model":
# No post rms_norm in Ovis2's AIMv2 ViT.
return AIMv2Model(
config=config.backbone_config,
+4 -4
View File
@@ -129,7 +129,7 @@ class ConformerEncoderLayer(nn.Module):
default False.
use_pt_scaled_dot_product_attention: bool, optional
if set to True, use pytorch's scaled dot product attention
implementation in training.
implementation in training. Default: True.
attn_group_sizes: int, optional
the number of groups to use for attention, default 1
(Multi-Head Attention),
@@ -163,7 +163,7 @@ class ConformerEncoderLayer(nn.Module):
attention_glu_type: str = "swish",
activation_checkpointing: str = "",
export: bool = False,
use_pt_scaled_dot_product_attention: bool = False,
use_pt_scaled_dot_product_attention: bool = True,
attn_group_sizes: int = 1,
) -> None:
super().__init__()
@@ -808,7 +808,7 @@ class ConformerEncoder(TransformerEncoderBase):
default 4
use_pt_scaled_dot_product_attention: whether to use pytorch scaled
dot product attention in training.
Default: False
Default: True
nemo_conv_settings: dict, optional
A dictionary of settings for NeMo Subsampling.
default: None
@@ -873,7 +873,7 @@ class ConformerEncoder(TransformerEncoderBase):
activation_checkpointing: str = "",
relative_attention_bias_args: dict[str, Any] | None = None,
time_reduction: int = 4,
use_pt_scaled_dot_product_attention: bool = False,
use_pt_scaled_dot_product_attention: bool = True,
nemo_conv_settings: dict[str, Any] | None = None,
conv2d_extra_padding: Literal["feat", "feat_time", "none", True] = "none",
replication_pad_for_subsample_embedding: bool = False,
+2 -5
View File
@@ -1642,7 +1642,7 @@ class MultiHeadedAttention(nn.Module):
NOTE: this will NOT be used in ONNX decoding due to a lack of
support. In that case, we use the original attention
implementation, which shows no regression.
default: False.
default: True.
n_value: int, optional
if set to values other than -1, use a different dimension for
value. With the default value (i.e. -1), it is backward compatible.
@@ -1666,7 +1666,7 @@ class MultiHeadedAttention(nn.Module):
attention_inner_dim: int = -1,
glu_type: str = "swish",
bias_in_glu: bool = True,
use_pt_scaled_dot_product_attention: bool = False,
use_pt_scaled_dot_product_attention: bool = True,
n_value: int = -1,
group_size: int = 1,
) -> None:
@@ -1690,7 +1690,6 @@ class MultiHeadedAttention(nn.Module):
self.linear_v = nn.Linear(n_value, attention_inner_dim // group_size)
self.linear_out = nn.Linear(attention_inner_dim // group_size, n_value)
self.attn = torch.jit.Attribute(None, Tensor | None)
self.dropout = nn.Dropout(p=dropout_rate)
self.dropout_rate = dropout_rate
self.use_pt_scaled_dot_product_attention = use_pt_scaled_dot_product_attention
@@ -1799,8 +1798,6 @@ class MultiHeadedAttention(nn.Module):
attn = masked_softmax(scores, mask) # (batch, head, time1, time2)
self.attn = attn
p_attn = self.dropout(attn)
x = torch.matmul(p_attn.to(v.dtype), v) # (batch, head, time1, d_k)
if pos_v is not None:
+3 -2
View File
@@ -32,6 +32,7 @@ from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import SiluAndMul, get_act_and_mul_fn
from vllm.model_executor.layers.attention import MMEncoderAttention
from vllm.model_executor.layers.conv import Conv2dLayer
from vllm.model_executor.layers.fusion.fused_act_quant import maybe_fused_act_quant
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -758,7 +759,7 @@ class FeedForward(nn.Module):
def forward(self, x):
x, _ = self.gate_up_proj(x)
x = self.act_fn(x)
x = maybe_fused_act_quant(self.act_fn, x, self.down_proj)
x, _ = self.down_proj(x)
return x
@@ -1259,7 +1260,7 @@ class PixtralHFMLP(nn.Module):
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate_up, _ = self.gate_up_proj(x)
x = self.act_and_mul(gate_up)
x = maybe_fused_act_quant(self.act_and_mul, gate_up, self.down_proj)
x, _ = self.down_proj(x)
return x
+6 -5
View File
@@ -29,6 +29,12 @@ from typing import ClassVar
import torch
from torch import nn
from transformers import (
Qwen3_5Config,
Qwen3_5MoeConfig,
Qwen3_5MoeTextConfig,
Qwen3_5TextConfig,
)
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
@@ -59,11 +65,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig
from vllm.transformers_utils.configs.qwen3_5_moe import (
Qwen3_5MoeConfig,
Qwen3_5MoeTextConfig,
)
from .interfaces import (
HasInnerState,
+10 -7
View File
@@ -8,6 +8,7 @@ from collections.abc import Iterable
import huggingface_hub
import torch
from torch import nn
from transformers import Qwen3_5MoeTextConfig, Qwen3_5TextConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import ModelConfig, VllmConfig
@@ -34,8 +35,6 @@ from vllm.model_executor.models.qwen3_next import (
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5TextConfig
from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig
from vllm.transformers_utils.repo_utils import try_get_local_file
from .interfaces import (
@@ -376,16 +375,20 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP):
) -> torch.Tensor | None:
return self.logits_processor(self.lm_head, hidden_states)
def is_unused_checkpoint_weight(self, name: str) -> bool:
return not name.startswith("mtp.") and not any(
key in name for key in ["embed_tokens", "lm_head"]
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
def remap_weight_names(weights):
for name, weight in weights:
if self.is_unused_checkpoint_weight(name):
continue
if name.startswith("mtp."):
name = name.replace("mtp.", "model.")
elif any(key in name for key in ["embed_tokens", "lm_head"]):
if "embed_tokens" in name:
name = name.replace("language_model.", "")
else:
continue
elif "embed_tokens" in name:
name = name.replace("language_model.", "")
yield name, weight
loader = AutoWeightsLoader(self)
+2
View File
@@ -39,6 +39,7 @@ from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
StreamingTranscriptionPostProcessor,
SupportsEagle3,
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
@@ -345,6 +346,7 @@ class Qwen3ASRForConditionalGeneration(
SupportsMRoPE,
SupportsTranscription,
SupportsLoRA,
SupportsEagle3,
):
# LoRA support
packed_modules_mapping = {
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.multimodal.inputs import NestedTensors
from vllm.platforms import current_platform
from vllm.transformers_utils.config import set_default_rope_theta
from vllm.transformers_utils.repo_utils import get_hf_file_bytes
from vllm.v1.attention.backend import AttentionType
@@ -269,6 +270,24 @@ class DFlashQwen3Attention(nn.Module):
sinks=self.attention_sink_bias,
)
self.causal = causal
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx1x
from vllm.v1.attention.backends.rocm_segmented_attn import (
RocmSegmentedAttentionImpl,
)
if on_gfx1x() and isinstance(self.attn.impl, RocmSegmentedAttentionImpl):
# Startup segmented tuning needs the draft's actual attention mode.
# Causality is normally supplied later through attention metadata.
self.attn.segmented_causal = causal
speculative_config = get_current_vllm_config().speculative_config
if (
speculative_config is not None
and speculative_config.num_speculative_tokens is not None
):
self.attn.segmented_query_limit = (
speculative_config.num_speculative_tokens + 1
)
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
+2 -1
View File
@@ -7,6 +7,7 @@ from itertools import islice
import torch
from torch import nn
from transformers import Qwen3NextConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, VllmConfig
@@ -58,7 +59,6 @@ from vllm.model_executor.models.qwen2_moe import Qwen2MoeMLP as Qwen3NextMLP
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
from vllm.v1.attention.backend import AttentionType
from .interfaces import (
@@ -232,6 +232,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
is_sequence_parallel=self.is_sequence_parallel,
n_shared_experts=1 if self.shared_expert is None else None,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
shared_expert_prefix=f"{prefix}.shared_expert",
shared_expert_gate=self.shared_expert_gate
if self.shared_expert is None
else None,
+8 -5
View File
@@ -6,6 +6,7 @@ from collections.abc import Iterable
import torch
from torch import nn
from transformers import Qwen3NextConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
@@ -29,7 +30,6 @@ from vllm.model_executor.models.qwen3_next import (
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
from .utils import (
AutoWeightsLoader,
@@ -227,15 +227,18 @@ class Qwen3NextMTP(nn.Module, QwenNextMixtureOfExperts):
) -> torch.Tensor | None:
return self.logits_processor(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
shared_weight_names = ["embed_tokens", "lm_head"]
def is_unused_checkpoint_weight(self, name: str) -> bool:
return not name.startswith("mtp.") and not any(
key in name for key in ["embed_tokens", "lm_head"]
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
def remap_weight_names(weights):
for name, weight in weights:
if self.is_unused_checkpoint_weight(name):
continue
if name.startswith("mtp."):
name = name.replace("mtp.", "model.")
elif not any(key in name for key in shared_weight_names):
continue
yield name, weight
loader = AutoWeightsLoader(self)
@@ -2311,8 +2311,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
llm_pos_ids_list.append(eos_pos)
st = offset + video_len + 1
else:
audio_bos_pos = np.broadcast_to(np.array([st_idx - 1]), (3, 1))
audio_bos_pos = np.broadcast_to(np.array([st_idx]), (3, 1))
llm_pos_ids_list.append(audio_bos_pos)
st_idx += 1
pos_ids, _ = self._compute_interleaved_positions(st_idx, data)
llm_pos_ids_list.append(pos_ids)
@@ -2320,7 +2321,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
eos_pos = np.broadcast_to(np.array([st_idx]), (3, 1))
llm_pos_ids_list.append(eos_pos)
llm_pos_ids_list.append(eos_pos)
llm_pos_ids_list.append(eos_pos + 1)
video_len = grid_t * grid_h * grid_w
audio_len = self._compute_audio_token_count(
+2
View File
@@ -246,6 +246,7 @@ _EMBEDDING_MODELS = {
# [Multimodal]
"CLIPModel": ("clip", "CLIPEmbeddingModel"),
"ColPaliForRetrieval": ("colpali", "ColPaliModel"),
"EmbeddingGemma2Model": ("embedding_gemma2", "EmbeddingGemma2Model"),
"LlamaNemotronVLModel": ("nemotron_vl", "LlamaNemotronVLForEmbedding"),
"LlavaNextForConditionalGeneration": (
"llava_next",
@@ -849,6 +850,7 @@ _OOT_SUPPORTED_MODELS = {
"BartForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
"Florence2ForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
"MBartForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
"Nemotron3_5AsrForRNNT": "https://github.com/Sohaib-Ahmed21/vllm-nemotron-asr-plugin",
}
+40 -3
View File
@@ -120,9 +120,35 @@ def _expand_tensor(input_tensor: torch.Tensor, scaling: int) -> torch.Tensor:
return (base.unsqueeze(1) + offsets).view(-1)
class VoxtralRealtimeProcessingInfo(VoxtralProcessingInfo):
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
# Each streaming update carries a single audio frame.
return {"audio": 1}
def get_max_audio_array_len(self) -> int:
# Profile with the largest streaming chunk, the first one: it covers
# the prompt prefix (BOS + left padding + delay), later chunks are
# ~1 token. get_max_audio_tokens keeps max_model_len so the encoder
# cache stays as large as before: each paused session keeps its last
# chunk cached until it resumes, and a cache sized to one chunk
# deadlocks a few concurrent sessions.
tokenizer = self.get_tokenizer()
audio_encoder = tokenizer.instruct.audio_encoder
prompt_tokens = (
tokenizer.instruct.start() + audio_encoder.encode_streaming_tokens()
)
return len(prompt_tokens) * audio_encoder.audio_config.raw_audio_length_per_tok
class VoxtralRealtimeBuffer:
def __init__(self, config: AudioConfig, prompt_tokens: list[int]) -> None:
def __init__(
self,
config: AudioConfig,
prompt_tokens: list[int],
max_model_len: int | None = None,
) -> None:
self._config = config
self._max_model_len = max_model_len
_look_ahead_in_ms = self._config.streaming_look_ahead_ms
_look_back_in_ms = self._config.streaming_look_back_ms
@@ -165,7 +191,16 @@ class VoxtralRealtimeBuffer:
await self._token_queue.put(token)
async def get_input_stream(self) -> AsyncGenerator[TokensPrompt]:
n_tokens = 0
for frame_size, num_tokens in self._generate_frame_size_and_num_tokens():
n_tokens += num_tokens
if self._max_model_len is not None and n_tokens >= self._max_model_len:
logger.warning(
"Stopping realtime audio stream: reached max_model_len (%d).",
self._max_model_len,
)
return
next_tokens = [await self._token_queue.get() for _ in range(num_tokens)]
audio_arrays: list[np.ndarray] = (
@@ -198,7 +233,7 @@ class VoxtralRealtimeBuffer:
@MULTIMODAL_REGISTRY.register_processor(
VoxtralRealtimeMultiModalProcessor,
info=VoxtralProcessingInfo,
info=VoxtralRealtimeProcessingInfo,
dummy_inputs=VoxtralDummyInputsBuilder,
)
@support_torch_compile
@@ -248,7 +283,9 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
# Get left/right padding audio
left_pad, right_pad = audio_encoder.get_padding_audio()
buffer = VoxtralRealtimeBuffer(config, prompt_tokens)
buffer = VoxtralRealtimeBuffer(
config, prompt_tokens, max_model_len=model_config.max_model_len
)
# Feed audio with padding into buffer in background
async def feed_audio():
+1
View File
@@ -896,6 +896,7 @@ class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixC
head_dim=hf_config.mamba_headdim,
state_size=hf_config.mamba_d_state,
conv_kernel=hf_config.mamba_d_conv,
num_spec=vllm_config.num_speculative_tokens,
)
@classmethod