[Chore] Remove vendored code made redundant by Transformers 5.16.1 (#60623)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-10-09 09:29:03 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent bc713cf8cc
commit 2c99ee9333
76 changed files with 719 additions and 3268 deletions
+1 -1
View File
@@ -29,6 +29,7 @@ from itertools import islice
import torch
from torch import nn
from transformers import AXK1Config
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
@@ -78,7 +79,6 @@ from vllm.model_executor.models.deepseek_v2 import (
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.AXK1 import AXK1Config
from .interfaces import MixtureOfExperts, SupportsEagle, SupportsLoRA, SupportsPP
from .utils import (
+10 -10
View File
@@ -7,6 +7,7 @@ from collections.abc import Iterable
import torch
import torch.nn as nn
from transformers import Aimv2VisionConfig
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.distributed.utils import divide
@@ -21,20 +22,19 @@ from vllm.model_executor.layers.linear import (
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
from vllm.transformers_utils.configs.ovis import AIMv2Config
class AIMv2SwiGLUFFN(nn.Module):
def __init__(
self,
config: AIMv2Config,
config: Aimv2VisionConfig,
quant_config: QuantizationConfig | None,
prefix: str,
):
super().__init__()
hidden_features = config.intermediate_size
in_features = config.hidden_size
bias = config.use_bias
bias = config.mlp_bias
self.fc13 = MergedColumnParallelLinear(
in_features,
@@ -60,7 +60,7 @@ class AIMv2SwiGLUFFN(nn.Module):
class AIMv2PatchEmbed(nn.Module):
def __init__(self, config: AIMv2Config):
def __init__(self, config: Aimv2VisionConfig):
super().__init__()
self.proj = Conv2dLayer(
config.num_channels,
@@ -77,7 +77,7 @@ class AIMv2PatchEmbed(nn.Module):
class AIMv2ViTPreprocessor(nn.Module):
def __init__(self, config: AIMv2Config):
def __init__(self, config: Aimv2VisionConfig):
super().__init__()
num_patches = (config.image_size // config.patch_size) ** 2
@@ -95,7 +95,7 @@ class AIMv2ViTPreprocessor(nn.Module):
class AIMv2Attention(nn.Module):
def __init__(
self,
config: AIMv2Config,
config: Aimv2VisionConfig,
quant_config: QuantizationConfig | None,
prefix: str,
):
@@ -124,7 +124,7 @@ class AIMv2Attention(nn.Module):
self.proj = RowParallelLinear(
input_size=self.embed_dim,
output_size=self.embed_dim,
bias=config.use_bias,
bias=config.mlp_bias,
quant_config=quant_config,
prefix=f"{prefix}.proj",
)
@@ -151,7 +151,7 @@ class AIMv2Attention(nn.Module):
class AIMv2Block(nn.Module):
def __init__(
self,
config: AIMv2Config,
config: Aimv2VisionConfig,
quant_config: QuantizationConfig | None,
prefix: str,
):
@@ -174,7 +174,7 @@ class AIMv2Block(nn.Module):
class AIMv2Transformer(nn.Module):
def __init__(
self,
config: AIMv2Config,
config: Aimv2VisionConfig,
quant_config: QuantizationConfig | None,
*,
require_post_norm: bool | None = None,
@@ -214,7 +214,7 @@ class AIMv2Model(torch.nn.Module):
def __init__(
self,
config: AIMv2Config,
config: Aimv2VisionConfig,
quant_config: QuantizationConfig | None,
*,
require_post_norm: bool | None = None,
-14
View File
@@ -403,20 +403,6 @@ class Cohere2MoeModel(nn.Module, EagleModelMixin):
config.vocab_size, config.hidden_size
)
# Decoder layers read per-layer MLP layout from config.mlp_layer_types
# (dense MLP vs MoE) and use it for weight loading. Transformers >=5.10
# populates this field; older versions only expose first_k_dense_replace.
# Normalize here so layer construction below sees a consistent layout.
if getattr(config, "mlp_layer_types", None) is None:
first_k_dense_replace = getattr(config, "first_k_dense_replace", None)
n = config.num_hidden_layers
if first_k_dense_replace is not None:
config.mlp_layer_types = ["dense"] * first_k_dense_replace + [
"sparse"
] * (n - first_k_dense_replace)
else:
config.mlp_layer_types = ["sparse"] * n
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: Cohere2MoeDecoderLayer(
+18 -1
View File
@@ -2,11 +2,28 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import regex
from transformers import Cosmos3OmniConfig
from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration
from vllm.model_executor.models.qwen3_vl import (
Qwen3VLDummyInputsBuilder,
Qwen3VLForConditionalGeneration,
Qwen3VLMultiModalProcessor,
Qwen3VLProcessingInfo,
)
from vllm.model_executor.models.utils import WeightsMapper
from vllm.multimodal import MULTIMODAL_REGISTRY
class Cosmos3ProcessingInfo(Qwen3VLProcessingInfo):
def get_hf_config(self) -> Cosmos3OmniConfig:
return self.ctx.get_hf_config(Cosmos3OmniConfig)
@MULTIMODAL_REGISTRY.register_processor(
Qwen3VLMultiModalProcessor,
info=Cosmos3ProcessingInfo,
dummy_inputs=Qwen3VLDummyInputsBuilder,
)
class Cosmos3ForConditionalGeneration(Qwen3VLForConditionalGeneration):
# Cosmos3 unified checkpoints store a Qwen3-VL understanding tower
# alongside a generation tower in a flat key layout. This mapper drops
+1 -2
View File
@@ -5,7 +5,7 @@ from collections.abc import Iterable
import torch
import torch.nn as nn
from transformers import ProcessorMixin
from transformers import NemotronHConfig, ProcessorMixin
from vllm.config import VllmConfig
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -20,7 +20,6 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
from vllm.utils.torch_utils import async_tensor_h2d
from .interfaces import (
+7 -20
View File
@@ -16,7 +16,7 @@ via Gemma4MultimodalEmbedder.
from __future__ import annotations
from collections.abc import Iterable, Mapping, Sequence
from collections.abc import Iterable, Sequence
from types import SimpleNamespace
from typing import Any
@@ -25,7 +25,7 @@ import torch
import torch._dynamo
from torch import nn
from torch.nn import functional as F
from transformers import AutoModel
from transformers import AutoModel, DiffusionGemmaConfig
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
@@ -113,26 +113,13 @@ class DiffusionGemmaSelfConditioning(nn.Module):
class DiffusionGemmaProcessingInfo(Gemma4ProcessingInfo):
"""Processing info for DiffusionGemma.
"""Processing info for DiffusionGemma, which has no audio tower."""
Overrides ``get_hf_config`` to accept ``DiffusionGemmaConfig``
(which inherits from ``PreTrainedConfig``, not ``Gemma4Config``).
Supports image and video modalities.
"""
def get_hf_config(self) -> DiffusionGemmaConfig:
return self.ctx.get_hf_config(DiffusionGemmaConfig)
def get_hf_config(self):
# DiffusionGemmaConfig doesn't inherit from Gemma4Config, so we
# accept any PreTrainedConfig here.
return self.ctx.get_hf_config()
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
# DiffusionGemma supports image and video inputs.
return {"image": None, "video": None}
def get_mm_max_tokens_per_item(
self, seq_len: int, mm_counts: Mapping[str, int]
) -> Mapping[str, int] | None:
return super().get_mm_max_tokens_per_item(seq_len, mm_counts)
def has_audio(self) -> bool:
return False
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
@@ -36,10 +36,7 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.model_executor.models.gemma4 import (
Gemma4MLP,
gemma4_layer_config,
)
from vllm.model_executor.models.gemma4 import Gemma4MLP
from vllm.model_executor.models.gemma4_mm import (
_SUPPORTED_SOFT_TOKENS,
Gemma4DummyInputsBuilder,
@@ -244,7 +241,7 @@ class EmbeddingGemma2DecoderLayer(nn.Module):
prefix: str,
):
super().__init__()
lc = gemma4_layer_config(config, idx)
lc = config.per_layer_config[idx]
self.self_attn = EmbeddingGemma2Attention(
config,
lc,
+1 -14
View File
@@ -154,20 +154,7 @@ class Gemma3Attention(nn.Module):
self.is_sliding = layer_type == "sliding_attention"
sliding_window = config.sliding_window if self.is_sliding else None
# Initialize the rotary embedding.
if layer_type in config.rope_parameters:
# Transformers v5 rope config.
rope_parameters = config.rope_parameters[layer_type]
else:
# Transformers v4 rope config.
# Global attention. Use the values in config.json.
rope_parameters = config.rope_parameters
# Local attention. Override the values in config.json.
if self.is_sliding:
rope_parameters = dict(
rope_type="default", rope_theta=config.rope_local_base_freq
)
rope_parameters = config.rope_parameters[layer_type]
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position_embeddings,
+1 -11
View File
@@ -350,17 +350,7 @@ class Gemma3nAttention(nn.Module):
is_sliding = layer_type == "sliding_attention"
self.sliding_window = config.sliding_window if is_sliding else None
# Initialize the rotary embedding.
if layer_type in config.rope_parameters:
# Transformers v5 rope config.
rope_parameters = config.rope_parameters[layer_type]
else:
# Transformers v4 rope config.
# Global attention. Use the values in config.json.
rope_parameters = config.rope_parameters.copy()
# Local attention. Override the values in config.json.
if is_sliding:
rope_parameters["rope_theta"] = config.rope_local_base_freq
rope_parameters = config.rope_parameters[layer_type]
kv_sharing_target_layer_name = None
if self.is_kv_shared_layer:
+2 -5
View File
@@ -59,7 +59,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import KVSharingFastPrefillMetadata
@@ -576,7 +575,7 @@ class Gemma4DecoderLayer(nn.Module):
self.layer_idx = layer_idx
# Gemma4 uses different head dimensions for sliding vs full attention
layer_config = gemma4_layer_config(config, layer_idx)
layer_config = config.per_layer_config[layer_idx]
head_dim = layer_config.head_dim
num_kv_heads = layer_config.num_key_value_heads
@@ -628,9 +627,7 @@ class Gemma4DecoderLayer(nn.Module):
)
# MoE (Mixture of Experts) — router + expert block parallel to MLP
self.enable_moe_block = getattr(config, "enable_moe_block", False) or getattr(
config, "use_second_mlp_block", False
)
self.enable_moe_block = getattr(config, "num_experts", None) is not None
self.router: Gemma4Router | None
self.experts: MoERunner | None
self.post_feedforward_layernorm_1: RMSNorm | None
+1 -2
View File
@@ -20,7 +20,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
from .gemma4_mtp import Gemma4MTPAttention, Gemma4MTPDecoderLayer
from .qwen3_dflash import DFlashQwen3Model, _dflash_layer_causal
@@ -48,7 +47,7 @@ class Gemma4DSparkAttention(Gemma4MTPAttention):
config, "attention_k_eq_v", False
)
layer_config = gemma4_layer_config(config, layer_idx)
layer_config = config.per_layer_config[layer_idx]
head_dim = layer_config.head_dim
num_kv_heads = layer_config.num_key_value_heads
super().__init__(
+7 -9
View File
@@ -222,12 +222,11 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
**kwargs,
)
def has_audio(self) -> bool:
return self.get_hf_config().audio_config is not None
def validate_num_items(self, modality: str, num_items: int) -> None:
if (
modality == "audio"
and num_items > 0
and self.get_hf_config().audio_config is None
):
if modality == "audio" and num_items > 0 and not self.has_audio():
model_config = self.ctx.model_config
model = get_served_model_name(
model_config.model, model_config.served_model_name
@@ -243,7 +242,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
limits: dict[str, int | None] = {"image": None}
if self.get_hf_config().audio_config is not None:
if self.has_audio():
limits["audio"] = None
limits["video"] = None
return limits
@@ -261,7 +260,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
if isinstance(val, int) and val in _SUPPORTED_SOFT_TOKENS:
tokens_per_image = val
tokens: dict[str, int] = {"image": tokens_per_image}
if config.audio_config is not None:
if self.has_audio():
# Audio max tokens from the processor's audio_seq_length.
processor = self.get_hf_processor()
tokens["audio"] = processor.audio_seq_length
@@ -275,9 +274,8 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
return tokens
def get_data_parser(self) -> MultiModalDataParser:
config = self.get_hf_config()
kwargs: dict[str, Any] = {"video_needs_metadata": True}
if getattr(config, "audio_config", None) is not None:
if self.has_audio():
processor = self.get_hf_processor()
kwargs["target_sr"] = processor.feature_extractor.sampling_rate
return MultiModalDataParser(**kwargs)
+1 -2
View File
@@ -45,7 +45,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
from .gemma4 import Gemma4MLP, _get_text_config
from .utils import (
@@ -265,7 +264,7 @@ class Gemma4MTPDecoderLayer(nn.Module):
self.hidden_size = config.hidden_size
layer_idx = extract_layer_index(prefix)
layer_config = gemma4_layer_config(config, layer_idx)
layer_config = config.per_layer_config[layer_idx]
head_dim = layer_config.head_dim
num_kv_heads = layer_config.num_key_value_heads
+42 -54
View File
@@ -493,13 +493,9 @@ class Granite4VisionForConditionalGeneration(
prefix=maybe_prefix(prefix, "vision_tower"),
)
# image_newline parameter
if config.use_image_newline_parameter:
self.image_newline = nn.Parameter(
torch.empty(config.text_config.hidden_size)
)
else:
self.image_newline = None
self.image_newline = nn.Parameter(
torch.empty(config.text_config.hidden_size)
)
cache_config = vllm_config.cache_config
@@ -517,20 +513,18 @@ class Granite4VisionForConditionalGeneration(
)
# Spatial projectors: 4 offset groups
self.spatial_projectors = None
if config.use_spatial_sampling:
self.spatial_projectors = nn.ModuleList(
[
WindowQFormerDownsampler(
config,
quant_config=quant_config,
cache_config=cache_config,
spatial_offset=i,
prefix=maybe_prefix(prefix, f"spatial_projectors.{i}"),
)
for i in range(4)
]
)
self.spatial_projectors = nn.ModuleList(
[
WindowQFormerDownsampler(
config,
quant_config=quant_config,
cache_config=cache_config,
spatial_offset=i,
prefix=maybe_prefix(prefix, f"spatial_projectors.{i}"),
)
for i in range(4)
]
)
# ----- Language model (marked as LM) -----
with self._mark_language_model(vllm_config):
@@ -545,12 +539,9 @@ class Granite4VisionForConditionalGeneration(
# Store config values we need
self._deepstack_layer_map = config.deepstack_layer_map # [[-19, 9], ...]
self._use_spatial_sampling = getattr(config, "use_spatial_sampling", False)
self._spatial_vision_layer = getattr(config, "spatial_vision_layer", -1)
self._spatial_target_layers = getattr(config, "spatial_target_layers", [])
self._vision_feature_select_strategy = getattr(
config, "vision_feature_select_strategy", "full"
)
self._spatial_vision_layer = config.spatial_vision_layer
self._spatial_target_layers = config.spatial_target_layers
self._vision_feature_select_strategy = config.vision_feature_select_strategy
self._downsample_rate = Fraction(config.downsample_rate)
# Ordered list of LLM layer indices for each deepstack level.
@@ -558,7 +549,7 @@ class Granite4VisionForConditionalGeneration(
# (before any embed_multimodal call).
self._ds_layer_indices: list[int] = [
llm_layer for _, llm_layer in config.deepstack_layer_map
] + list(getattr(config, "spatial_target_layers", []))
] + list(config.spatial_target_layers)
# Share ds_layer_indices with the LLM causal model so
# make_empty_intermediate_tensors includes the correct keys
@@ -571,7 +562,7 @@ class Granite4VisionForConditionalGeneration(
# the updated values written just before each prefill.
# Shape: (max_num_batched_tokens, lm_hidden_size) per level.
n_layerwise = len(config.deepstack_layer_map)
n_spatial = len(getattr(config, "spatial_target_layers", []))
n_spatial = len(config.spatial_target_layers)
num_ds_levels = n_layerwise + n_spatial
lm_hidden = config.text_config.hidden_size
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
@@ -644,26 +635,24 @@ class Granite4VisionForConditionalGeneration(
)
image_feature = unpad_image(image_feature, image_sizes[image_idx])
if self.image_newline is not None:
image_feature = torch.cat(
(
image_feature,
self.image_newline[:, None, None]
.expand(*image_feature.shape[:-1], 1)
.to(image_feature.device, image_feature.dtype),
),
dim=-1,
)
image_feature = torch.cat(
(
image_feature,
self.image_newline[:, None, None]
.expand(*image_feature.shape[:-1], 1)
.to(image_feature.device, image_feature.dtype),
),
dim=-1,
)
image_feature = image_feature.flatten(1, 2).transpose(0, 1)
image_feature = torch.cat((base_image_feature, image_feature), dim=0)
else:
image_feature = image_feature[0]
if self.image_newline is not None:
image_feature = torch.cat(
(image_feature, self.image_newline[None].to(image_feature)),
dim=0,
)
image_feature = torch.cat(
(image_feature, self.image_newline[None].to(image_feature)),
dim=0,
)
new_image_features.append(image_feature)
@@ -718,16 +707,15 @@ class Granite4VisionForConditionalGeneration(
)
levels.append((llm_layer, per_image))
if self._use_spatial_sampling and self.spatial_projectors is not None:
spatial_hidden = all_hidden_states[self._spatial_vision_layer]
if select_strategy == "default":
spatial_hidden = spatial_hidden[:, 1:]
for group_idx, llm_layer in enumerate(self._spatial_target_layers):
projected = self.spatial_projectors[group_idx](spatial_hidden)
per_image = self._pack_and_unpad_image_features(
torch.split(projected, image_num_patches, dim=0), image_sizes
)
levels.append((llm_layer, per_image))
spatial_hidden = all_hidden_states[self._spatial_vision_layer]
if select_strategy == "default":
spatial_hidden = spatial_hidden[:, 1:]
for group_idx, llm_layer in enumerate(self._spatial_target_layers):
projected = self.spatial_projectors[group_idx](spatial_hidden)
per_image = self._pack_and_unpad_image_features(
torch.split(projected, image_num_patches, dim=0), image_sizes
)
levels.append((llm_layer, per_image))
llm_layer_indices = [llm_layer for llm_layer, _ in levels]
num_images = len(image_sizes)
+2 -5
View File
@@ -158,9 +158,7 @@ class GraniteSpeechMultiModalProcessor(
feature_extractor = processor.audio_processor
vocab = tokenizer.get_vocab()
# Use getattr with default to be compatible with transformers<4.48
audio_token = getattr(processor, "audio_token", "<|audio|>")
audio_token_id = vocab[audio_token]
audio_token_id = vocab[processor.audio_token]
def get_replacement(item_idx: int):
audios = mm_items.get_items("audio", AudioProcessorItems)
@@ -221,8 +219,7 @@ class GraniteSpeechDummyInputsBuilder(
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_audios = mm_counts.get("audio", 0)
hf_processor = self.info.get_hf_processor()
audio_token = getattr(hf_processor, "audio_token", "<|audio|>")
return audio_token * num_audios
return hf_processor.audio_token * num_audios
### QFormer Projector
@@ -320,10 +320,6 @@ class GraniteMoeHybridAttention(nn.Module):
ALL_DECODER_LAYER_TYPES = {
# Transformers < 5.13.0
"attention": GraniteMoeHybridAttentionDecoderLayer,
"mamba": GraniteMoeHybridMambaDecoderLayer,
# Transformers >= 5.13.0
"full_attention": GraniteMoeHybridAttentionDecoderLayer,
"linear_attention": GraniteMoeHybridMambaDecoderLayer,
}
+1 -2
View File
@@ -31,7 +31,7 @@ from typing import Any
import torch
from torch import nn
from transformers import PreTrainedConfig
from transformers import HYV3Config, PreTrainedConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
@@ -67,7 +67,6 @@ from vllm.model_executor.model_loader.weight_utils import (
maybe_remap_kv_scale_name,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.hy_v3 import HYV3Config
from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
from .utils import (
+6 -11
View File
@@ -31,6 +31,7 @@ from itertools import islice
import torch
from torch import nn
from transformers import HyperCLOVAXConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
@@ -51,7 +52,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.hyperclovax import HyperCLOVAXConfig
from .interfaces import SupportsLoRA, SupportsPP
from .utils import (
@@ -137,9 +137,7 @@ class HyperCLOVAXAttention(nn.Module):
# the KV heads across multiple tensor parallel GPUs.
assert tp_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
self.head_dim = getattr(
config, "head_dim", self.hidden_size // self.total_num_heads
)
self.head_dim = config.head_dim
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
self.scaling = config.attention_multiplier
@@ -166,7 +164,7 @@ class HyperCLOVAXAttention(nn.Module):
self.head_dim,
max_position=max_position_embeddings,
is_neox_style=True,
rope_parameters=getattr(config, "rope_parameters", None),
rope_parameters=config.rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
)
@@ -217,18 +215,15 @@ class HyperCLOVAXDecoderLayer(nn.Module):
"dual_chunk_attention_config",
None,
)
attention_bias = getattr(config, "attention_bias", False)
self.self_attn = HyperCLOVAXAttention(
config=config,
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
num_kv_heads=getattr(
config, "num_key_value_heads", config.num_attention_heads
),
num_kv_heads=config.num_key_value_heads,
max_position_embeddings=max_position_embeddings,
quant_config=quant_config,
bias=attention_bias,
bias=config.attention_bias,
cache_config=cache_config,
prefix=f"{prefix}.self_attn",
dual_chunk_attention_config=dual_chunk_attention_config,
@@ -238,7 +233,7 @@ class HyperCLOVAXDecoderLayer(nn.Module):
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
bias=getattr(config, "mlp_bias", False),
bias=config.mlp_bias,
prefix=f"{prefix}.mlp",
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+8 -9
View File
@@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Literal
import torch
from torch import nn
from transformers import BatchFeature
from transformers import BatchFeature, Kimi_K25Config
from vllm.config import VllmConfig
from vllm.config.multimodal import MultiModalDummyOptions
@@ -63,7 +63,6 @@ from vllm.multimodal.processing import (
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.kimi_k25 import KimiK25Config
from vllm.transformers_utils.processor import cached_get_image_processor
from vllm.transformers_utils.processors.kimi_k25 import KimiK25Processor
from vllm.transformers_utils.processors.kimi_k25_vision_fused import (
@@ -135,7 +134,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
# Resolve token ID from the tokenizer because transformers v5
# may remap token IDs vs config.json.
config_token_id = hf_config.media_placeholder_token_id
config_token_id = hf_config.image_token_id
resolved_token_id = tokenizer.convert_tokens_to_ids("<|media_pad|>")
unk_token_id = getattr(tokenizer, "unk_token_id", None)
is_valid_resolved = isinstance(resolved_token_id, int) and (
@@ -143,7 +142,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
)
if is_valid_resolved and resolved_token_id != config_token_id:
logger.warning_once(
"Kimi-K2.5 config.media_placeholder_token_id (%d) disagrees "
"Kimi-K2.5 config.image_token_id (%d) disagrees "
"with tokenizer mapping for <|media_pad|> (%d). "
"Using tokenizer value.",
config_token_id,
@@ -151,7 +150,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
)
media_token_id = resolved_token_id
# Patch config so downstream code also sees the correct ID.
hf_config.media_placeholder_token_id = resolved_token_id
hf_config.image_token_id = resolved_token_id
else:
media_token_id = config_token_id
@@ -170,7 +169,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
return self.hf_processor
def get_hf_config(self):
return self.ctx.get_hf_config(KimiK25Config)
return self.ctx.get_hf_config(Kimi_K25Config)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
# None means unlimited
@@ -338,7 +337,7 @@ class KimiK25ForConditionalGeneration(
) -> None:
super().__init__()
model_config = vllm_config.model_config
config: KimiK25Config = model_config.hf_config
config: Kimi_K25Config = model_config.hf_config
self.config = config
quant_config = vllm_config.quant_config
@@ -347,7 +346,6 @@ class KimiK25ForConditionalGeneration(
)
self.hidden_size = config.text_config.hidden_size
self.device = current_platform.current_device()
# Build vision tower directly with KimiK25VisionConfig
with self._mark_tower_model(vllm_config, "vision_chunk"):
self.vision_tower = MoonViT3dPretrainedModel(
config.vision_config,
@@ -364,6 +362,7 @@ class KimiK25ForConditionalGeneration(
self.mm_projector = KimiK25MultiModalProjector(
config=config.vision_config,
out_hidden_size=config.text_config.hidden_size,
use_data_parallel=self.use_data_parallel,
quant_config=self._maybe_ignore_quant_config(quant_config),
prefix=maybe_prefix(prefix, "mm_projector"),
@@ -383,7 +382,7 @@ class KimiK25ForConditionalGeneration(
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors
)
self.media_placeholder: int = self.config.media_placeholder_token_id
self.media_placeholder: int = self.config.image_token_id
def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig | None):
if isinstance(quant_config, compressed_tensors.CompressedTensorsConfig):
+26 -18
View File
@@ -15,6 +15,7 @@ import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import Kimi_K25VisionConfig
from transformers.activations import GELUActivation
from vllm.distributed import divide, get_tensor_model_parallel_world_size
@@ -40,13 +41,21 @@ from vllm.model_executor.models.vision import (
run_dp_sharded_mrope_vision_model,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig
from vllm.transformers_utils.configs.kimi_k3 import KimiK3VisionConfig
from vllm.triton_utils import HAS_TRITON
from vllm.utils.torch_utils import async_tensor_h2d
logger = init_logger(__name__)
def _get_pos_emb_size(
config: Kimi_K25VisionConfig | KimiK3VisionConfig, dim: str
) -> int:
# Legacy checkpoints and Kimi-K3 use `init_pos_emb_*`
legacy = getattr(config, f"init_pos_emb_{dim}", None)
return legacy if legacy is not None else getattr(config, f"pos_emb_{dim}")
def _apply_rope_input_validation(x, freqs_cis):
assert x.ndim == freqs_cis.ndim + 1, (x.shape, freqs_cis.shape)
assert x.shape[:-2] == freqs_cis.shape[:-1], (x.shape, freqs_cis.shape)
@@ -698,14 +707,11 @@ def tpool_patch_merger_packed(
class MoonViT3dPretrainedModel(nn.Module):
"""Main vision tower model.
Uses KimiK25VisionConfig directly from transformers_utils/configs/kimi_k25.py.
"""
"""Main vision tower model."""
def __init__(
self,
config: KimiK25VisionConfig,
config: Kimi_K25VisionConfig | KimiK3VisionConfig,
quant_config: QuantizationConfig | None = None,
input_norm: nn.Module | None = None,
prefix: str = "",
@@ -713,17 +719,18 @@ class MoonViT3dPretrainedModel(nn.Module):
super().__init__()
config = deepcopy(config)
self.config = config # Required for run_dp_sharded_mrope_vision_model
self.merge_kernel_size = config.merge_kernel_size
merge_h, merge_w = config.merge_kernel_size
self.merge_kernel_size = (merge_h, merge_w)
self.patch_size = config.patch_size
self.merge_type = config.merge_type
self.merge_type = getattr(config, "merge_type", "sd2_tpool")
self.patch_embed = MoonVision3dPatchEmbed(
out_dim=config.hidden_size,
patch_size=config.patch_size,
pos_emb_height=config.init_pos_emb_height,
pos_emb_width=config.init_pos_emb_width,
pos_emb_time=config.init_pos_emb_time,
pos_emb_type=config.pos_emb_type,
pos_emb_height=_get_pos_emb_size(config, "height"),
pos_emb_width=_get_pos_emb_size(config, "width"),
pos_emb_time=_get_pos_emb_size(config, "time"),
pos_emb_type=getattr(config, "pos_emb_type", "divided_fixed"),
patch_embed_proj_bias=getattr(config, "patch_embed_proj_bias", True),
pos_emb_interpolation_mode=getattr(
config, "pos_emb_interpolation_mode", "bicubic"
@@ -740,14 +747,14 @@ class MoonViT3dPretrainedModel(nn.Module):
"qkv_hidden_size": getattr(config, "qkv_hidden_size", None),
"mlp_dim": config.intermediate_size,
"activation": get_act_fn(
getattr(config, "activation_func", "gelu_pytorch_tanh")
getattr(config, "activation_func", None) or config.hidden_act
),
"attn_bias": getattr(config, "attn_bias", True),
"norm_type": getattr(config, "norm_type", "layernorm"),
"mlp_type": getattr(config, "mlp_type", "mlp2"),
"linear_bias": getattr(config, "linear_bias", True),
},
video_attn_type=config.video_attn_type,
video_attn_type=getattr(config, "video_attn_type", "spatial_temporal"),
quant_config=quant_config,
prefix=maybe_prefix(prefix, "encoder"),
)
@@ -886,7 +893,8 @@ class KimiK25MultiModalProjector(nn.Module):
def __init__(
self,
config: KimiK25VisionConfig,
config: Kimi_K25VisionConfig | KimiK3VisionConfig,
out_hidden_size: int,
use_data_parallel: bool = False,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
@@ -909,13 +917,13 @@ class KimiK25MultiModalProjector(nn.Module):
)
self.linear_2 = ReplicatedLinear(
self.hidden_size,
getattr(config, "text_hidden_size", config.mm_hidden_size),
out_hidden_size,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.linear_2",
)
self.post_norm = torch.nn.RMSNorm(
getattr(config, "text_hidden_size", config.mm_hidden_size),
out_hidden_size,
eps=config.projector_ln_eps,
)
self.act = GELUActivation()
@@ -931,7 +939,7 @@ class KimiK25MultiModalProjector(nn.Module):
)
self.linear_2 = ReplicatedLinear(
self.hidden_size,
config.mm_hidden_size,
out_hidden_size,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.linear_2",
+1 -1
View File
@@ -5,6 +5,7 @@ from itertools import islice
import torch
import torch.nn as nn
from transformers import Lfm2MoeConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
@@ -40,7 +41,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.lfm2_moe import Lfm2MoeConfig
from .interfaces import (
HasInnerState,
+6 -22
View File
@@ -94,28 +94,12 @@ class ModernBertAttention(nn.Module):
prefix=f"{prefix}.Wqkv",
)
if layer_types := getattr(config, "layer_types", None):
# Transformers v5
layer_type = layer_types[layer_id]
rope_parameters = config.rope_parameters[layer_type]
sliding_window: int | None = None
if layer_type == "sliding_attention":
# Treats the local attention boundary as inclusive
sliding_window = config.sliding_window + 1
else:
# Transformers v4
sliding_window = None
if layer_id % config.global_attn_every_n_layers != 0:
# ModernBertConfig does not expose sliding_window
sliding_window = config.local_attention // 2
rope_theta = (
config.local_rope_theta
if config.local_rope_theta is not None
else config.global_rope_theta
)
else:
rope_theta = config.global_rope_theta
rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
layer_type = config.layer_types[layer_id]
rope_parameters = config.rope_parameters[layer_type]
sliding_window: int | None = None
if layer_type == "sliding_attention":
# Treats the local attention boundary as inclusive
sliding_window = config.sliding_window + 1
self.rotary_emb = get_rope(
head_size=self.head_dim,
+1 -1
View File
@@ -29,6 +29,7 @@ from itertools import islice
import torch
from torch import nn
from transformers import NemotronConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
@@ -49,7 +50,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.nemotron import NemotronConfig
from .interfaces import SupportsLoRA, SupportsPP
from .utils import (
+1 -1
View File
@@ -23,6 +23,7 @@ from itertools import islice
import torch
from torch import nn
from transformers import NemotronHConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import (
@@ -87,7 +88,6 @@ from vllm.model_executor.models.utils import (
sequence_parallel_chunk,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
class NemotronHMLP(nn.Module):
+1 -1
View File
@@ -7,6 +7,7 @@ from collections.abc import Callable, Iterable
import torch
import torch.nn as nn
from transformers import NemotronHConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, VllmConfig
@@ -33,7 +34,6 @@ from vllm.model_executor.models.utils import (
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
from .interfaces import SupportsPP, SupportsQuant
from .nemotron_h import (
+1 -1
View File
@@ -125,7 +125,7 @@ class VisualTokenizer(torch.nn.Module):
prefix: str = "",
) -> nn.Module:
model_type = config.backbone_config.model_type
if model_type == "aimv2":
if model_type == "aimv2_vision_model":
# No post rms_norm in Ovis2's AIMv2 ViT.
return AIMv2Model(
config=config.backbone_config,
+6 -5
View File
@@ -29,6 +29,12 @@ from typing import ClassVar
import torch
from torch import nn
from transformers import (
Qwen3_5Config,
Qwen3_5MoeConfig,
Qwen3_5MoeTextConfig,
Qwen3_5TextConfig,
)
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
@@ -59,11 +65,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig
from vllm.transformers_utils.configs.qwen3_5_moe import (
Qwen3_5MoeConfig,
Qwen3_5MoeTextConfig,
)
from .interfaces import (
HasInnerState,
+1 -2
View File
@@ -6,6 +6,7 @@ from collections.abc import Iterable
import torch
from torch import nn
from transformers import Qwen3_5MoeTextConfig, Qwen3_5TextConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
@@ -32,8 +33,6 @@ from vllm.model_executor.models.qwen3_next import (
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5TextConfig
from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig
from .interfaces import (
MultiModalEmbeddings,
+1 -1
View File
@@ -7,6 +7,7 @@ from itertools import islice
import torch
from torch import nn
from transformers import Qwen3NextConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, VllmConfig
@@ -58,7 +59,6 @@ from vllm.model_executor.models.qwen2_moe import Qwen2MoeMLP as Qwen3NextMLP
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
from vllm.v1.attention.backend import AttentionType
from .interfaces import (
+1 -1
View File
@@ -6,6 +6,7 @@ from collections.abc import Iterable
import torch
from torch import nn
from transformers import Qwen3NextConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
@@ -29,7 +30,6 @@ from vllm.model_executor.models.qwen3_next import (
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
from .utils import (
AutoWeightsLoader,