[Chore] Remove vendored code made redundant by Transformers 5.16.1 (#60623)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
bc713cf8cc
commit
2c99ee9333
@@ -29,6 +29,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import AXK1Config
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
@@ -78,7 +79,6 @@ from vllm.model_executor.models.deepseek_v2 import (
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.AXK1 import AXK1Config
|
||||
|
||||
from .interfaces import MixtureOfExperts, SupportsEagle, SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
|
||||
@@ -7,6 +7,7 @@ from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import Aimv2VisionConfig
|
||||
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.distributed.utils import divide
|
||||
@@ -21,20 +22,19 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
|
||||
from vllm.transformers_utils.configs.ovis import AIMv2Config
|
||||
|
||||
|
||||
class AIMv2SwiGLUFFN(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: AIMv2Config,
|
||||
config: Aimv2VisionConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
):
|
||||
super().__init__()
|
||||
hidden_features = config.intermediate_size
|
||||
in_features = config.hidden_size
|
||||
bias = config.use_bias
|
||||
bias = config.mlp_bias
|
||||
|
||||
self.fc13 = MergedColumnParallelLinear(
|
||||
in_features,
|
||||
@@ -60,7 +60,7 @@ class AIMv2SwiGLUFFN(nn.Module):
|
||||
|
||||
|
||||
class AIMv2PatchEmbed(nn.Module):
|
||||
def __init__(self, config: AIMv2Config):
|
||||
def __init__(self, config: Aimv2VisionConfig):
|
||||
super().__init__()
|
||||
self.proj = Conv2dLayer(
|
||||
config.num_channels,
|
||||
@@ -77,7 +77,7 @@ class AIMv2PatchEmbed(nn.Module):
|
||||
|
||||
|
||||
class AIMv2ViTPreprocessor(nn.Module):
|
||||
def __init__(self, config: AIMv2Config):
|
||||
def __init__(self, config: Aimv2VisionConfig):
|
||||
super().__init__()
|
||||
num_patches = (config.image_size // config.patch_size) ** 2
|
||||
|
||||
@@ -95,7 +95,7 @@ class AIMv2ViTPreprocessor(nn.Module):
|
||||
class AIMv2Attention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: AIMv2Config,
|
||||
config: Aimv2VisionConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
):
|
||||
@@ -124,7 +124,7 @@ class AIMv2Attention(nn.Module):
|
||||
self.proj = RowParallelLinear(
|
||||
input_size=self.embed_dim,
|
||||
output_size=self.embed_dim,
|
||||
bias=config.use_bias,
|
||||
bias=config.mlp_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.proj",
|
||||
)
|
||||
@@ -151,7 +151,7 @@ class AIMv2Attention(nn.Module):
|
||||
class AIMv2Block(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: AIMv2Config,
|
||||
config: Aimv2VisionConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
):
|
||||
@@ -174,7 +174,7 @@ class AIMv2Block(nn.Module):
|
||||
class AIMv2Transformer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: AIMv2Config,
|
||||
config: Aimv2VisionConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
*,
|
||||
require_post_norm: bool | None = None,
|
||||
@@ -214,7 +214,7 @@ class AIMv2Model(torch.nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AIMv2Config,
|
||||
config: Aimv2VisionConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
*,
|
||||
require_post_norm: bool | None = None,
|
||||
|
||||
@@ -403,20 +403,6 @@ class Cohere2MoeModel(nn.Module, EagleModelMixin):
|
||||
config.vocab_size, config.hidden_size
|
||||
)
|
||||
|
||||
# Decoder layers read per-layer MLP layout from config.mlp_layer_types
|
||||
# (dense MLP vs MoE) and use it for weight loading. Transformers >=5.10
|
||||
# populates this field; older versions only expose first_k_dense_replace.
|
||||
# Normalize here so layer construction below sees a consistent layout.
|
||||
if getattr(config, "mlp_layer_types", None) is None:
|
||||
first_k_dense_replace = getattr(config, "first_k_dense_replace", None)
|
||||
n = config.num_hidden_layers
|
||||
if first_k_dense_replace is not None:
|
||||
config.mlp_layer_types = ["dense"] * first_k_dense_replace + [
|
||||
"sparse"
|
||||
] * (n - first_k_dense_replace)
|
||||
else:
|
||||
config.mlp_layer_types = ["sparse"] * n
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: Cohere2MoeDecoderLayer(
|
||||
|
||||
@@ -2,11 +2,28 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import regex
|
||||
from transformers import Cosmos3OmniConfig
|
||||
|
||||
from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration
|
||||
from vllm.model_executor.models.qwen3_vl import (
|
||||
Qwen3VLDummyInputsBuilder,
|
||||
Qwen3VLForConditionalGeneration,
|
||||
Qwen3VLMultiModalProcessor,
|
||||
Qwen3VLProcessingInfo,
|
||||
)
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
|
||||
class Cosmos3ProcessingInfo(Qwen3VLProcessingInfo):
|
||||
def get_hf_config(self) -> Cosmos3OmniConfig:
|
||||
return self.ctx.get_hf_config(Cosmos3OmniConfig)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
Qwen3VLMultiModalProcessor,
|
||||
info=Cosmos3ProcessingInfo,
|
||||
dummy_inputs=Qwen3VLDummyInputsBuilder,
|
||||
)
|
||||
class Cosmos3ForConditionalGeneration(Qwen3VLForConditionalGeneration):
|
||||
# Cosmos3 unified checkpoints store a Qwen3-VL understanding tower
|
||||
# alongside a generation tower in a flat key layout. This mapper drops
|
||||
|
||||
@@ -5,7 +5,7 @@ from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import ProcessorMixin
|
||||
from transformers import NemotronHConfig, ProcessorMixin
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -20,7 +20,6 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
from .interfaces import (
|
||||
|
||||
@@ -16,7 +16,7 @@ via Gemma4MultimodalEmbedder.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from collections.abc import Iterable, Sequence
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
@@ -25,7 +25,7 @@ import torch
|
||||
import torch._dynamo
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
from transformers import AutoModel
|
||||
from transformers import AutoModel, DiffusionGemmaConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
@@ -113,26 +113,13 @@ class DiffusionGemmaSelfConditioning(nn.Module):
|
||||
|
||||
|
||||
class DiffusionGemmaProcessingInfo(Gemma4ProcessingInfo):
|
||||
"""Processing info for DiffusionGemma.
|
||||
"""Processing info for DiffusionGemma, which has no audio tower."""
|
||||
|
||||
Overrides ``get_hf_config`` to accept ``DiffusionGemmaConfig``
|
||||
(which inherits from ``PreTrainedConfig``, not ``Gemma4Config``).
|
||||
Supports image and video modalities.
|
||||
"""
|
||||
def get_hf_config(self) -> DiffusionGemmaConfig:
|
||||
return self.ctx.get_hf_config(DiffusionGemmaConfig)
|
||||
|
||||
def get_hf_config(self):
|
||||
# DiffusionGemmaConfig doesn't inherit from Gemma4Config, so we
|
||||
# accept any PreTrainedConfig here.
|
||||
return self.ctx.get_hf_config()
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
# DiffusionGemma supports image and video inputs.
|
||||
return {"image": None, "video": None}
|
||||
|
||||
def get_mm_max_tokens_per_item(
|
||||
self, seq_len: int, mm_counts: Mapping[str, int]
|
||||
) -> Mapping[str, int] | None:
|
||||
return super().get_mm_max_tokens_per_item(seq_len, mm_counts)
|
||||
def has_audio(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
|
||||
|
||||
@@ -36,10 +36,7 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.models.gemma4 import (
|
||||
Gemma4MLP,
|
||||
gemma4_layer_config,
|
||||
)
|
||||
from vllm.model_executor.models.gemma4 import Gemma4MLP
|
||||
from vllm.model_executor.models.gemma4_mm import (
|
||||
_SUPPORTED_SOFT_TOKENS,
|
||||
Gemma4DummyInputsBuilder,
|
||||
@@ -244,7 +241,7 @@ class EmbeddingGemma2DecoderLayer(nn.Module):
|
||||
prefix: str,
|
||||
):
|
||||
super().__init__()
|
||||
lc = gemma4_layer_config(config, idx)
|
||||
lc = config.per_layer_config[idx]
|
||||
self.self_attn = EmbeddingGemma2Attention(
|
||||
config,
|
||||
lc,
|
||||
|
||||
@@ -154,20 +154,7 @@ class Gemma3Attention(nn.Module):
|
||||
self.is_sliding = layer_type == "sliding_attention"
|
||||
sliding_window = config.sliding_window if self.is_sliding else None
|
||||
|
||||
# Initialize the rotary embedding.
|
||||
if layer_type in config.rope_parameters:
|
||||
# Transformers v5 rope config.
|
||||
rope_parameters = config.rope_parameters[layer_type]
|
||||
else:
|
||||
# Transformers v4 rope config.
|
||||
# Global attention. Use the values in config.json.
|
||||
rope_parameters = config.rope_parameters
|
||||
# Local attention. Override the values in config.json.
|
||||
if self.is_sliding:
|
||||
rope_parameters = dict(
|
||||
rope_type="default", rope_theta=config.rope_local_base_freq
|
||||
)
|
||||
|
||||
rope_parameters = config.rope_parameters[layer_type]
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
|
||||
@@ -350,17 +350,7 @@ class Gemma3nAttention(nn.Module):
|
||||
is_sliding = layer_type == "sliding_attention"
|
||||
self.sliding_window = config.sliding_window if is_sliding else None
|
||||
|
||||
# Initialize the rotary embedding.
|
||||
if layer_type in config.rope_parameters:
|
||||
# Transformers v5 rope config.
|
||||
rope_parameters = config.rope_parameters[layer_type]
|
||||
else:
|
||||
# Transformers v4 rope config.
|
||||
# Global attention. Use the values in config.json.
|
||||
rope_parameters = config.rope_parameters.copy()
|
||||
# Local attention. Override the values in config.json.
|
||||
if is_sliding:
|
||||
rope_parameters["rope_theta"] = config.rope_local_base_freq
|
||||
rope_parameters = config.rope_parameters[layer_type]
|
||||
|
||||
kv_sharing_target_layer_name = None
|
||||
if self.is_kv_shared_layer:
|
||||
|
||||
@@ -59,7 +59,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import KVSharingFastPrefillMetadata
|
||||
|
||||
@@ -576,7 +575,7 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
self.layer_idx = layer_idx
|
||||
|
||||
# Gemma4 uses different head dimensions for sliding vs full attention
|
||||
layer_config = gemma4_layer_config(config, layer_idx)
|
||||
layer_config = config.per_layer_config[layer_idx]
|
||||
head_dim = layer_config.head_dim
|
||||
num_kv_heads = layer_config.num_key_value_heads
|
||||
|
||||
@@ -628,9 +627,7 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# MoE (Mixture of Experts) — router + expert block parallel to MLP
|
||||
self.enable_moe_block = getattr(config, "enable_moe_block", False) or getattr(
|
||||
config, "use_second_mlp_block", False
|
||||
)
|
||||
self.enable_moe_block = getattr(config, "num_experts", None) is not None
|
||||
self.router: Gemma4Router | None
|
||||
self.experts: MoERunner | None
|
||||
self.post_feedforward_layernorm_1: RMSNorm | None
|
||||
|
||||
@@ -20,7 +20,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
|
||||
|
||||
from .gemma4_mtp import Gemma4MTPAttention, Gemma4MTPDecoderLayer
|
||||
from .qwen3_dflash import DFlashQwen3Model, _dflash_layer_causal
|
||||
@@ -48,7 +47,7 @@ class Gemma4DSparkAttention(Gemma4MTPAttention):
|
||||
config, "attention_k_eq_v", False
|
||||
)
|
||||
|
||||
layer_config = gemma4_layer_config(config, layer_idx)
|
||||
layer_config = config.per_layer_config[layer_idx]
|
||||
head_dim = layer_config.head_dim
|
||||
num_kv_heads = layer_config.num_key_value_heads
|
||||
super().__init__(
|
||||
|
||||
@@ -222,12 +222,11 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def has_audio(self) -> bool:
|
||||
return self.get_hf_config().audio_config is not None
|
||||
|
||||
def validate_num_items(self, modality: str, num_items: int) -> None:
|
||||
if (
|
||||
modality == "audio"
|
||||
and num_items > 0
|
||||
and self.get_hf_config().audio_config is None
|
||||
):
|
||||
if modality == "audio" and num_items > 0 and not self.has_audio():
|
||||
model_config = self.ctx.model_config
|
||||
model = get_served_model_name(
|
||||
model_config.model, model_config.served_model_name
|
||||
@@ -243,7 +242,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
limits: dict[str, int | None] = {"image": None}
|
||||
if self.get_hf_config().audio_config is not None:
|
||||
if self.has_audio():
|
||||
limits["audio"] = None
|
||||
limits["video"] = None
|
||||
return limits
|
||||
@@ -261,7 +260,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
if isinstance(val, int) and val in _SUPPORTED_SOFT_TOKENS:
|
||||
tokens_per_image = val
|
||||
tokens: dict[str, int] = {"image": tokens_per_image}
|
||||
if config.audio_config is not None:
|
||||
if self.has_audio():
|
||||
# Audio max tokens from the processor's audio_seq_length.
|
||||
processor = self.get_hf_processor()
|
||||
tokens["audio"] = processor.audio_seq_length
|
||||
@@ -275,9 +274,8 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
return tokens
|
||||
|
||||
def get_data_parser(self) -> MultiModalDataParser:
|
||||
config = self.get_hf_config()
|
||||
kwargs: dict[str, Any] = {"video_needs_metadata": True}
|
||||
if getattr(config, "audio_config", None) is not None:
|
||||
if self.has_audio():
|
||||
processor = self.get_hf_processor()
|
||||
kwargs["target_sr"] = processor.feature_extractor.sampling_rate
|
||||
return MultiModalDataParser(**kwargs)
|
||||
|
||||
@@ -45,7 +45,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
|
||||
|
||||
from .gemma4 import Gemma4MLP, _get_text_config
|
||||
from .utils import (
|
||||
@@ -265,7 +264,7 @@ class Gemma4MTPDecoderLayer(nn.Module):
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
layer_config = gemma4_layer_config(config, layer_idx)
|
||||
layer_config = config.per_layer_config[layer_idx]
|
||||
head_dim = layer_config.head_dim
|
||||
num_kv_heads = layer_config.num_key_value_heads
|
||||
|
||||
|
||||
@@ -493,13 +493,9 @@ class Granite4VisionForConditionalGeneration(
|
||||
prefix=maybe_prefix(prefix, "vision_tower"),
|
||||
)
|
||||
|
||||
# image_newline parameter
|
||||
if config.use_image_newline_parameter:
|
||||
self.image_newline = nn.Parameter(
|
||||
torch.empty(config.text_config.hidden_size)
|
||||
)
|
||||
else:
|
||||
self.image_newline = None
|
||||
self.image_newline = nn.Parameter(
|
||||
torch.empty(config.text_config.hidden_size)
|
||||
)
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
|
||||
@@ -517,20 +513,18 @@ class Granite4VisionForConditionalGeneration(
|
||||
)
|
||||
|
||||
# Spatial projectors: 4 offset groups
|
||||
self.spatial_projectors = None
|
||||
if config.use_spatial_sampling:
|
||||
self.spatial_projectors = nn.ModuleList(
|
||||
[
|
||||
WindowQFormerDownsampler(
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
cache_config=cache_config,
|
||||
spatial_offset=i,
|
||||
prefix=maybe_prefix(prefix, f"spatial_projectors.{i}"),
|
||||
)
|
||||
for i in range(4)
|
||||
]
|
||||
)
|
||||
self.spatial_projectors = nn.ModuleList(
|
||||
[
|
||||
WindowQFormerDownsampler(
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
cache_config=cache_config,
|
||||
spatial_offset=i,
|
||||
prefix=maybe_prefix(prefix, f"spatial_projectors.{i}"),
|
||||
)
|
||||
for i in range(4)
|
||||
]
|
||||
)
|
||||
|
||||
# ----- Language model (marked as LM) -----
|
||||
with self._mark_language_model(vllm_config):
|
||||
@@ -545,12 +539,9 @@ class Granite4VisionForConditionalGeneration(
|
||||
|
||||
# Store config values we need
|
||||
self._deepstack_layer_map = config.deepstack_layer_map # [[-19, 9], ...]
|
||||
self._use_spatial_sampling = getattr(config, "use_spatial_sampling", False)
|
||||
self._spatial_vision_layer = getattr(config, "spatial_vision_layer", -1)
|
||||
self._spatial_target_layers = getattr(config, "spatial_target_layers", [])
|
||||
self._vision_feature_select_strategy = getattr(
|
||||
config, "vision_feature_select_strategy", "full"
|
||||
)
|
||||
self._spatial_vision_layer = config.spatial_vision_layer
|
||||
self._spatial_target_layers = config.spatial_target_layers
|
||||
self._vision_feature_select_strategy = config.vision_feature_select_strategy
|
||||
self._downsample_rate = Fraction(config.downsample_rate)
|
||||
|
||||
# Ordered list of LLM layer indices for each deepstack level.
|
||||
@@ -558,7 +549,7 @@ class Granite4VisionForConditionalGeneration(
|
||||
# (before any embed_multimodal call).
|
||||
self._ds_layer_indices: list[int] = [
|
||||
llm_layer for _, llm_layer in config.deepstack_layer_map
|
||||
] + list(getattr(config, "spatial_target_layers", []))
|
||||
] + list(config.spatial_target_layers)
|
||||
|
||||
# Share ds_layer_indices with the LLM causal model so
|
||||
# make_empty_intermediate_tensors includes the correct keys
|
||||
@@ -571,7 +562,7 @@ class Granite4VisionForConditionalGeneration(
|
||||
# the updated values written just before each prefill.
|
||||
# Shape: (max_num_batched_tokens, lm_hidden_size) per level.
|
||||
n_layerwise = len(config.deepstack_layer_map)
|
||||
n_spatial = len(getattr(config, "spatial_target_layers", []))
|
||||
n_spatial = len(config.spatial_target_layers)
|
||||
num_ds_levels = n_layerwise + n_spatial
|
||||
lm_hidden = config.text_config.hidden_size
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
@@ -644,26 +635,24 @@ class Granite4VisionForConditionalGeneration(
|
||||
)
|
||||
image_feature = unpad_image(image_feature, image_sizes[image_idx])
|
||||
|
||||
if self.image_newline is not None:
|
||||
image_feature = torch.cat(
|
||||
(
|
||||
image_feature,
|
||||
self.image_newline[:, None, None]
|
||||
.expand(*image_feature.shape[:-1], 1)
|
||||
.to(image_feature.device, image_feature.dtype),
|
||||
),
|
||||
dim=-1,
|
||||
)
|
||||
image_feature = torch.cat(
|
||||
(
|
||||
image_feature,
|
||||
self.image_newline[:, None, None]
|
||||
.expand(*image_feature.shape[:-1], 1)
|
||||
.to(image_feature.device, image_feature.dtype),
|
||||
),
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
image_feature = image_feature.flatten(1, 2).transpose(0, 1)
|
||||
image_feature = torch.cat((base_image_feature, image_feature), dim=0)
|
||||
else:
|
||||
image_feature = image_feature[0]
|
||||
if self.image_newline is not None:
|
||||
image_feature = torch.cat(
|
||||
(image_feature, self.image_newline[None].to(image_feature)),
|
||||
dim=0,
|
||||
)
|
||||
image_feature = torch.cat(
|
||||
(image_feature, self.image_newline[None].to(image_feature)),
|
||||
dim=0,
|
||||
)
|
||||
|
||||
new_image_features.append(image_feature)
|
||||
|
||||
@@ -718,16 +707,15 @@ class Granite4VisionForConditionalGeneration(
|
||||
)
|
||||
levels.append((llm_layer, per_image))
|
||||
|
||||
if self._use_spatial_sampling and self.spatial_projectors is not None:
|
||||
spatial_hidden = all_hidden_states[self._spatial_vision_layer]
|
||||
if select_strategy == "default":
|
||||
spatial_hidden = spatial_hidden[:, 1:]
|
||||
for group_idx, llm_layer in enumerate(self._spatial_target_layers):
|
||||
projected = self.spatial_projectors[group_idx](spatial_hidden)
|
||||
per_image = self._pack_and_unpad_image_features(
|
||||
torch.split(projected, image_num_patches, dim=0), image_sizes
|
||||
)
|
||||
levels.append((llm_layer, per_image))
|
||||
spatial_hidden = all_hidden_states[self._spatial_vision_layer]
|
||||
if select_strategy == "default":
|
||||
spatial_hidden = spatial_hidden[:, 1:]
|
||||
for group_idx, llm_layer in enumerate(self._spatial_target_layers):
|
||||
projected = self.spatial_projectors[group_idx](spatial_hidden)
|
||||
per_image = self._pack_and_unpad_image_features(
|
||||
torch.split(projected, image_num_patches, dim=0), image_sizes
|
||||
)
|
||||
levels.append((llm_layer, per_image))
|
||||
|
||||
llm_layer_indices = [llm_layer for llm_layer, _ in levels]
|
||||
num_images = len(image_sizes)
|
||||
|
||||
@@ -158,9 +158,7 @@ class GraniteSpeechMultiModalProcessor(
|
||||
feature_extractor = processor.audio_processor
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
# Use getattr with default to be compatible with transformers<4.48
|
||||
audio_token = getattr(processor, "audio_token", "<|audio|>")
|
||||
audio_token_id = vocab[audio_token]
|
||||
audio_token_id = vocab[processor.audio_token]
|
||||
|
||||
def get_replacement(item_idx: int):
|
||||
audios = mm_items.get_items("audio", AudioProcessorItems)
|
||||
@@ -221,8 +219,7 @@ class GraniteSpeechDummyInputsBuilder(
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_audios = mm_counts.get("audio", 0)
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
audio_token = getattr(hf_processor, "audio_token", "<|audio|>")
|
||||
return audio_token * num_audios
|
||||
return hf_processor.audio_token * num_audios
|
||||
|
||||
|
||||
### QFormer Projector
|
||||
|
||||
@@ -320,10 +320,6 @@ class GraniteMoeHybridAttention(nn.Module):
|
||||
|
||||
|
||||
ALL_DECODER_LAYER_TYPES = {
|
||||
# Transformers < 5.13.0
|
||||
"attention": GraniteMoeHybridAttentionDecoderLayer,
|
||||
"mamba": GraniteMoeHybridMambaDecoderLayer,
|
||||
# Transformers >= 5.13.0
|
||||
"full_attention": GraniteMoeHybridAttentionDecoderLayer,
|
||||
"linear_attention": GraniteMoeHybridMambaDecoderLayer,
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PreTrainedConfig
|
||||
from transformers import HYV3Config, PreTrainedConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
|
||||
@@ -67,7 +67,6 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.hy_v3 import HYV3Config
|
||||
|
||||
from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
|
||||
@@ -31,6 +31,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import HyperCLOVAXConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
@@ -51,7 +52,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.hyperclovax import HyperCLOVAXConfig
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
@@ -137,9 +137,7 @@ class HyperCLOVAXAttention(nn.Module):
|
||||
# the KV heads across multiple tensor parallel GPUs.
|
||||
assert tp_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
self.head_dim = getattr(
|
||||
config, "head_dim", self.hidden_size // self.total_num_heads
|
||||
)
|
||||
self.head_dim = config.head_dim
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
self.scaling = config.attention_multiplier
|
||||
@@ -166,7 +164,7 @@ class HyperCLOVAXAttention(nn.Module):
|
||||
self.head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
is_neox_style=True,
|
||||
rope_parameters=getattr(config, "rope_parameters", None),
|
||||
rope_parameters=config.rope_parameters,
|
||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||
)
|
||||
|
||||
@@ -217,18 +215,15 @@ class HyperCLOVAXDecoderLayer(nn.Module):
|
||||
"dual_chunk_attention_config",
|
||||
None,
|
||||
)
|
||||
attention_bias = getattr(config, "attention_bias", False)
|
||||
|
||||
self.self_attn = HyperCLOVAXAttention(
|
||||
config=config,
|
||||
hidden_size=self.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
num_kv_heads=getattr(
|
||||
config, "num_key_value_heads", config.num_attention_heads
|
||||
),
|
||||
num_kv_heads=config.num_key_value_heads,
|
||||
max_position_embeddings=max_position_embeddings,
|
||||
quant_config=quant_config,
|
||||
bias=attention_bias,
|
||||
bias=config.attention_bias,
|
||||
cache_config=cache_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||
@@ -238,7 +233,7 @@ class HyperCLOVAXDecoderLayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
bias=getattr(config, "mlp_bias", False),
|
||||
bias=config.mlp_bias,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Literal
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import BatchFeature
|
||||
from transformers import BatchFeature, Kimi_K25Config
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
@@ -63,7 +63,6 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.kimi_k25 import KimiK25Config
|
||||
from vllm.transformers_utils.processor import cached_get_image_processor
|
||||
from vllm.transformers_utils.processors.kimi_k25 import KimiK25Processor
|
||||
from vllm.transformers_utils.processors.kimi_k25_vision_fused import (
|
||||
@@ -135,7 +134,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
# Resolve token ID from the tokenizer because transformers v5
|
||||
# may remap token IDs vs config.json.
|
||||
config_token_id = hf_config.media_placeholder_token_id
|
||||
config_token_id = hf_config.image_token_id
|
||||
resolved_token_id = tokenizer.convert_tokens_to_ids("<|media_pad|>")
|
||||
unk_token_id = getattr(tokenizer, "unk_token_id", None)
|
||||
is_valid_resolved = isinstance(resolved_token_id, int) and (
|
||||
@@ -143,7 +142,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
|
||||
)
|
||||
if is_valid_resolved and resolved_token_id != config_token_id:
|
||||
logger.warning_once(
|
||||
"Kimi-K2.5 config.media_placeholder_token_id (%d) disagrees "
|
||||
"Kimi-K2.5 config.image_token_id (%d) disagrees "
|
||||
"with tokenizer mapping for <|media_pad|> (%d). "
|
||||
"Using tokenizer value.",
|
||||
config_token_id,
|
||||
@@ -151,7 +150,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
|
||||
)
|
||||
media_token_id = resolved_token_id
|
||||
# Patch config so downstream code also sees the correct ID.
|
||||
hf_config.media_placeholder_token_id = resolved_token_id
|
||||
hf_config.image_token_id = resolved_token_id
|
||||
else:
|
||||
media_token_id = config_token_id
|
||||
|
||||
@@ -170,7 +169,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
|
||||
return self.hf_processor
|
||||
|
||||
def get_hf_config(self):
|
||||
return self.ctx.get_hf_config(KimiK25Config)
|
||||
return self.ctx.get_hf_config(Kimi_K25Config)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
# None means unlimited
|
||||
@@ -338,7 +337,7 @@ class KimiK25ForConditionalGeneration(
|
||||
) -> None:
|
||||
super().__init__()
|
||||
model_config = vllm_config.model_config
|
||||
config: KimiK25Config = model_config.hf_config
|
||||
config: Kimi_K25Config = model_config.hf_config
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
@@ -347,7 +346,6 @@ class KimiK25ForConditionalGeneration(
|
||||
)
|
||||
self.hidden_size = config.text_config.hidden_size
|
||||
self.device = current_platform.current_device()
|
||||
# Build vision tower directly with KimiK25VisionConfig
|
||||
with self._mark_tower_model(vllm_config, "vision_chunk"):
|
||||
self.vision_tower = MoonViT3dPretrainedModel(
|
||||
config.vision_config,
|
||||
@@ -364,6 +362,7 @@ class KimiK25ForConditionalGeneration(
|
||||
|
||||
self.mm_projector = KimiK25MultiModalProjector(
|
||||
config=config.vision_config,
|
||||
out_hidden_size=config.text_config.hidden_size,
|
||||
use_data_parallel=self.use_data_parallel,
|
||||
quant_config=self._maybe_ignore_quant_config(quant_config),
|
||||
prefix=maybe_prefix(prefix, "mm_projector"),
|
||||
@@ -383,7 +382,7 @@ class KimiK25ForConditionalGeneration(
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
self.media_placeholder: int = self.config.media_placeholder_token_id
|
||||
self.media_placeholder: int = self.config.image_token_id
|
||||
|
||||
def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig | None):
|
||||
if isinstance(quant_config, compressed_tensors.CompressedTensorsConfig):
|
||||
|
||||
@@ -15,6 +15,7 @@ import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from transformers import Kimi_K25VisionConfig
|
||||
from transformers.activations import GELUActivation
|
||||
|
||||
from vllm.distributed import divide, get_tensor_model_parallel_world_size
|
||||
@@ -40,13 +41,21 @@ from vllm.model_executor.models.vision import (
|
||||
run_dp_sharded_mrope_vision_model,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig
|
||||
from vllm.transformers_utils.configs.kimi_k3 import KimiK3VisionConfig
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_pos_emb_size(
|
||||
config: Kimi_K25VisionConfig | KimiK3VisionConfig, dim: str
|
||||
) -> int:
|
||||
# Legacy checkpoints and Kimi-K3 use `init_pos_emb_*`
|
||||
legacy = getattr(config, f"init_pos_emb_{dim}", None)
|
||||
return legacy if legacy is not None else getattr(config, f"pos_emb_{dim}")
|
||||
|
||||
|
||||
def _apply_rope_input_validation(x, freqs_cis):
|
||||
assert x.ndim == freqs_cis.ndim + 1, (x.shape, freqs_cis.shape)
|
||||
assert x.shape[:-2] == freqs_cis.shape[:-1], (x.shape, freqs_cis.shape)
|
||||
@@ -698,14 +707,11 @@ def tpool_patch_merger_packed(
|
||||
|
||||
|
||||
class MoonViT3dPretrainedModel(nn.Module):
|
||||
"""Main vision tower model.
|
||||
|
||||
Uses KimiK25VisionConfig directly from transformers_utils/configs/kimi_k25.py.
|
||||
"""
|
||||
"""Main vision tower model."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiK25VisionConfig,
|
||||
config: Kimi_K25VisionConfig | KimiK3VisionConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
input_norm: nn.Module | None = None,
|
||||
prefix: str = "",
|
||||
@@ -713,17 +719,18 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
super().__init__()
|
||||
config = deepcopy(config)
|
||||
self.config = config # Required for run_dp_sharded_mrope_vision_model
|
||||
self.merge_kernel_size = config.merge_kernel_size
|
||||
merge_h, merge_w = config.merge_kernel_size
|
||||
self.merge_kernel_size = (merge_h, merge_w)
|
||||
self.patch_size = config.patch_size
|
||||
self.merge_type = config.merge_type
|
||||
self.merge_type = getattr(config, "merge_type", "sd2_tpool")
|
||||
|
||||
self.patch_embed = MoonVision3dPatchEmbed(
|
||||
out_dim=config.hidden_size,
|
||||
patch_size=config.patch_size,
|
||||
pos_emb_height=config.init_pos_emb_height,
|
||||
pos_emb_width=config.init_pos_emb_width,
|
||||
pos_emb_time=config.init_pos_emb_time,
|
||||
pos_emb_type=config.pos_emb_type,
|
||||
pos_emb_height=_get_pos_emb_size(config, "height"),
|
||||
pos_emb_width=_get_pos_emb_size(config, "width"),
|
||||
pos_emb_time=_get_pos_emb_size(config, "time"),
|
||||
pos_emb_type=getattr(config, "pos_emb_type", "divided_fixed"),
|
||||
patch_embed_proj_bias=getattr(config, "patch_embed_proj_bias", True),
|
||||
pos_emb_interpolation_mode=getattr(
|
||||
config, "pos_emb_interpolation_mode", "bicubic"
|
||||
@@ -740,14 +747,14 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
"qkv_hidden_size": getattr(config, "qkv_hidden_size", None),
|
||||
"mlp_dim": config.intermediate_size,
|
||||
"activation": get_act_fn(
|
||||
getattr(config, "activation_func", "gelu_pytorch_tanh")
|
||||
getattr(config, "activation_func", None) or config.hidden_act
|
||||
),
|
||||
"attn_bias": getattr(config, "attn_bias", True),
|
||||
"norm_type": getattr(config, "norm_type", "layernorm"),
|
||||
"mlp_type": getattr(config, "mlp_type", "mlp2"),
|
||||
"linear_bias": getattr(config, "linear_bias", True),
|
||||
},
|
||||
video_attn_type=config.video_attn_type,
|
||||
video_attn_type=getattr(config, "video_attn_type", "spatial_temporal"),
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "encoder"),
|
||||
)
|
||||
@@ -886,7 +893,8 @@ class KimiK25MultiModalProjector(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiK25VisionConfig,
|
||||
config: Kimi_K25VisionConfig | KimiK3VisionConfig,
|
||||
out_hidden_size: int,
|
||||
use_data_parallel: bool = False,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
@@ -909,13 +917,13 @@ class KimiK25MultiModalProjector(nn.Module):
|
||||
)
|
||||
self.linear_2 = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
getattr(config, "text_hidden_size", config.mm_hidden_size),
|
||||
out_hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_2",
|
||||
)
|
||||
self.post_norm = torch.nn.RMSNorm(
|
||||
getattr(config, "text_hidden_size", config.mm_hidden_size),
|
||||
out_hidden_size,
|
||||
eps=config.projector_ln_eps,
|
||||
)
|
||||
self.act = GELUActivation()
|
||||
@@ -931,7 +939,7 @@ class KimiK25MultiModalProjector(nn.Module):
|
||||
)
|
||||
self.linear_2 = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
config.mm_hidden_size,
|
||||
out_hidden_size,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_2",
|
||||
|
||||
@@ -5,6 +5,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import Lfm2MoeConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
@@ -40,7 +41,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.lfm2_moe import Lfm2MoeConfig
|
||||
|
||||
from .interfaces import (
|
||||
HasInnerState,
|
||||
|
||||
@@ -94,28 +94,12 @@ class ModernBertAttention(nn.Module):
|
||||
prefix=f"{prefix}.Wqkv",
|
||||
)
|
||||
|
||||
if layer_types := getattr(config, "layer_types", None):
|
||||
# Transformers v5
|
||||
layer_type = layer_types[layer_id]
|
||||
rope_parameters = config.rope_parameters[layer_type]
|
||||
sliding_window: int | None = None
|
||||
if layer_type == "sliding_attention":
|
||||
# Treats the local attention boundary as inclusive
|
||||
sliding_window = config.sliding_window + 1
|
||||
else:
|
||||
# Transformers v4
|
||||
sliding_window = None
|
||||
if layer_id % config.global_attn_every_n_layers != 0:
|
||||
# ModernBertConfig does not expose sliding_window
|
||||
sliding_window = config.local_attention // 2
|
||||
rope_theta = (
|
||||
config.local_rope_theta
|
||||
if config.local_rope_theta is not None
|
||||
else config.global_rope_theta
|
||||
)
|
||||
else:
|
||||
rope_theta = config.global_rope_theta
|
||||
rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
|
||||
layer_type = config.layer_types[layer_id]
|
||||
rope_parameters = config.rope_parameters[layer_type]
|
||||
sliding_window: int | None = None
|
||||
if layer_type == "sliding_attention":
|
||||
# Treats the local attention boundary as inclusive
|
||||
sliding_window = config.sliding_window + 1
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
head_size=self.head_dim,
|
||||
|
||||
@@ -29,6 +29,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import NemotronConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
@@ -49,7 +50,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.nemotron import NemotronConfig
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
|
||||
@@ -23,6 +23,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import NemotronHConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import (
|
||||
@@ -87,7 +88,6 @@ from vllm.model_executor.models.utils import (
|
||||
sequence_parallel_chunk,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
|
||||
|
||||
|
||||
class NemotronHMLP(nn.Module):
|
||||
|
||||
@@ -7,6 +7,7 @@ from collections.abc import Callable, Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import NemotronHConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
@@ -33,7 +34,6 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
|
||||
|
||||
from .interfaces import SupportsPP, SupportsQuant
|
||||
from .nemotron_h import (
|
||||
|
||||
@@ -125,7 +125,7 @@ class VisualTokenizer(torch.nn.Module):
|
||||
prefix: str = "",
|
||||
) -> nn.Module:
|
||||
model_type = config.backbone_config.model_type
|
||||
if model_type == "aimv2":
|
||||
if model_type == "aimv2_vision_model":
|
||||
# No post rms_norm in Ovis2's AIMv2 ViT.
|
||||
return AIMv2Model(
|
||||
config=config.backbone_config,
|
||||
|
||||
@@ -29,6 +29,12 @@ from typing import ClassVar
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import (
|
||||
Qwen3_5Config,
|
||||
Qwen3_5MoeConfig,
|
||||
Qwen3_5MoeTextConfig,
|
||||
Qwen3_5TextConfig,
|
||||
)
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
@@ -59,11 +65,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig
|
||||
from vllm.transformers_utils.configs.qwen3_5_moe import (
|
||||
Qwen3_5MoeConfig,
|
||||
Qwen3_5MoeTextConfig,
|
||||
)
|
||||
|
||||
from .interfaces import (
|
||||
HasInnerState,
|
||||
|
||||
@@ -6,6 +6,7 @@ from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen3_5MoeTextConfig, Qwen3_5TextConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
@@ -32,8 +33,6 @@ from vllm.model_executor.models.qwen3_next import (
|
||||
)
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5TextConfig
|
||||
from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig
|
||||
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
|
||||
@@ -7,6 +7,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen3NextConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
@@ -58,7 +59,6 @@ from vllm.model_executor.models.qwen2_moe import Qwen2MoeMLP as Qwen3NextMLP
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
|
||||
from .interfaces import (
|
||||
|
||||
@@ -6,6 +6,7 @@ from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen3NextConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
@@ -29,7 +30,6 @@ from vllm.model_executor.models.qwen3_next import (
|
||||
)
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
|
||||
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
|
||||
Reference in New Issue
Block a user