Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -29,6 +29,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import AXK1Config
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
@@ -78,7 +79,6 @@ from vllm.model_executor.models.deepseek_v2 import (
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.AXK1 import AXK1Config
|
||||
|
||||
from .interfaces import MixtureOfExperts, SupportsEagle, SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
@@ -199,6 +199,7 @@ class AXK1MoE(nn.Module):
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else None,
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
shared_expert_prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -7,6 +7,7 @@ from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import Aimv2VisionConfig
|
||||
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.distributed.utils import divide
|
||||
@@ -21,20 +22,19 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
|
||||
from vllm.transformers_utils.configs.ovis import AIMv2Config
|
||||
|
||||
|
||||
class AIMv2SwiGLUFFN(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: AIMv2Config,
|
||||
config: Aimv2VisionConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
):
|
||||
super().__init__()
|
||||
hidden_features = config.intermediate_size
|
||||
in_features = config.hidden_size
|
||||
bias = config.use_bias
|
||||
bias = config.mlp_bias
|
||||
|
||||
self.fc13 = MergedColumnParallelLinear(
|
||||
in_features,
|
||||
@@ -60,7 +60,7 @@ class AIMv2SwiGLUFFN(nn.Module):
|
||||
|
||||
|
||||
class AIMv2PatchEmbed(nn.Module):
|
||||
def __init__(self, config: AIMv2Config):
|
||||
def __init__(self, config: Aimv2VisionConfig):
|
||||
super().__init__()
|
||||
self.proj = Conv2dLayer(
|
||||
config.num_channels,
|
||||
@@ -77,7 +77,7 @@ class AIMv2PatchEmbed(nn.Module):
|
||||
|
||||
|
||||
class AIMv2ViTPreprocessor(nn.Module):
|
||||
def __init__(self, config: AIMv2Config):
|
||||
def __init__(self, config: Aimv2VisionConfig):
|
||||
super().__init__()
|
||||
num_patches = (config.image_size // config.patch_size) ** 2
|
||||
|
||||
@@ -95,7 +95,7 @@ class AIMv2ViTPreprocessor(nn.Module):
|
||||
class AIMv2Attention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: AIMv2Config,
|
||||
config: Aimv2VisionConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
):
|
||||
@@ -124,7 +124,7 @@ class AIMv2Attention(nn.Module):
|
||||
self.proj = RowParallelLinear(
|
||||
input_size=self.embed_dim,
|
||||
output_size=self.embed_dim,
|
||||
bias=config.use_bias,
|
||||
bias=config.mlp_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.proj",
|
||||
)
|
||||
@@ -151,7 +151,7 @@ class AIMv2Attention(nn.Module):
|
||||
class AIMv2Block(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: AIMv2Config,
|
||||
config: Aimv2VisionConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
):
|
||||
@@ -174,7 +174,7 @@ class AIMv2Block(nn.Module):
|
||||
class AIMv2Transformer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: AIMv2Config,
|
||||
config: Aimv2VisionConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
*,
|
||||
require_post_norm: bool | None = None,
|
||||
@@ -214,7 +214,7 @@ class AIMv2Model(torch.nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: AIMv2Config,
|
||||
config: Aimv2VisionConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
*,
|
||||
require_post_norm: bool | None = None,
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
@@ -520,8 +521,12 @@ class BailingMoeV3VLForConditionalGeneration(
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
return num_image_tokens * self.visual.spatial_merge_size**2
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
return num_vision_tokens // self.visual.spatial_merge_size**2
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
return num_mm_embeds * self.visual.spatial_merge_size**2, num_mm_embeds
|
||||
|
||||
@@ -403,20 +403,6 @@ class Cohere2MoeModel(nn.Module, EagleModelMixin):
|
||||
config.vocab_size, config.hidden_size
|
||||
)
|
||||
|
||||
# Decoder layers read per-layer MLP layout from config.mlp_layer_types
|
||||
# (dense MLP vs MoE) and use it for weight loading. Transformers >=5.10
|
||||
# populates this field; older versions only expose first_k_dense_replace.
|
||||
# Normalize here so layer construction below sees a consistent layout.
|
||||
if getattr(config, "mlp_layer_types", None) is None:
|
||||
first_k_dense_replace = getattr(config, "first_k_dense_replace", None)
|
||||
n = config.num_hidden_layers
|
||||
if first_k_dense_replace is not None:
|
||||
config.mlp_layer_types = ["dense"] * first_k_dense_replace + [
|
||||
"sparse"
|
||||
] * (n - first_k_dense_replace)
|
||||
else:
|
||||
config.mlp_layer_types = ["sparse"] * n
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: Cohere2MoeDecoderLayer(
|
||||
|
||||
@@ -82,6 +82,7 @@ from vllm.multimodal.inputs import (
|
||||
ImageItem,
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import (
|
||||
@@ -2085,24 +2086,16 @@ class CohereCompassForConditionalGeneration(
|
||||
tower_model="visual.",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
num_image_tokens: int,
|
||||
) -> int:
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
|
||||
return num_image_tokens * merge_size**2
|
||||
|
||||
def get_num_mm_connector_tokens(
|
||||
self,
|
||||
num_vision_tokens: int,
|
||||
) -> int:
|
||||
hf_config = self.config
|
||||
vision_config = hf_config.vision_config
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
return num_vision_tokens // merge_size**2
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
del modality, mm_kwargs
|
||||
merge_size = self.config.vision_config.spatial_merge_size
|
||||
return num_mm_embeds * merge_size**2, num_mm_embeds
|
||||
|
||||
|
||||
@lru_cache
|
||||
|
||||
@@ -43,6 +43,26 @@ class DeepseekV32ForCausalLM(VerifyAndUpdateConfig):
|
||||
class GlmMoeDsaForCausalLM(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
if (
|
||||
cache_config.cache_dtype == "auto"
|
||||
and current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
):
|
||||
cache_config.cache_dtype = "fp8_e4m3"
|
||||
logger.info_once("Using fp8 kv-cache for GlmMoeDsaForCausalLM on SM10x")
|
||||
# MTP shares the target's KV cache layout; a separate draft model
|
||||
# must not inherit a default it was never validated with.
|
||||
spec_config = vllm_config.speculative_config
|
||||
if (
|
||||
spec_config is not None
|
||||
and spec_config.method != "mtp"
|
||||
and spec_config.kv_cache_dtype is None
|
||||
):
|
||||
spec_config.kv_cache_dtype = "auto"
|
||||
|
||||
# For Glm-Moe-DSA, qrep + a2a is better than the default all-gather + ag-rs
|
||||
# in most cases.
|
||||
vllm_config.parallel_config.set_dcp_defaults(
|
||||
@@ -270,6 +290,31 @@ class Gemma4Config(VerifyAndUpdateConfig):
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingGemma2ModelConfig(Gemma4Config):
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
Gemma4Config.verify_and_update_config(vllm_config)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
attn_config = vllm_config.attention_config
|
||||
if attn_config.backend is None:
|
||||
attn_config.backend = AttentionBackendEnum.TRITON_ATTN
|
||||
logger.info(
|
||||
"EmbeddingGemma2: attention backend not specified; defaulting to "
|
||||
"TRITON_ATTN (validated backend for heterogeneous head dimensions)."
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"EmbeddingGemma2: using attention backend %s", attn_config.backend
|
||||
)
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
if model_config.max_model_len > 8192 and getattr(
|
||||
model_config, "original_max_model_len", None
|
||||
) in (None, -1):
|
||||
model_config.max_model_len = 8192
|
||||
|
||||
|
||||
class DiffusionGemmaModelForBlockDiffusionConfig(VerifyAndUpdateConfig):
|
||||
@classmethod
|
||||
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
|
||||
@@ -1013,6 +1058,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"DeepseekV41ForCausalLM": DeepseekV4ForCausalLMConfig,
|
||||
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
|
||||
"DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501
|
||||
"EmbeddingGemma2Model": EmbeddingGemma2ModelConfig,
|
||||
"Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501
|
||||
"FalconMambaForCausalLM": MambaModelConfig,
|
||||
"Gemma3TextModel": Gemma3TextModelConfig,
|
||||
|
||||
@@ -2,11 +2,28 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import regex
|
||||
from transformers import Cosmos3OmniConfig
|
||||
|
||||
from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration
|
||||
from vllm.model_executor.models.qwen3_vl import (
|
||||
Qwen3VLDummyInputsBuilder,
|
||||
Qwen3VLForConditionalGeneration,
|
||||
Qwen3VLMultiModalProcessor,
|
||||
Qwen3VLProcessingInfo,
|
||||
)
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
|
||||
class Cosmos3ProcessingInfo(Qwen3VLProcessingInfo):
|
||||
def get_hf_config(self) -> Cosmos3OmniConfig:
|
||||
return self.ctx.get_hf_config(Cosmos3OmniConfig)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
Qwen3VLMultiModalProcessor,
|
||||
info=Cosmos3ProcessingInfo,
|
||||
dummy_inputs=Qwen3VLDummyInputsBuilder,
|
||||
)
|
||||
class Cosmos3ForConditionalGeneration(Qwen3VLForConditionalGeneration):
|
||||
# Cosmos3 unified checkpoints store a Qwen3-VL understanding tower
|
||||
# alongside a generation tower in a flat key layout. This mapper drops
|
||||
|
||||
@@ -5,7 +5,7 @@ from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import ProcessorMixin
|
||||
from transformers import NemotronHConfig, ProcessorMixin
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -20,7 +20,6 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
from .interfaces import (
|
||||
|
||||
@@ -396,6 +396,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else None,
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
shared_expert_prefix=f"{prefix}.shared_experts",
|
||||
router_logits_dtype=self.gate.out_dtype,
|
||||
)
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ via Gemma4MultimodalEmbedder.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from collections.abc import Iterable, Sequence
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
@@ -25,7 +25,7 @@ import torch
|
||||
import torch._dynamo
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
from transformers import AutoModel
|
||||
from transformers import AutoModel, DiffusionGemmaConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
@@ -113,26 +113,13 @@ class DiffusionGemmaSelfConditioning(nn.Module):
|
||||
|
||||
|
||||
class DiffusionGemmaProcessingInfo(Gemma4ProcessingInfo):
|
||||
"""Processing info for DiffusionGemma.
|
||||
"""Processing info for DiffusionGemma, which has no audio tower."""
|
||||
|
||||
Overrides ``get_hf_config`` to accept ``DiffusionGemmaConfig``
|
||||
(which inherits from ``PreTrainedConfig``, not ``Gemma4Config``).
|
||||
Supports image and video modalities.
|
||||
"""
|
||||
def get_hf_config(self) -> DiffusionGemmaConfig:
|
||||
return self.ctx.get_hf_config(DiffusionGemmaConfig)
|
||||
|
||||
def get_hf_config(self):
|
||||
# DiffusionGemmaConfig doesn't inherit from Gemma4Config, so we
|
||||
# accept any PreTrainedConfig here.
|
||||
return self.ctx.get_hf_config()
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
# DiffusionGemma supports image and video inputs.
|
||||
return {"image": None, "video": None}
|
||||
|
||||
def get_mm_max_tokens_per_item(
|
||||
self, seq_len: int, mm_counts: Mapping[str, int]
|
||||
) -> Mapping[str, int] | None:
|
||||
return super().get_mm_max_tokens_per_item(seq_len, mm_counts)
|
||||
def has_audio(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
|
||||
@@ -445,11 +432,6 @@ def _mask_rows_to_allowed(
|
||||
return logits if out is None else out
|
||||
|
||||
|
||||
# Tiles specialize the graph on batch size 1, on a tile as wide as the canvas,
|
||||
# on compute_sc and on sizes that coincide with the state buffers. That set
|
||||
# is small but passes Dynamo's default of 8, after which every step would run
|
||||
# eager: about twice as slow for a self-conditioned step.
|
||||
@torch._dynamo.config.patch(recompile_limit=64)
|
||||
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
|
||||
def _denoise_temperature(
|
||||
step_tensor: torch.Tensor,
|
||||
@@ -464,6 +446,12 @@ def _denoise_temperature(
|
||||
return t_min + (t_max - t_min) * (remaining / max_denoising_steps)
|
||||
|
||||
|
||||
# Tiles specialize the graph on batch size 1, on a tile as wide as the canvas,
|
||||
# on compute_sc and on sizes that coincide with the state buffers. That set
|
||||
# is small but passes Dynamo's default of 8, after which every step would run
|
||||
# eager: about twice as slow for a self-conditioned step.
|
||||
@torch._dynamo.config.patch(recompile_limit=64)
|
||||
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
|
||||
def _compiled_sample_step(
|
||||
# Per-position statistics of the temperature-scaled logits, from
|
||||
# sample_row_stats: [num_decode, CL] each, and the softmax
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -569,6 +569,7 @@ class FalconH1ForCausalLM(
|
||||
head_dim=hf_config.mamba_d_head,
|
||||
state_size=hf_config.mamba_d_state,
|
||||
conv_kernel=hf_config.mamba_d_conv,
|
||||
num_spec=vllm_config.num_speculative_tokens,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -154,20 +154,7 @@ class Gemma3Attention(nn.Module):
|
||||
self.is_sliding = layer_type == "sliding_attention"
|
||||
sliding_window = config.sliding_window if self.is_sliding else None
|
||||
|
||||
# Initialize the rotary embedding.
|
||||
if layer_type in config.rope_parameters:
|
||||
# Transformers v5 rope config.
|
||||
rope_parameters = config.rope_parameters[layer_type]
|
||||
else:
|
||||
# Transformers v4 rope config.
|
||||
# Global attention. Use the values in config.json.
|
||||
rope_parameters = config.rope_parameters
|
||||
# Local attention. Override the values in config.json.
|
||||
if self.is_sliding:
|
||||
rope_parameters = dict(
|
||||
rope_type="default", rope_theta=config.rope_local_base_freq
|
||||
)
|
||||
|
||||
rope_parameters = config.rope_parameters[layer_type]
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
|
||||
@@ -350,17 +350,7 @@ class Gemma3nAttention(nn.Module):
|
||||
is_sliding = layer_type == "sliding_attention"
|
||||
self.sliding_window = config.sliding_window if is_sliding else None
|
||||
|
||||
# Initialize the rotary embedding.
|
||||
if layer_type in config.rope_parameters:
|
||||
# Transformers v5 rope config.
|
||||
rope_parameters = config.rope_parameters[layer_type]
|
||||
else:
|
||||
# Transformers v4 rope config.
|
||||
# Global attention. Use the values in config.json.
|
||||
rope_parameters = config.rope_parameters.copy()
|
||||
# Local attention. Override the values in config.json.
|
||||
if is_sliding:
|
||||
rope_parameters["rope_theta"] = config.rope_local_base_freq
|
||||
rope_parameters = config.rope_parameters[layer_type]
|
||||
|
||||
kv_sharing_target_layer_name = None
|
||||
if self.is_kv_shared_layer:
|
||||
|
||||
@@ -59,7 +59,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import KVSharingFastPrefillMetadata
|
||||
|
||||
@@ -576,7 +575,7 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
self.layer_idx = layer_idx
|
||||
|
||||
# Gemma4 uses different head dimensions for sliding vs full attention
|
||||
layer_config = gemma4_layer_config(config, layer_idx)
|
||||
layer_config = config.per_layer_config[layer_idx]
|
||||
head_dim = layer_config.head_dim
|
||||
num_kv_heads = layer_config.num_key_value_heads
|
||||
|
||||
@@ -628,9 +627,7 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
# MoE (Mixture of Experts) — router + expert block parallel to MLP
|
||||
self.enable_moe_block = getattr(config, "enable_moe_block", False) or getattr(
|
||||
config, "use_second_mlp_block", False
|
||||
)
|
||||
self.enable_moe_block = getattr(config, "num_experts", None) is not None
|
||||
self.router: Gemma4Router | None
|
||||
self.experts: MoERunner | None
|
||||
self.post_feedforward_layernorm_1: RMSNorm | None
|
||||
|
||||
@@ -20,7 +20,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
|
||||
|
||||
from .gemma4_mtp import Gemma4MTPAttention, Gemma4MTPDecoderLayer
|
||||
from .qwen3_dflash import DFlashQwen3Model, _dflash_layer_causal
|
||||
@@ -48,7 +47,7 @@ class Gemma4DSparkAttention(Gemma4MTPAttention):
|
||||
config, "attention_k_eq_v", False
|
||||
)
|
||||
|
||||
layer_config = gemma4_layer_config(config, layer_idx)
|
||||
layer_config = config.per_layer_config[layer_idx]
|
||||
head_dim = layer_config.head_dim
|
||||
num_kv_heads = layer_config.num_key_value_heads
|
||||
super().__init__(
|
||||
|
||||
@@ -222,12 +222,11 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def has_audio(self) -> bool:
|
||||
return self.get_hf_config().audio_config is not None
|
||||
|
||||
def validate_num_items(self, modality: str, num_items: int) -> None:
|
||||
if (
|
||||
modality == "audio"
|
||||
and num_items > 0
|
||||
and self.get_hf_config().audio_config is None
|
||||
):
|
||||
if modality == "audio" and num_items > 0 and not self.has_audio():
|
||||
model_config = self.ctx.model_config
|
||||
model = get_served_model_name(
|
||||
model_config.model, model_config.served_model_name
|
||||
@@ -243,7 +242,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
limits: dict[str, int | None] = {"image": None}
|
||||
if self.get_hf_config().audio_config is not None:
|
||||
if self.has_audio():
|
||||
limits["audio"] = None
|
||||
limits["video"] = None
|
||||
return limits
|
||||
@@ -261,7 +260,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
if isinstance(val, int) and val in _SUPPORTED_SOFT_TOKENS:
|
||||
tokens_per_image = val
|
||||
tokens: dict[str, int] = {"image": tokens_per_image}
|
||||
if config.audio_config is not None:
|
||||
if self.has_audio():
|
||||
# Audio max tokens from the processor's audio_seq_length.
|
||||
processor = self.get_hf_processor()
|
||||
tokens["audio"] = processor.audio_seq_length
|
||||
@@ -275,9 +274,8 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
return tokens
|
||||
|
||||
def get_data_parser(self) -> MultiModalDataParser:
|
||||
config = self.get_hf_config()
|
||||
kwargs: dict[str, Any] = {"video_needs_metadata": True}
|
||||
if getattr(config, "audio_config", None) is not None:
|
||||
if self.has_audio():
|
||||
processor = self.get_hf_processor()
|
||||
kwargs["target_sr"] = processor.feature_extractor.sampling_rate
|
||||
return MultiModalDataParser(**kwargs)
|
||||
|
||||
@@ -45,7 +45,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.gemma4 import gemma4_layer_config
|
||||
|
||||
from .gemma4 import Gemma4MLP, _get_text_config
|
||||
from .utils import (
|
||||
@@ -265,7 +264,7 @@ class Gemma4MTPDecoderLayer(nn.Module):
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
layer_config = gemma4_layer_config(config, layer_idx)
|
||||
layer_config = config.per_layer_config[layer_idx]
|
||||
head_dim = layer_config.head_dim
|
||||
num_kv_heads = layer_config.num_key_value_heads
|
||||
|
||||
|
||||
@@ -204,6 +204,7 @@ class Glm4MoE(nn.Module):
|
||||
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
|
||||
),
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
shared_expert_prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -493,13 +493,9 @@ class Granite4VisionForConditionalGeneration(
|
||||
prefix=maybe_prefix(prefix, "vision_tower"),
|
||||
)
|
||||
|
||||
# image_newline parameter
|
||||
if config.use_image_newline_parameter:
|
||||
self.image_newline = nn.Parameter(
|
||||
torch.empty(config.text_config.hidden_size)
|
||||
)
|
||||
else:
|
||||
self.image_newline = None
|
||||
self.image_newline = nn.Parameter(
|
||||
torch.empty(config.text_config.hidden_size)
|
||||
)
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
|
||||
@@ -517,20 +513,18 @@ class Granite4VisionForConditionalGeneration(
|
||||
)
|
||||
|
||||
# Spatial projectors: 4 offset groups
|
||||
self.spatial_projectors = None
|
||||
if config.use_spatial_sampling:
|
||||
self.spatial_projectors = nn.ModuleList(
|
||||
[
|
||||
WindowQFormerDownsampler(
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
cache_config=cache_config,
|
||||
spatial_offset=i,
|
||||
prefix=maybe_prefix(prefix, f"spatial_projectors.{i}"),
|
||||
)
|
||||
for i in range(4)
|
||||
]
|
||||
)
|
||||
self.spatial_projectors = nn.ModuleList(
|
||||
[
|
||||
WindowQFormerDownsampler(
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
cache_config=cache_config,
|
||||
spatial_offset=i,
|
||||
prefix=maybe_prefix(prefix, f"spatial_projectors.{i}"),
|
||||
)
|
||||
for i in range(4)
|
||||
]
|
||||
)
|
||||
|
||||
# ----- Language model (marked as LM) -----
|
||||
with self._mark_language_model(vllm_config):
|
||||
@@ -545,12 +539,9 @@ class Granite4VisionForConditionalGeneration(
|
||||
|
||||
# Store config values we need
|
||||
self._deepstack_layer_map = config.deepstack_layer_map # [[-19, 9], ...]
|
||||
self._use_spatial_sampling = getattr(config, "use_spatial_sampling", False)
|
||||
self._spatial_vision_layer = getattr(config, "spatial_vision_layer", -1)
|
||||
self._spatial_target_layers = getattr(config, "spatial_target_layers", [])
|
||||
self._vision_feature_select_strategy = getattr(
|
||||
config, "vision_feature_select_strategy", "full"
|
||||
)
|
||||
self._spatial_vision_layer = config.spatial_vision_layer
|
||||
self._spatial_target_layers = config.spatial_target_layers
|
||||
self._vision_feature_select_strategy = config.vision_feature_select_strategy
|
||||
self._downsample_rate = Fraction(config.downsample_rate)
|
||||
|
||||
# Ordered list of LLM layer indices for each deepstack level.
|
||||
@@ -558,7 +549,7 @@ class Granite4VisionForConditionalGeneration(
|
||||
# (before any embed_multimodal call).
|
||||
self._ds_layer_indices: list[int] = [
|
||||
llm_layer for _, llm_layer in config.deepstack_layer_map
|
||||
] + list(getattr(config, "spatial_target_layers", []))
|
||||
] + list(config.spatial_target_layers)
|
||||
|
||||
# Share ds_layer_indices with the LLM causal model so
|
||||
# make_empty_intermediate_tensors includes the correct keys
|
||||
@@ -571,7 +562,7 @@ class Granite4VisionForConditionalGeneration(
|
||||
# the updated values written just before each prefill.
|
||||
# Shape: (max_num_batched_tokens, lm_hidden_size) per level.
|
||||
n_layerwise = len(config.deepstack_layer_map)
|
||||
n_spatial = len(getattr(config, "spatial_target_layers", []))
|
||||
n_spatial = len(config.spatial_target_layers)
|
||||
num_ds_levels = n_layerwise + n_spatial
|
||||
lm_hidden = config.text_config.hidden_size
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
@@ -644,26 +635,24 @@ class Granite4VisionForConditionalGeneration(
|
||||
)
|
||||
image_feature = unpad_image(image_feature, image_sizes[image_idx])
|
||||
|
||||
if self.image_newline is not None:
|
||||
image_feature = torch.cat(
|
||||
(
|
||||
image_feature,
|
||||
self.image_newline[:, None, None]
|
||||
.expand(*image_feature.shape[:-1], 1)
|
||||
.to(image_feature.device, image_feature.dtype),
|
||||
),
|
||||
dim=-1,
|
||||
)
|
||||
image_feature = torch.cat(
|
||||
(
|
||||
image_feature,
|
||||
self.image_newline[:, None, None]
|
||||
.expand(*image_feature.shape[:-1], 1)
|
||||
.to(image_feature.device, image_feature.dtype),
|
||||
),
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
image_feature = image_feature.flatten(1, 2).transpose(0, 1)
|
||||
image_feature = torch.cat((base_image_feature, image_feature), dim=0)
|
||||
else:
|
||||
image_feature = image_feature[0]
|
||||
if self.image_newline is not None:
|
||||
image_feature = torch.cat(
|
||||
(image_feature, self.image_newline[None].to(image_feature)),
|
||||
dim=0,
|
||||
)
|
||||
image_feature = torch.cat(
|
||||
(image_feature, self.image_newline[None].to(image_feature)),
|
||||
dim=0,
|
||||
)
|
||||
|
||||
new_image_features.append(image_feature)
|
||||
|
||||
@@ -718,16 +707,15 @@ class Granite4VisionForConditionalGeneration(
|
||||
)
|
||||
levels.append((llm_layer, per_image))
|
||||
|
||||
if self._use_spatial_sampling and self.spatial_projectors is not None:
|
||||
spatial_hidden = all_hidden_states[self._spatial_vision_layer]
|
||||
if select_strategy == "default":
|
||||
spatial_hidden = spatial_hidden[:, 1:]
|
||||
for group_idx, llm_layer in enumerate(self._spatial_target_layers):
|
||||
projected = self.spatial_projectors[group_idx](spatial_hidden)
|
||||
per_image = self._pack_and_unpad_image_features(
|
||||
torch.split(projected, image_num_patches, dim=0), image_sizes
|
||||
)
|
||||
levels.append((llm_layer, per_image))
|
||||
spatial_hidden = all_hidden_states[self._spatial_vision_layer]
|
||||
if select_strategy == "default":
|
||||
spatial_hidden = spatial_hidden[:, 1:]
|
||||
for group_idx, llm_layer in enumerate(self._spatial_target_layers):
|
||||
projected = self.spatial_projectors[group_idx](spatial_hidden)
|
||||
per_image = self._pack_and_unpad_image_features(
|
||||
torch.split(projected, image_num_patches, dim=0), image_sizes
|
||||
)
|
||||
levels.append((llm_layer, per_image))
|
||||
|
||||
llm_layer_indices = [llm_layer for llm_layer, _ in levels]
|
||||
num_images = len(image_sizes)
|
||||
|
||||
@@ -158,9 +158,7 @@ class GraniteSpeechMultiModalProcessor(
|
||||
feature_extractor = processor.audio_processor
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
# Use getattr with default to be compatible with transformers<4.48
|
||||
audio_token = getattr(processor, "audio_token", "<|audio|>")
|
||||
audio_token_id = vocab[audio_token]
|
||||
audio_token_id = vocab[processor.audio_token]
|
||||
|
||||
def get_replacement(item_idx: int):
|
||||
audios = mm_items.get_items("audio", AudioProcessorItems)
|
||||
@@ -221,8 +219,7 @@ class GraniteSpeechDummyInputsBuilder(
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_audios = mm_counts.get("audio", 0)
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
audio_token = getattr(hf_processor, "audio_token", "<|audio|>")
|
||||
return audio_token * num_audios
|
||||
return hf_processor.audio_token * num_audios
|
||||
|
||||
|
||||
### QFormer Projector
|
||||
|
||||
@@ -320,10 +320,6 @@ class GraniteMoeHybridAttention(nn.Module):
|
||||
|
||||
|
||||
ALL_DECODER_LAYER_TYPES = {
|
||||
# Transformers < 5.13.0
|
||||
"attention": GraniteMoeHybridAttentionDecoderLayer,
|
||||
"mamba": GraniteMoeHybridMambaDecoderLayer,
|
||||
# Transformers >= 5.13.0
|
||||
"full_attention": GraniteMoeHybridAttentionDecoderLayer,
|
||||
"linear_attention": GraniteMoeHybridMambaDecoderLayer,
|
||||
}
|
||||
@@ -647,6 +643,7 @@ class GraniteMoeHybridForCausalLM(
|
||||
head_dim=hf_config.mamba_d_head,
|
||||
state_size=hf_config.mamba_d_state,
|
||||
conv_kernel=hf_config.mamba_d_conv,
|
||||
num_spec=vllm_config.num_speculative_tokens,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -31,7 +31,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PreTrainedConfig
|
||||
from transformers import HYV3Config, PreTrainedConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
|
||||
@@ -67,7 +67,6 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.hy_v3 import HYV3Config
|
||||
|
||||
from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
|
||||
@@ -31,6 +31,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import HyperCLOVAXConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
@@ -51,7 +52,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.hyperclovax import HyperCLOVAXConfig
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
@@ -137,9 +137,7 @@ class HyperCLOVAXAttention(nn.Module):
|
||||
# the KV heads across multiple tensor parallel GPUs.
|
||||
assert tp_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
self.head_dim = getattr(
|
||||
config, "head_dim", self.hidden_size // self.total_num_heads
|
||||
)
|
||||
self.head_dim = config.head_dim
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
self.scaling = config.attention_multiplier
|
||||
@@ -166,7 +164,7 @@ class HyperCLOVAXAttention(nn.Module):
|
||||
self.head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
is_neox_style=True,
|
||||
rope_parameters=getattr(config, "rope_parameters", None),
|
||||
rope_parameters=config.rope_parameters,
|
||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||
)
|
||||
|
||||
@@ -217,18 +215,15 @@ class HyperCLOVAXDecoderLayer(nn.Module):
|
||||
"dual_chunk_attention_config",
|
||||
None,
|
||||
)
|
||||
attention_bias = getattr(config, "attention_bias", False)
|
||||
|
||||
self.self_attn = HyperCLOVAXAttention(
|
||||
config=config,
|
||||
hidden_size=self.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
num_kv_heads=getattr(
|
||||
config, "num_key_value_heads", config.num_attention_heads
|
||||
),
|
||||
num_kv_heads=config.num_key_value_heads,
|
||||
max_position_embeddings=max_position_embeddings,
|
||||
quant_config=quant_config,
|
||||
bias=attention_bias,
|
||||
bias=config.attention_bias,
|
||||
cache_config=cache_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||
@@ -238,7 +233,7 @@ class HyperCLOVAXDecoderLayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
bias=getattr(config, "mlp_bias", False),
|
||||
bias=config.mlp_bias,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -21,6 +21,7 @@ from typing import (
|
||||
ClassVar,
|
||||
Literal,
|
||||
Protocol,
|
||||
Self,
|
||||
TypeAlias,
|
||||
overload,
|
||||
runtime_checkable,
|
||||
@@ -31,7 +32,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
from torch import Tensor
|
||||
from transformers.models.whisper.tokenization_whisper import LANGUAGES
|
||||
from typing_extensions import Self, TypeIs, deprecated
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -423,17 +424,6 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
|
||||
yield
|
||||
|
||||
@deprecated(
|
||||
"get_num_mm_encoder_tokens is deprecated; use get_mm_lora_token_counts instead."
|
||||
)
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: ...
|
||||
|
||||
@deprecated(
|
||||
"get_num_mm_connector_tokens is deprecated; use "
|
||||
"get_mm_lora_token_counts instead."
|
||||
)
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: ...
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
@@ -447,12 +437,9 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
connector forwards. Models with multiple modalities can override this
|
||||
when each modality has different encoder padding or pooling behavior.
|
||||
"""
|
||||
del modality, mm_kwargs
|
||||
num_encoder_tokens = self.get_num_mm_encoder_tokens(num_mm_embeds)
|
||||
num_connector_tokens = self.get_num_mm_connector_tokens(num_encoder_tokens)
|
||||
return (
|
||||
num_encoder_tokens,
|
||||
num_connector_tokens if isinstance(num_connector_tokens, int) else None,
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} must implement get_mm_lora_token_counts "
|
||||
"to support tower/connector LoRA."
|
||||
)
|
||||
|
||||
def _embed_text_input_ids(
|
||||
|
||||
@@ -11,12 +11,13 @@ from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Literal
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import BatchFeature
|
||||
from transformers import BatchFeature, Kimi_K25Config
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors import (
|
||||
compressed_tensors,
|
||||
@@ -62,7 +63,6 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.kimi_k25 import KimiK25Config
|
||||
from vllm.transformers_utils.processor import cached_get_image_processor
|
||||
from vllm.transformers_utils.processors.kimi_k25 import KimiK25Processor
|
||||
from vllm.transformers_utils.processors.kimi_k25_vision_fused import (
|
||||
@@ -134,7 +134,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
# Resolve token ID from the tokenizer because transformers v5
|
||||
# may remap token IDs vs config.json.
|
||||
config_token_id = hf_config.media_placeholder_token_id
|
||||
config_token_id = hf_config.image_token_id
|
||||
resolved_token_id = tokenizer.convert_tokens_to_ids("<|media_pad|>")
|
||||
unk_token_id = getattr(tokenizer, "unk_token_id", None)
|
||||
is_valid_resolved = isinstance(resolved_token_id, int) and (
|
||||
@@ -142,7 +142,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
|
||||
)
|
||||
if is_valid_resolved and resolved_token_id != config_token_id:
|
||||
logger.warning_once(
|
||||
"Kimi-K2.5 config.media_placeholder_token_id (%d) disagrees "
|
||||
"Kimi-K2.5 config.image_token_id (%d) disagrees "
|
||||
"with tokenizer mapping for <|media_pad|> (%d). "
|
||||
"Using tokenizer value.",
|
||||
config_token_id,
|
||||
@@ -150,7 +150,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
|
||||
)
|
||||
media_token_id = resolved_token_id
|
||||
# Patch config so downstream code also sees the correct ID.
|
||||
hf_config.media_placeholder_token_id = resolved_token_id
|
||||
hf_config.image_token_id = resolved_token_id
|
||||
else:
|
||||
media_token_id = config_token_id
|
||||
|
||||
@@ -169,7 +169,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo):
|
||||
return self.hf_processor
|
||||
|
||||
def get_hf_config(self):
|
||||
return self.ctx.get_hf_config(KimiK25Config)
|
||||
return self.ctx.get_hf_config(Kimi_K25Config)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
# None means unlimited
|
||||
@@ -306,6 +306,7 @@ class KimiK25ForConditionalGeneration(
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
supports_encoder_cudagraph: ClassVar[Literal[True]] = True
|
||||
supports_mm_device_do_normalize = True
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
@@ -336,7 +337,7 @@ class KimiK25ForConditionalGeneration(
|
||||
) -> None:
|
||||
super().__init__()
|
||||
model_config = vllm_config.model_config
|
||||
config: KimiK25Config = model_config.hf_config
|
||||
config: Kimi_K25Config = model_config.hf_config
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
@@ -345,11 +346,11 @@ class KimiK25ForConditionalGeneration(
|
||||
)
|
||||
self.hidden_size = config.text_config.hidden_size
|
||||
self.device = current_platform.current_device()
|
||||
# Build vision tower directly with KimiK25VisionConfig
|
||||
with self._mark_tower_model(vllm_config, "vision_chunk"):
|
||||
self.vision_tower = MoonViT3dPretrainedModel(
|
||||
config.vision_config,
|
||||
quant_config=self._maybe_ignore_quant_config(quant_config),
|
||||
input_norm=build_mm_input_norm(vllm_config.model_config),
|
||||
prefix=maybe_prefix(prefix, "vision_tower"),
|
||||
)
|
||||
if self._maybe_ignore_quant_config(quant_config) is not None:
|
||||
@@ -361,6 +362,7 @@ class KimiK25ForConditionalGeneration(
|
||||
|
||||
self.mm_projector = KimiK25MultiModalProjector(
|
||||
config=config.vision_config,
|
||||
out_hidden_size=config.text_config.hidden_size,
|
||||
use_data_parallel=self.use_data_parallel,
|
||||
quant_config=self._maybe_ignore_quant_config(quant_config),
|
||||
prefix=maybe_prefix(prefix, "mm_projector"),
|
||||
@@ -380,7 +382,7 @@ class KimiK25ForConditionalGeneration(
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
self.media_placeholder: int = self.config.media_placeholder_token_id
|
||||
self.media_placeholder: int = self.config.image_token_id
|
||||
|
||||
def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig | None):
|
||||
if isinstance(quant_config, compressed_tensors.CompressedTensorsConfig):
|
||||
@@ -404,9 +406,6 @@ class KimiK25ForConditionalGeneration(
|
||||
pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
|
||||
)
|
||||
|
||||
# The batch dimension of pixel_values has been flattened into shape[0]
|
||||
target_dtype = next(self.vision_tower.parameters()).dtype
|
||||
pixel_values = pixel_values.to(target_dtype)
|
||||
assert isinstance(grid_thws, torch.Tensor), (
|
||||
f"expect grid_thws to be a tensor, got {type(grid_thws)}"
|
||||
)
|
||||
@@ -479,30 +478,16 @@ class KimiK25ForConditionalGeneration(
|
||||
# Image-only (t == 1). Video chunks (t > 1) fall back to eager. #
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
@property
|
||||
def _encoder_cudagraph_pad_totals(self) -> dict[int, int]:
|
||||
"""Row count of each captured buffer set, keyed by cu_seqlens ptr."""
|
||||
totals = self.__dict__.get("_encoder_cg_pad_totals")
|
||||
if totals is None:
|
||||
totals = {}
|
||||
self.__dict__["_encoder_cg_pad_totals"] = totals
|
||||
return totals
|
||||
|
||||
def get_encoder_cudagraph_config(self) -> "EncoderCudaGraphConfig":
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
|
||||
|
||||
pad_totals = self._encoder_cudagraph_pad_totals
|
||||
|
||||
def pad_cu_seqlens(dst: torch.Tensor, src: torch.Tensor) -> None:
|
||||
# Varlen attention requires cu_seqlens[-1] to equal the number of
|
||||
# rows actually passed in. The captured buffers are sized for the
|
||||
# full token budget, so a smaller real batch has to be completed
|
||||
# with one trailing padding sequence; declaring fewer rows than the
|
||||
# buffer holds is undefined behaviour and returns NaN on FlashAttn.
|
||||
total = pad_totals.get(dst.data_ptr())
|
||||
# Zero-length padding (same as Qwen): tail slots repeat the last
|
||||
# real offset, so FlashAttn schedules no work for padding rows.
|
||||
n = min(src.shape[0], dst.shape[0])
|
||||
dst[:n].copy_(src[:n])
|
||||
dst[n:] = total if total is not None else src[-1]
|
||||
if n < dst.shape[0]:
|
||||
dst[n:] = src[-1] if n else 0
|
||||
|
||||
return EncoderCudaGraphConfig(
|
||||
modalities=["vision_chunk"],
|
||||
@@ -623,13 +608,18 @@ class KimiK25ForConditionalGeneration(
|
||||
ps = (ps, ps)
|
||||
total_patches = max_batch_size * ho * kh * wo * kw
|
||||
dummy_pixel_values = torch.zeros(
|
||||
total_patches, 3, ps[0], ps[1], device=device, dtype=dtype
|
||||
total_patches,
|
||||
3,
|
||||
ps[0],
|
||||
ps[1],
|
||||
device=device,
|
||||
dtype=self.vision_tower.patch_embed.input_norm.input_dtype or dtype,
|
||||
)
|
||||
|
||||
# max_seqlen must cover the worst case: one item consuming the full
|
||||
# budget, i.e. token_budget * kh * kw patches.
|
||||
# max_batch_size + 1 leaves a spare cu_seqlens slot so replay can append
|
||||
# a padding sequence covering rows the real batch does not fill.
|
||||
# max_batch_size + 1 leaves a spare cu_seqlens slot so replay can
|
||||
# pad the tail with zero-length sequences.
|
||||
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
|
||||
grid_thw_list,
|
||||
max_batch_size=max_batch_size + 1,
|
||||
@@ -640,10 +630,6 @@ class KimiK25ForConditionalGeneration(
|
||||
values: dict[str, torch.Tensor] = {"pixel_values": dummy_pixel_values}
|
||||
values.update({k: v for k, v in metadata.items() if v is not None})
|
||||
|
||||
cu_seqlens = values.get("cu_seqlens")
|
||||
if cu_seqlens is not None:
|
||||
self._encoder_cudagraph_pad_totals[cu_seqlens.data_ptr()] = total_patches
|
||||
|
||||
return EncoderCudaGraphCaptureInputs(values=values)
|
||||
|
||||
def prepare_encoder_cudagraph_replay_buffers(
|
||||
@@ -658,8 +644,8 @@ class KimiK25ForConditionalGeneration(
|
||||
grid_thw_list = self._get_grid_thw_list(mm_kwargs)
|
||||
pixel_values = mm_kwargs["pixel_values"]
|
||||
|
||||
# Unpadded: pad_cu_seqlens completes the tail with the padding sequence
|
||||
# so cu_seqlens[-1] matches the captured buffer's row count.
|
||||
# Unpadded: pad_cu_seqlens pads the tail with zero-length sequences
|
||||
# whose entries repeat the real total.
|
||||
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
|
||||
grid_thw_list,
|
||||
max_batch_size=None,
|
||||
|
||||
@@ -15,6 +15,7 @@ import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from transformers import Kimi_K25VisionConfig
|
||||
from transformers.activations import GELUActivation
|
||||
|
||||
from vllm.distributed import divide, get_tensor_model_parallel_world_size
|
||||
@@ -22,6 +23,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.conv import Conv2dLayer
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import IdentityInputNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
@@ -39,13 +41,21 @@ from vllm.model_executor.models.vision import (
|
||||
run_dp_sharded_mrope_vision_model,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig
|
||||
from vllm.transformers_utils.configs.kimi_k3 import KimiK3VisionConfig
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_pos_emb_size(
|
||||
config: Kimi_K25VisionConfig | KimiK3VisionConfig, dim: str
|
||||
) -> int:
|
||||
# Legacy checkpoints and Kimi-K3 use `init_pos_emb_*`
|
||||
legacy = getattr(config, f"init_pos_emb_{dim}", None)
|
||||
return legacy if legacy is not None else getattr(config, f"pos_emb_{dim}")
|
||||
|
||||
|
||||
def _apply_rope_input_validation(x, freqs_cis):
|
||||
assert x.ndim == freqs_cis.ndim + 1, (x.shape, freqs_cis.shape)
|
||||
assert x.shape[:-2] == freqs_cis.shape[:-1], (x.shape, freqs_cis.shape)
|
||||
@@ -187,6 +197,7 @@ class MoonVision3dPatchEmbed(nn.Module):
|
||||
pos_emb_type: str = "divided_fixed",
|
||||
patch_embed_proj_bias: bool = True,
|
||||
pos_emb_interpolation_mode: str = "bicubic",
|
||||
input_norm: nn.Module | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
assert isinstance(patch_size, int | Sequence), (
|
||||
@@ -206,6 +217,7 @@ class MoonVision3dPatchEmbed(nn.Module):
|
||||
stride=patch_size,
|
||||
bias=patch_embed_proj_bias,
|
||||
)
|
||||
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
|
||||
|
||||
if pos_emb_type == "divided_fixed":
|
||||
self.pos_emb = Learnable2DInterpPosEmbDivided_fixed(
|
||||
@@ -225,6 +237,7 @@ class MoonVision3dPatchEmbed(nn.Module):
|
||||
*,
|
||||
pos_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
x = self.input_norm(x.flatten(1), self.proj.weight.dtype).view_as(x)
|
||||
# forward_native dispatches this non-overlapping patch projection to GEMM.
|
||||
x = self.proj.forward_native(x).view(x.size(0), self.proj.out_channels)
|
||||
if pos_embeds is not None:
|
||||
@@ -694,35 +707,35 @@ def tpool_patch_merger_packed(
|
||||
|
||||
|
||||
class MoonViT3dPretrainedModel(nn.Module):
|
||||
"""Main vision tower model.
|
||||
|
||||
Uses KimiK25VisionConfig directly from transformers_utils/configs/kimi_k25.py.
|
||||
"""
|
||||
"""Main vision tower model."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiK25VisionConfig,
|
||||
config: Kimi_K25VisionConfig | KimiK3VisionConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
input_norm: nn.Module | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
config = deepcopy(config)
|
||||
self.config = config # Required for run_dp_sharded_mrope_vision_model
|
||||
self.merge_kernel_size = config.merge_kernel_size
|
||||
merge_h, merge_w = config.merge_kernel_size
|
||||
self.merge_kernel_size = (merge_h, merge_w)
|
||||
self.patch_size = config.patch_size
|
||||
self.merge_type = config.merge_type
|
||||
self.merge_type = getattr(config, "merge_type", "sd2_tpool")
|
||||
|
||||
self.patch_embed = MoonVision3dPatchEmbed(
|
||||
out_dim=config.hidden_size,
|
||||
patch_size=config.patch_size,
|
||||
pos_emb_height=config.init_pos_emb_height,
|
||||
pos_emb_width=config.init_pos_emb_width,
|
||||
pos_emb_time=config.init_pos_emb_time,
|
||||
pos_emb_type=config.pos_emb_type,
|
||||
pos_emb_height=_get_pos_emb_size(config, "height"),
|
||||
pos_emb_width=_get_pos_emb_size(config, "width"),
|
||||
pos_emb_time=_get_pos_emb_size(config, "time"),
|
||||
pos_emb_type=getattr(config, "pos_emb_type", "divided_fixed"),
|
||||
patch_embed_proj_bias=getattr(config, "patch_embed_proj_bias", True),
|
||||
pos_emb_interpolation_mode=getattr(
|
||||
config, "pos_emb_interpolation_mode", "bicubic"
|
||||
),
|
||||
input_norm=input_norm,
|
||||
)
|
||||
|
||||
self.encoder = MoonViT3dEncoder(
|
||||
@@ -734,14 +747,14 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
"qkv_hidden_size": getattr(config, "qkv_hidden_size", None),
|
||||
"mlp_dim": config.intermediate_size,
|
||||
"activation": get_act_fn(
|
||||
getattr(config, "activation_func", "gelu_pytorch_tanh")
|
||||
getattr(config, "activation_func", None) or config.hidden_act
|
||||
),
|
||||
"attn_bias": getattr(config, "attn_bias", True),
|
||||
"norm_type": getattr(config, "norm_type", "layernorm"),
|
||||
"mlp_type": getattr(config, "mlp_type", "mlp2"),
|
||||
"linear_bias": getattr(config, "linear_bias", True),
|
||||
},
|
||||
video_attn_type=config.video_attn_type,
|
||||
video_attn_type=getattr(config, "video_attn_type", "spatial_temporal"),
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "encoder"),
|
||||
)
|
||||
@@ -880,7 +893,8 @@ class KimiK25MultiModalProjector(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiK25VisionConfig,
|
||||
config: Kimi_K25VisionConfig | KimiK3VisionConfig,
|
||||
out_hidden_size: int,
|
||||
use_data_parallel: bool = False,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
@@ -903,13 +917,13 @@ class KimiK25MultiModalProjector(nn.Module):
|
||||
)
|
||||
self.linear_2 = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
getattr(config, "text_hidden_size", config.mm_hidden_size),
|
||||
out_hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_2",
|
||||
)
|
||||
self.post_norm = torch.nn.RMSNorm(
|
||||
getattr(config, "text_hidden_size", config.mm_hidden_size),
|
||||
out_hidden_size,
|
||||
eps=config.projector_ln_eps,
|
||||
)
|
||||
self.act = GELUActivation()
|
||||
@@ -925,7 +939,7 @@ class KimiK25MultiModalProjector(nn.Module):
|
||||
)
|
||||
self.linear_2 = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
config.mm_hidden_size,
|
||||
out_hidden_size,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_2",
|
||||
|
||||
@@ -5,6 +5,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import Lfm2MoeConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
@@ -40,7 +41,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.lfm2_moe import Lfm2MoeConfig
|
||||
|
||||
from .interfaces import (
|
||||
HasInnerState,
|
||||
|
||||
@@ -36,10 +36,15 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
from vllm.model_executor.models.llama4 import Llama4DecoderLayer, Llama4ForCausalLM
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
|
||||
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsMultiModalEmbeddings,
|
||||
_require_is_multimodal,
|
||||
)
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
_merge_multimodal_embeddings,
|
||||
maybe_prefix,
|
||||
process_eagle_weight,
|
||||
)
|
||||
@@ -204,7 +209,21 @@ class EagleLlama4ForCausalLM(_EagleLlama4ForCausalLMBase, SupportsMultiModalEmbe
|
||||
def get_language_model(self) -> torch.nn.Module:
|
||||
return self.model
|
||||
|
||||
embed_input_ids = SupportsMultiModal.embed_input_ids # type: ignore
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self.model.embed_input_ids(input_ids)
|
||||
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
|
||||
return inputs_embeds
|
||||
return _merge_multimodal_embeddings(
|
||||
inputs_embeds=inputs_embeds,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=_require_is_multimodal(is_multimodal),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -58,13 +58,17 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.int8_utils import block_dequant
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
LoaderFunction,
|
||||
composed_weight_loader,
|
||||
default_weight_loader,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_v2 import DeepseekV2MLAAttention
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
@@ -80,6 +84,23 @@ from .utils import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _scaled_weight_loader(scale: float) -> LoaderFunction:
|
||||
return composed_weight_loader(default_weight_loader, lambda x: x * scale)
|
||||
|
||||
|
||||
def scale_mla_lora_norms_on_load(self_attn: nn.Module, config) -> None:
|
||||
"""Fold LongCat's MLA LoRA scaling into the q/kv_a_layernorm weight loaders."""
|
||||
for norm, rank, enabled in (
|
||||
(self_attn.q_a_layernorm, config.q_lora_rank, config.mla_scale_q_lora),
|
||||
(self_attn.kv_a_layernorm, config.kv_lora_rank, config.mla_scale_kv_lora),
|
||||
):
|
||||
if enabled:
|
||||
scale = (config.hidden_size / rank) ** 0.5
|
||||
set_weight_attrs(
|
||||
norm.weight, {"weight_loader": _scaled_weight_loader(scale)}
|
||||
)
|
||||
|
||||
|
||||
class FlashMLP(nn.Module):
|
||||
"""Flash MLP layer."""
|
||||
|
||||
@@ -265,6 +286,8 @@ class FlashDecoderLayer(nn.Module):
|
||||
for i in range(2)
|
||||
]
|
||||
)
|
||||
for self_attn in self.self_attn:
|
||||
scale_mla_lora_norms_on_load(self_attn, config)
|
||||
self.input_layernorm = nn.ModuleList(
|
||||
[RMSNorm(config.hidden_size, eps=config.rms_norm_eps) for i in range(2)]
|
||||
)
|
||||
@@ -450,6 +473,9 @@ class FlashModel(nn.Module):
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
# The MTP layers are loaded by LongCatFlashMTP.
|
||||
if name.startswith("mtp."):
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
@@ -461,9 +487,6 @@ class FlashModel(nn.Module):
|
||||
name.endswith(".bias") or name.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
continue
|
||||
# Skip mtp
|
||||
if ".mtp." in name:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
@@ -478,9 +501,6 @@ class FlashModel(nn.Module):
|
||||
continue
|
||||
is_expert_weight = True
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
# Skip mtp
|
||||
if ".mtp." in name_mapped:
|
||||
continue
|
||||
if (
|
||||
name_mapped.endswith(".bias") or name_mapped.endswith("_bias")
|
||||
) and name not in params_dict:
|
||||
@@ -515,9 +535,6 @@ class FlashModel(nn.Module):
|
||||
# Skip loading kv_scale from ckpts towards new design.
|
||||
if name.endswith(".kv_scale") and name not in params_dict:
|
||||
continue
|
||||
# Skip mtp
|
||||
if ".mtp." in name:
|
||||
continue
|
||||
if name is None:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
@@ -528,54 +545,6 @@ class FlashModel(nn.Module):
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
for layer_id in range(self.config.num_layers):
|
||||
for i in range(2):
|
||||
if isinstance(self.layers[layer_id], PPMissingLayer):
|
||||
continue
|
||||
self_attn = self.layers[layer_id].self_attn[i]
|
||||
if (
|
||||
self.quant_config is not None
|
||||
and hasattr(self.quant_config, "weight_block_size")
|
||||
and self_attn.kv_b_proj.weight.dtype
|
||||
in (
|
||||
torch.float8_e4m3fn,
|
||||
torch.float8_e4m3fnuz,
|
||||
)
|
||||
):
|
||||
weight_block_size = self.quant_config.weight_block_size
|
||||
if weight_block_size is not None:
|
||||
assert hasattr(self_attn.kv_b_proj, "weight_scale_inv")
|
||||
dtype = torch.get_default_dtype()
|
||||
w = block_dequant(
|
||||
self_attn.kv_b_proj.weight,
|
||||
self_attn.kv_b_proj.weight_scale_inv,
|
||||
weight_block_size,
|
||||
).to(dtype)
|
||||
else:
|
||||
w = self_attn.kv_b_proj.weight
|
||||
|
||||
w_kc, w_vc = w.unflatten(
|
||||
0, (-1, self_attn.qk_nope_head_dim + self_attn.v_head_dim)
|
||||
).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1)
|
||||
self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2)
|
||||
self_attn.w_vc = w_vc.contiguous().transpose(1, 2)
|
||||
# Guard against compounding on incremental load_weights calls:
|
||||
# the in-place ``*=`` would otherwise re-apply the MLA LoRA
|
||||
# scaling to the layernorm weights on each pass.
|
||||
if self.config.mla_scale_q_lora and not getattr(
|
||||
self_attn, "_mla_q_lora_scaled", False
|
||||
):
|
||||
self_attn.q_a_layernorm.weight.data *= (
|
||||
self.config.hidden_size / self.config.q_lora_rank
|
||||
) ** 0.5
|
||||
self_attn._mla_q_lora_scaled = True
|
||||
if self.config.mla_scale_kv_lora and not getattr(
|
||||
self_attn, "_mla_kv_lora_scaled", False
|
||||
):
|
||||
self_attn.kv_a_layernorm.weight.data *= (
|
||||
self.config.hidden_size / self.config.kv_lora_rank
|
||||
) ** 0.5
|
||||
self_attn._mla_kv_lora_scaled = True
|
||||
return loaded_params
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.int8_utils import block_dequant
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
@@ -23,6 +22,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .deepseek_v2 import DeepseekV2DecoderLayer
|
||||
from .longcat_flash import scale_mla_lora_norms_on_load
|
||||
from .utils import maybe_prefix
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ class LongCatMultiTokenPredictorLayer(nn.Module):
|
||||
prefix="eh_proj",
|
||||
)
|
||||
self.mtp_block = DeepseekV2DecoderLayer(vllm_config, prefix)
|
||||
scale_mla_lora_norms_on_load(self.mtp_block.self_attn, config)
|
||||
self.final_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
def forward(
|
||||
@@ -263,50 +264,6 @@ class LongCatFlashMTP(nn.Module):
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
spec_layer_id = self.model.mtp_start_layer_idx
|
||||
self_attn = self.model.layers[str(spec_layer_id)].mtp_block.self_attn
|
||||
if (
|
||||
self.quant_config is not None
|
||||
and hasattr(self.quant_config, "weight_block_size")
|
||||
and self_attn.kv_b_proj.weight.dtype
|
||||
in (
|
||||
torch.float8_e4m3fn,
|
||||
torch.float8_e4m3fnuz,
|
||||
)
|
||||
):
|
||||
weight_block_size = self.quant_config.weight_block_size
|
||||
if weight_block_size is not None:
|
||||
dtype = torch.get_default_dtype()
|
||||
w = block_dequant(
|
||||
self_attn.kv_b_proj.weight,
|
||||
self_attn.kv_b_proj.weight_scale_inv,
|
||||
weight_block_size,
|
||||
).to(dtype)
|
||||
else:
|
||||
w = self_attn.kv_b_proj.weight
|
||||
else:
|
||||
w = self_attn.kv_b_proj.weight
|
||||
w_kc, w_vc = w.unflatten(
|
||||
0, (-1, self_attn.qk_nope_head_dim + self_attn.v_head_dim)
|
||||
).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1)
|
||||
self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2)
|
||||
self_attn.w_vc = w_vc.contiguous().transpose(1, 2)
|
||||
# Guard against compounding on incremental load_weights calls (the
|
||||
# in-place *= would otherwise double-apply the LoRA scaling).
|
||||
if self.config.mla_scale_q_lora and not getattr(
|
||||
self_attn, "_mla_q_lora_scaled", False
|
||||
):
|
||||
self_attn.q_a_layernorm.weight.data *= (
|
||||
self.config.hidden_size / self.config.q_lora_rank
|
||||
) ** 0.5
|
||||
self_attn._mla_q_lora_scaled = True
|
||||
if self.config.mla_scale_kv_lora and not getattr(
|
||||
self_attn, "_mla_kv_lora_scaled", False
|
||||
):
|
||||
self_attn.kv_a_layernorm.weight.data *= (
|
||||
self.config.hidden_size / self.config.kv_lora_rank
|
||||
) ** 0.5
|
||||
self_attn._mla_kv_lora_scaled = True
|
||||
return loaded_params
|
||||
|
||||
def _rewrite_spec_layer_name(
|
||||
|
||||
@@ -73,7 +73,7 @@ class MiMoMultiTokenPredictorLayer(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
assert inputs_embeds is not None
|
||||
# masking inputs at position 0, as not needed by MTP
|
||||
inputs_embeds[positions == 0] = 0
|
||||
inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
|
||||
inputs_embeds = self.token_layernorm(inputs_embeds)
|
||||
previous_hidden_states = self.hidden_layernorm(previous_hidden_states)
|
||||
|
||||
|
||||
@@ -372,6 +372,10 @@ class MiMoV2FlashDecoderLayer(nn.Module):
|
||||
max_position_embeddings = getattr(config, "max_position_embeddings", 32768)
|
||||
|
||||
v_scale = getattr(config, "attention_value_scale", None)
|
||||
# cache_config.sliding_window holds MiMo's SWA window; global layers
|
||||
# would otherwise fall back to it in Attention.
|
||||
cache_config = vllm_config.cache_config
|
||||
cache_config.sliding_window = None
|
||||
|
||||
if self.is_compressed_softmax_layer():
|
||||
self.self_attn = MiMoV2Attention(
|
||||
@@ -389,6 +393,7 @@ class MiMoV2FlashDecoderLayer(nn.Module):
|
||||
layer_id=layer_id,
|
||||
rope_theta=getattr(config, "swa_rope_theta", rope_theta),
|
||||
max_position_embeddings=max_position_embeddings,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
|
||||
prefix=f"{prefix}.self_attn",
|
||||
@@ -406,6 +411,7 @@ class MiMoV2FlashDecoderLayer(nn.Module):
|
||||
layer_id=layer_id,
|
||||
rope_theta=rope_theta,
|
||||
max_position_embeddings=max_position_embeddings,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
|
||||
prefix=f"{prefix}.self_attn",
|
||||
|
||||
@@ -24,7 +24,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -64,6 +64,7 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
self,
|
||||
config: PreTrainedConfig,
|
||||
prefix: str,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -100,6 +101,7 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
layer_id=0,
|
||||
rope_theta=swa_rope_theta,
|
||||
max_position_embeddings=getattr(config, "max_position_embeddings", 32768),
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
partial_rotary_factor=getattr(config, "partial_rotary_factor", 1.0),
|
||||
prefix=f"{prefix}.self_attn",
|
||||
@@ -147,6 +149,7 @@ class _MiMoV2MTPLayers(nn.Module):
|
||||
self,
|
||||
config: PreTrainedConfig,
|
||||
num_mtp_layers: int,
|
||||
cache_config: CacheConfig | None,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
) -> None:
|
||||
@@ -156,6 +159,7 @@ class _MiMoV2MTPLayers(nn.Module):
|
||||
str(i): MiMoV2MTPLayer(
|
||||
config=config,
|
||||
prefix=f"{prefix}.{i}",
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
for i in range(num_mtp_layers)
|
||||
@@ -182,6 +186,7 @@ class MiMoV2MultiTokenPredictor(nn.Module):
|
||||
self.mtp = _MiMoV2MTPLayers(
|
||||
config=config,
|
||||
num_mtp_layers=num_mtp_layers,
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, "mtp.layers"),
|
||||
)
|
||||
|
||||
@@ -23,8 +23,17 @@ from vllm.model_executor.models.deepseek_v2 import (
|
||||
)
|
||||
from vllm.model_executor.models.mistral_large_3 import MistralLarge3ForCausalLM
|
||||
|
||||
from .interfaces import SupportsMultiModal, SupportsMultiModalEmbeddings
|
||||
from .utils import WeightsMapper, make_empty_intermediate_tensors_factory, maybe_prefix
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsMultiModalEmbeddings,
|
||||
_require_is_multimodal,
|
||||
)
|
||||
from .utils import (
|
||||
WeightsMapper,
|
||||
_merge_multimodal_embeddings,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -148,7 +157,21 @@ class EagleMistralLarge3ForCausalLM(
|
||||
def get_language_model(self) -> torch.nn.Module:
|
||||
return self.model
|
||||
|
||||
embed_input_ids = SupportsMultiModal.embed_input_ids # type: ignore
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
inputs_embeds = self.model.embed_input_ids(input_ids)
|
||||
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
|
||||
return inputs_embeds
|
||||
return _merge_multimodal_embeddings(
|
||||
inputs_embeds=inputs_embeds,
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=_require_is_multimodal(is_multimodal),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -94,28 +94,12 @@ class ModernBertAttention(nn.Module):
|
||||
prefix=f"{prefix}.Wqkv",
|
||||
)
|
||||
|
||||
if layer_types := getattr(config, "layer_types", None):
|
||||
# Transformers v5
|
||||
layer_type = layer_types[layer_id]
|
||||
rope_parameters = config.rope_parameters[layer_type]
|
||||
sliding_window: int | None = None
|
||||
if layer_type == "sliding_attention":
|
||||
# Treats the local attention boundary as inclusive
|
||||
sliding_window = config.sliding_window + 1
|
||||
else:
|
||||
# Transformers v4
|
||||
sliding_window = None
|
||||
if layer_id % config.global_attn_every_n_layers != 0:
|
||||
# ModernBertConfig does not expose sliding_window
|
||||
sliding_window = config.local_attention // 2
|
||||
rope_theta = (
|
||||
config.local_rope_theta
|
||||
if config.local_rope_theta is not None
|
||||
else config.global_rope_theta
|
||||
)
|
||||
else:
|
||||
rope_theta = config.global_rope_theta
|
||||
rope_parameters = {"rope_type": "default", "rope_theta": rope_theta}
|
||||
layer_type = config.layer_types[layer_id]
|
||||
rope_parameters = config.rope_parameters[layer_type]
|
||||
sliding_window: int | None = None
|
||||
if layer_type == "sliding_attention":
|
||||
# Treats the local attention boundary as inclusive
|
||||
sliding_window = config.sliding_window + 1
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
head_size=self.head_dim,
|
||||
|
||||
@@ -29,6 +29,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import NemotronConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
@@ -49,7 +50,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.nemotron import NemotronConfig
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
|
||||
@@ -23,6 +23,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import NemotronHConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import (
|
||||
@@ -87,7 +88,6 @@ from vllm.model_executor.models.utils import (
|
||||
sequence_parallel_chunk,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
|
||||
|
||||
|
||||
class NemotronHMLP(nn.Module):
|
||||
@@ -235,7 +235,7 @@ class NemotronHMoE(nn.Module):
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
renormalize=config.norm_topk_prob,
|
||||
quant_config=quant_config,
|
||||
ckpt_names=("up_proj", "down_proj", ""),
|
||||
ckpt_names=("up_proj", "down_proj", None),
|
||||
use_grouped_topk=True,
|
||||
num_expert_group=config.n_group,
|
||||
topk_group=config.topk_group,
|
||||
@@ -707,7 +707,7 @@ class NemotronHModel(nn.Module, EagleModelMixin):
|
||||
self,
|
||||
ckpt_gate_proj_name="up_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="",
|
||||
ckpt_up_proj_name=None,
|
||||
num_experts=self._get_max_n_routed_experts(),
|
||||
num_redundant_experts=getattr(self, "num_redundant_experts", 0),
|
||||
)
|
||||
|
||||
@@ -7,6 +7,7 @@ from collections.abc import Callable, Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import NemotronHConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
@@ -33,7 +34,6 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
|
||||
|
||||
from .interfaces import SupportsPP, SupportsQuant
|
||||
from .nemotron_h import (
|
||||
@@ -426,6 +426,11 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant):
|
||||
)
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
"""Only MTP, embedding and LM head weights are loaded."""
|
||||
name = name.removeprefix("language_model.")
|
||||
return not name.startswith(("mtp.", "lm_head.")) and "embeddings" not in name
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load MTP weights with proper name remapping."""
|
||||
stacked_params_mapping = [
|
||||
@@ -444,7 +449,7 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant):
|
||||
self,
|
||||
ckpt_gate_proj_name="up_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="", # Empty - non-gated MoE
|
||||
ckpt_up_proj_name=None, # non-gated MoE
|
||||
num_experts=num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
@@ -456,17 +461,11 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant):
|
||||
# MTP weights are nested in "language_model."
|
||||
# in Multimodal Nemotron-H checkpoints.
|
||||
name = name.removeprefix("language_model.")
|
||||
if self.is_unused_checkpoint_weight(name):
|
||||
continue
|
||||
is_lm_head_weight = name.startswith("lm_head.")
|
||||
if is_lm_head_weight:
|
||||
self.has_own_lm_head = True
|
||||
# Only process MTP and LM head weights -
|
||||
# skip all non-MTP and non-LM head weights
|
||||
if (
|
||||
not name.startswith("mtp.")
|
||||
and "embeddings" not in name
|
||||
and not is_lm_head_weight
|
||||
):
|
||||
continue
|
||||
# Skip rotary embeddings (computed, not loaded)
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
@@ -568,6 +568,15 @@ class NemotronParseForConditionalGeneration(nn.Module, SupportsMultiModal):
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.decoder.vocab_size, config.decoder.d_model, quant_config=quant_config
|
||||
)
|
||||
# Some checkpoints (e.g. compact exports) tie the output head to the
|
||||
# decoder's input embeddings instead of materializing a separate
|
||||
# lm_head.weight tensor.
|
||||
tie_word_embeddings = bool(
|
||||
getattr(config, "tie_word_embeddings", False)
|
||||
or getattr(config.decoder, "tie_word_embeddings", False)
|
||||
)
|
||||
if tie_word_embeddings:
|
||||
self.lm_head = self.lm_head.tie_weights(self.decoder.embed_tokens)
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.vocab_size, config.decoder.vocab_size
|
||||
)
|
||||
|
||||
@@ -125,7 +125,7 @@ class VisualTokenizer(torch.nn.Module):
|
||||
prefix: str = "",
|
||||
) -> nn.Module:
|
||||
model_type = config.backbone_config.model_type
|
||||
if model_type == "aimv2":
|
||||
if model_type == "aimv2_vision_model":
|
||||
# No post rms_norm in Ovis2's AIMv2 ViT.
|
||||
return AIMv2Model(
|
||||
config=config.backbone_config,
|
||||
|
||||
@@ -129,7 +129,7 @@ class ConformerEncoderLayer(nn.Module):
|
||||
default False.
|
||||
use_pt_scaled_dot_product_attention: bool, optional
|
||||
if set to True, use pytorch's scaled dot product attention
|
||||
implementation in training.
|
||||
implementation in training. Default: True.
|
||||
attn_group_sizes: int, optional
|
||||
the number of groups to use for attention, default 1
|
||||
(Multi-Head Attention),
|
||||
@@ -163,7 +163,7 @@ class ConformerEncoderLayer(nn.Module):
|
||||
attention_glu_type: str = "swish",
|
||||
activation_checkpointing: str = "",
|
||||
export: bool = False,
|
||||
use_pt_scaled_dot_product_attention: bool = False,
|
||||
use_pt_scaled_dot_product_attention: bool = True,
|
||||
attn_group_sizes: int = 1,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -808,7 +808,7 @@ class ConformerEncoder(TransformerEncoderBase):
|
||||
default 4
|
||||
use_pt_scaled_dot_product_attention: whether to use pytorch scaled
|
||||
dot product attention in training.
|
||||
Default: False
|
||||
Default: True
|
||||
nemo_conv_settings: dict, optional
|
||||
A dictionary of settings for NeMo Subsampling.
|
||||
default: None
|
||||
@@ -873,7 +873,7 @@ class ConformerEncoder(TransformerEncoderBase):
|
||||
activation_checkpointing: str = "",
|
||||
relative_attention_bias_args: dict[str, Any] | None = None,
|
||||
time_reduction: int = 4,
|
||||
use_pt_scaled_dot_product_attention: bool = False,
|
||||
use_pt_scaled_dot_product_attention: bool = True,
|
||||
nemo_conv_settings: dict[str, Any] | None = None,
|
||||
conv2d_extra_padding: Literal["feat", "feat_time", "none", True] = "none",
|
||||
replication_pad_for_subsample_embedding: bool = False,
|
||||
|
||||
@@ -1642,7 +1642,7 @@ class MultiHeadedAttention(nn.Module):
|
||||
NOTE: this will NOT be used in ONNX decoding due to a lack of
|
||||
support. In that case, we use the original attention
|
||||
implementation, which shows no regression.
|
||||
default: False.
|
||||
default: True.
|
||||
n_value: int, optional
|
||||
if set to values other than -1, use a different dimension for
|
||||
value. With the default value (i.e. -1), it is backward compatible.
|
||||
@@ -1666,7 +1666,7 @@ class MultiHeadedAttention(nn.Module):
|
||||
attention_inner_dim: int = -1,
|
||||
glu_type: str = "swish",
|
||||
bias_in_glu: bool = True,
|
||||
use_pt_scaled_dot_product_attention: bool = False,
|
||||
use_pt_scaled_dot_product_attention: bool = True,
|
||||
n_value: int = -1,
|
||||
group_size: int = 1,
|
||||
) -> None:
|
||||
@@ -1690,7 +1690,6 @@ class MultiHeadedAttention(nn.Module):
|
||||
self.linear_v = nn.Linear(n_value, attention_inner_dim // group_size)
|
||||
self.linear_out = nn.Linear(attention_inner_dim // group_size, n_value)
|
||||
|
||||
self.attn = torch.jit.Attribute(None, Tensor | None)
|
||||
self.dropout = nn.Dropout(p=dropout_rate)
|
||||
self.dropout_rate = dropout_rate
|
||||
self.use_pt_scaled_dot_product_attention = use_pt_scaled_dot_product_attention
|
||||
@@ -1799,8 +1798,6 @@ class MultiHeadedAttention(nn.Module):
|
||||
|
||||
attn = masked_softmax(scores, mask) # (batch, head, time1, time2)
|
||||
|
||||
self.attn = attn
|
||||
|
||||
p_attn = self.dropout(attn)
|
||||
x = torch.matmul(p_attn.to(v.dtype), v) # (batch, head, time1, d_k)
|
||||
if pos_v is not None:
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, get_act_and_mul_fn
|
||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.conv import Conv2dLayer
|
||||
from vllm.model_executor.layers.fusion.fused_act_quant import maybe_fused_act_quant
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -758,7 +759,7 @@ class FeedForward(nn.Module):
|
||||
|
||||
def forward(self, x):
|
||||
x, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(x)
|
||||
x = maybe_fused_act_quant(self.act_fn, x, self.down_proj)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
@@ -1259,7 +1260,7 @@ class PixtralHFMLP(nn.Module):
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_and_mul(gate_up)
|
||||
x = maybe_fused_act_quant(self.act_and_mul, gate_up, self.down_proj)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
@@ -29,6 +29,12 @@ from typing import ClassVar
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import (
|
||||
Qwen3_5Config,
|
||||
Qwen3_5MoeConfig,
|
||||
Qwen3_5MoeTextConfig,
|
||||
Qwen3_5TextConfig,
|
||||
)
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
@@ -59,11 +65,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig
|
||||
from vllm.transformers_utils.configs.qwen3_5_moe import (
|
||||
Qwen3_5MoeConfig,
|
||||
Qwen3_5MoeTextConfig,
|
||||
)
|
||||
|
||||
from .interfaces import (
|
||||
HasInnerState,
|
||||
|
||||
@@ -8,6 +8,7 @@ from collections.abc import Iterable
|
||||
import huggingface_hub
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen3_5MoeTextConfig, Qwen3_5TextConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
@@ -34,8 +35,6 @@ from vllm.model_executor.models.qwen3_next import (
|
||||
)
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5TextConfig
|
||||
from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig
|
||||
from vllm.transformers_utils.repo_utils import try_get_local_file
|
||||
|
||||
from .interfaces import (
|
||||
@@ -376,16 +375,20 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP):
|
||||
) -> torch.Tensor | None:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
return not name.startswith("mtp.") and not any(
|
||||
key in name for key in ["embed_tokens", "lm_head"]
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
def remap_weight_names(weights):
|
||||
for name, weight in weights:
|
||||
if self.is_unused_checkpoint_weight(name):
|
||||
continue
|
||||
if name.startswith("mtp."):
|
||||
name = name.replace("mtp.", "model.")
|
||||
elif any(key in name for key in ["embed_tokens", "lm_head"]):
|
||||
if "embed_tokens" in name:
|
||||
name = name.replace("language_model.", "")
|
||||
else:
|
||||
continue
|
||||
elif "embed_tokens" in name:
|
||||
name = name.replace("language_model.", "")
|
||||
yield name, weight
|
||||
|
||||
loader = AutoWeightsLoader(self)
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
StreamingTranscriptionPostProcessor,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
@@ -345,6 +346,7 @@ class Qwen3ASRForConditionalGeneration(
|
||||
SupportsMRoPE,
|
||||
SupportsTranscription,
|
||||
SupportsLoRA,
|
||||
SupportsEagle3,
|
||||
):
|
||||
# LoRA support
|
||||
packed_modules_mapping = {
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.multimodal.inputs import NestedTensors
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.config import set_default_rope_theta
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_bytes
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
@@ -269,6 +270,24 @@ class DFlashQwen3Attention(nn.Module):
|
||||
sinks=self.attention_sink_bias,
|
||||
)
|
||||
self.causal = causal
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx1x
|
||||
from vllm.v1.attention.backends.rocm_segmented_attn import (
|
||||
RocmSegmentedAttentionImpl,
|
||||
)
|
||||
|
||||
if on_gfx1x() and isinstance(self.attn.impl, RocmSegmentedAttentionImpl):
|
||||
# Startup segmented tuning needs the draft's actual attention mode.
|
||||
# Causality is normally supplied later through attention metadata.
|
||||
self.attn.segmented_causal = causal
|
||||
speculative_config = get_current_vllm_config().speculative_config
|
||||
if (
|
||||
speculative_config is not None
|
||||
and speculative_config.num_speculative_tokens is not None
|
||||
):
|
||||
self.attn.segmented_query_limit = (
|
||||
speculative_config.num_speculative_tokens + 1
|
||||
)
|
||||
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen3NextConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
@@ -58,7 +59,6 @@ from vllm.model_executor.models.qwen2_moe import Qwen2MoeMLP as Qwen3NextMLP
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
|
||||
from .interfaces import (
|
||||
@@ -232,6 +232,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
n_shared_experts=1 if self.shared_expert is None else None,
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
shared_expert_prefix=f"{prefix}.shared_expert",
|
||||
shared_expert_gate=self.shared_expert_gate
|
||||
if self.shared_expert is None
|
||||
else None,
|
||||
|
||||
@@ -6,6 +6,7 @@ from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen3NextConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
@@ -29,7 +30,6 @@ from vllm.model_executor.models.qwen3_next import (
|
||||
)
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
|
||||
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
@@ -227,15 +227,18 @@ class Qwen3NextMTP(nn.Module, QwenNextMixtureOfExperts):
|
||||
) -> torch.Tensor | None:
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
shared_weight_names = ["embed_tokens", "lm_head"]
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
return not name.startswith("mtp.") and not any(
|
||||
key in name for key in ["embed_tokens", "lm_head"]
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
def remap_weight_names(weights):
|
||||
for name, weight in weights:
|
||||
if self.is_unused_checkpoint_weight(name):
|
||||
continue
|
||||
if name.startswith("mtp."):
|
||||
name = name.replace("mtp.", "model.")
|
||||
elif not any(key in name for key in shared_weight_names):
|
||||
continue
|
||||
yield name, weight
|
||||
|
||||
loader = AutoWeightsLoader(self)
|
||||
|
||||
@@ -2311,8 +2311,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
llm_pos_ids_list.append(eos_pos)
|
||||
st = offset + video_len + 1
|
||||
else:
|
||||
audio_bos_pos = np.broadcast_to(np.array([st_idx - 1]), (3, 1))
|
||||
audio_bos_pos = np.broadcast_to(np.array([st_idx]), (3, 1))
|
||||
llm_pos_ids_list.append(audio_bos_pos)
|
||||
st_idx += 1
|
||||
|
||||
pos_ids, _ = self._compute_interleaved_positions(st_idx, data)
|
||||
llm_pos_ids_list.append(pos_ids)
|
||||
@@ -2320,7 +2321,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
|
||||
eos_pos = np.broadcast_to(np.array([st_idx]), (3, 1))
|
||||
llm_pos_ids_list.append(eos_pos)
|
||||
llm_pos_ids_list.append(eos_pos)
|
||||
llm_pos_ids_list.append(eos_pos + 1)
|
||||
|
||||
video_len = grid_t * grid_h * grid_w
|
||||
audio_len = self._compute_audio_token_count(
|
||||
|
||||
@@ -246,6 +246,7 @@ _EMBEDDING_MODELS = {
|
||||
# [Multimodal]
|
||||
"CLIPModel": ("clip", "CLIPEmbeddingModel"),
|
||||
"ColPaliForRetrieval": ("colpali", "ColPaliModel"),
|
||||
"EmbeddingGemma2Model": ("embedding_gemma2", "EmbeddingGemma2Model"),
|
||||
"LlamaNemotronVLModel": ("nemotron_vl", "LlamaNemotronVLForEmbedding"),
|
||||
"LlavaNextForConditionalGeneration": (
|
||||
"llava_next",
|
||||
@@ -849,6 +850,7 @@ _OOT_SUPPORTED_MODELS = {
|
||||
"BartForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
|
||||
"Florence2ForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
|
||||
"MBartForConditionalGeneration": "https://github.com/vllm-project/bart-plugin",
|
||||
"Nemotron3_5AsrForRNNT": "https://github.com/Sohaib-Ahmed21/vllm-nemotron-asr-plugin",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -120,9 +120,35 @@ def _expand_tensor(input_tensor: torch.Tensor, scaling: int) -> torch.Tensor:
|
||||
return (base.unsqueeze(1) + offsets).view(-1)
|
||||
|
||||
|
||||
class VoxtralRealtimeProcessingInfo(VoxtralProcessingInfo):
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
# Each streaming update carries a single audio frame.
|
||||
return {"audio": 1}
|
||||
|
||||
def get_max_audio_array_len(self) -> int:
|
||||
# Profile with the largest streaming chunk, the first one: it covers
|
||||
# the prompt prefix (BOS + left padding + delay), later chunks are
|
||||
# ~1 token. get_max_audio_tokens keeps max_model_len so the encoder
|
||||
# cache stays as large as before: each paused session keeps its last
|
||||
# chunk cached until it resumes, and a cache sized to one chunk
|
||||
# deadlocks a few concurrent sessions.
|
||||
tokenizer = self.get_tokenizer()
|
||||
audio_encoder = tokenizer.instruct.audio_encoder
|
||||
prompt_tokens = (
|
||||
tokenizer.instruct.start() + audio_encoder.encode_streaming_tokens()
|
||||
)
|
||||
return len(prompt_tokens) * audio_encoder.audio_config.raw_audio_length_per_tok
|
||||
|
||||
|
||||
class VoxtralRealtimeBuffer:
|
||||
def __init__(self, config: AudioConfig, prompt_tokens: list[int]) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
config: AudioConfig,
|
||||
prompt_tokens: list[int],
|
||||
max_model_len: int | None = None,
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._max_model_len = max_model_len
|
||||
|
||||
_look_ahead_in_ms = self._config.streaming_look_ahead_ms
|
||||
_look_back_in_ms = self._config.streaming_look_back_ms
|
||||
@@ -165,7 +191,16 @@ class VoxtralRealtimeBuffer:
|
||||
await self._token_queue.put(token)
|
||||
|
||||
async def get_input_stream(self) -> AsyncGenerator[TokensPrompt]:
|
||||
n_tokens = 0
|
||||
for frame_size, num_tokens in self._generate_frame_size_and_num_tokens():
|
||||
n_tokens += num_tokens
|
||||
if self._max_model_len is not None and n_tokens >= self._max_model_len:
|
||||
logger.warning(
|
||||
"Stopping realtime audio stream: reached max_model_len (%d).",
|
||||
self._max_model_len,
|
||||
)
|
||||
return
|
||||
|
||||
next_tokens = [await self._token_queue.get() for _ in range(num_tokens)]
|
||||
|
||||
audio_arrays: list[np.ndarray] = (
|
||||
@@ -198,7 +233,7 @@ class VoxtralRealtimeBuffer:
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
VoxtralRealtimeMultiModalProcessor,
|
||||
info=VoxtralProcessingInfo,
|
||||
info=VoxtralRealtimeProcessingInfo,
|
||||
dummy_inputs=VoxtralDummyInputsBuilder,
|
||||
)
|
||||
@support_torch_compile
|
||||
@@ -248,7 +283,9 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
|
||||
# Get left/right padding audio
|
||||
left_pad, right_pad = audio_encoder.get_padding_audio()
|
||||
|
||||
buffer = VoxtralRealtimeBuffer(config, prompt_tokens)
|
||||
buffer = VoxtralRealtimeBuffer(
|
||||
config, prompt_tokens, max_model_len=model_config.max_model_len
|
||||
)
|
||||
|
||||
# Feed audio with padding into buffer in background
|
||||
async def feed_audio():
|
||||
|
||||
@@ -896,6 +896,7 @@ class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixC
|
||||
head_dim=hf_config.mamba_headdim,
|
||||
state_size=hf_config.mamba_d_state,
|
||||
conv_kernel=hf_config.mamba_d_conv,
|
||||
num_spec=vllm_config.num_speculative_tokens,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
Reference in New Issue
Block a user