Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -400,8 +400,7 @@ class AXK1Attention(nn.Module):
|
||||
|
||||
|
||||
class AXK1MLAAttention(nn.Module):
|
||||
"""
|
||||
Main reference: DeepseekV2 paper, and FlashInfer Implementation
|
||||
"""Main reference: DeepseekV2 paper, and FlashInfer Implementation
|
||||
(https://arxiv.org/abs/2405.04434 and https://github.com/flashinfer-ai/flashinfer/pull/551).
|
||||
|
||||
For more info see MLACommonImpl in:
|
||||
|
||||
@@ -39,7 +39,6 @@ _GENERATE_SUFFIXES = [
|
||||
|
||||
def _load_st_projector(model_config: "ModelConfig") -> nn.Module | None:
|
||||
"""Load Sentence-Transformers Dense projection layers."""
|
||||
|
||||
dense_modules = try_get_dense_modules(
|
||||
model_config.model, revision=model_config.revision
|
||||
)
|
||||
@@ -249,8 +248,7 @@ def _create_pooling_model_cls(orig_cls: type[_T]) -> type[_T]:
|
||||
|
||||
|
||||
def as_embedding_model(cls: type[_T]) -> type[_T]:
|
||||
"""
|
||||
Subclass an existing vLLM model to support embeddings.
|
||||
"""Subclass an existing vLLM model to support embeddings.
|
||||
|
||||
By default, the embeddings of the whole prompt are extracted from the
|
||||
normalized hidden state corresponding to the last token.
|
||||
@@ -258,6 +256,7 @@ def as_embedding_model(cls: type[_T]) -> type[_T]:
|
||||
Note:
|
||||
We assume that no extra layers are added to the original model;
|
||||
please implement your own model if this is not the case.
|
||||
|
||||
"""
|
||||
# Avoid modifying existing embedding models
|
||||
if is_pooling_model(cls):
|
||||
@@ -306,8 +305,7 @@ def _resolve_num_labels(hf_config: Any, text_config: Any) -> int:
|
||||
|
||||
|
||||
def as_seq_cls_model(cls: type[_T]) -> type[_T]:
|
||||
"""
|
||||
Subclass an existing vLLM model to support classify and score tasks.
|
||||
"""Subclass an existing vLLM model to support classify and score tasks.
|
||||
|
||||
By default, the class probabilities are extracted from the softmaxed
|
||||
hidden state corresponding to the last token.
|
||||
@@ -316,6 +314,7 @@ def as_seq_cls_model(cls: type[_T]) -> type[_T]:
|
||||
We assume that the classification head is a single linear layer
|
||||
stored as the attribute `score` of the top-level model;
|
||||
please implement your own model if this is not the case.
|
||||
|
||||
"""
|
||||
# Avoid modifying existing classification models
|
||||
if is_pooling_model(cls):
|
||||
@@ -446,8 +445,7 @@ class SequenceClassificationConfig(VerifyAndUpdateConfig):
|
||||
|
||||
|
||||
def _get_language_model_for_seq_cls(model: nn.Module) -> nn.Module:
|
||||
"""
|
||||
Get the language model component for sequence classification conversion.
|
||||
"""Get the language model component for sequence classification conversion.
|
||||
For VLMs, returns the inner language model. For standard LLMs, returns model itself.
|
||||
"""
|
||||
multimodal_model: object = model
|
||||
@@ -481,8 +479,7 @@ def _get_language_model_for_seq_cls(model: nn.Module) -> nn.Module:
|
||||
|
||||
@contextmanager
|
||||
def _disable_seq_cls_loading_on_inner_model(language_model, is_vlm: bool):
|
||||
"""
|
||||
Context manager to temporarily disable sequence classification loading
|
||||
"""Context manager to temporarily disable sequence classification loading
|
||||
on inner VLM models to prevent recursive seq_cls_model_loader calls.
|
||||
"""
|
||||
if not is_vlm:
|
||||
|
||||
@@ -47,7 +47,8 @@ from .utils import (
|
||||
|
||||
class ArceeMLP(nn.Module):
|
||||
"""Feed-forward layer for Arcee using ReLU^2 activation
|
||||
(no gating as in LLaMA)."""
|
||||
(no gating as in LLaMA).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -95,7 +96,8 @@ class ArceeMLP(nn.Module):
|
||||
|
||||
class ArceeDecoderLayer(nn.Module):
|
||||
"""Transformer decoder block for Arcee, with self-attention and
|
||||
ReLU^2 MLP."""
|
||||
ReLU^2 MLP.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -175,7 +177,8 @@ class ArceeDecoderLayer(nn.Module):
|
||||
@support_torch_compile
|
||||
class ArceeModel(nn.Module, EagleModelMixin):
|
||||
"""The transformer model backbone for Arcee (embedding layer + stacked
|
||||
decoder blocks + final norm)."""
|
||||
decoder blocks + final norm).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -277,7 +280,8 @@ class ArceeForCausalLM(
|
||||
nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
|
||||
):
|
||||
"""Arcee Model for causal language modeling, integrated with vLLM
|
||||
runtime."""
|
||||
runtime.
|
||||
"""
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
@@ -358,7 +362,8 @@ class ArceeForCausalLM(
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights into the model (delegates to inner model and handles
|
||||
tied embeddings)."""
|
||||
tied embeddings).
|
||||
"""
|
||||
loader = AutoWeightsLoader(self)
|
||||
# AutoWeightLoader handles weight name remapping, including fusing
|
||||
# separate q_proj, k_proj, v_proj into qkv_proj
|
||||
|
||||
@@ -41,13 +41,12 @@ from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
|
||||
|
||||
class AriaImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- b: Batch size
|
||||
- n: Number of images
|
||||
- c: Number of channels
|
||||
- h: Height of each image
|
||||
- w: Width of each image
|
||||
"""Dimensions:
|
||||
- b: Batch size
|
||||
- n: Number of images
|
||||
- c: Number of channels
|
||||
- h: Height of each image
|
||||
- w: Width of each image
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
@@ -119,8 +118,7 @@ class AriaProjectorMLP(nn.Module):
|
||||
|
||||
|
||||
class AriaProjector(nn.Module):
|
||||
"""
|
||||
A projection module with one cross attention layer and one FFN layer, which
|
||||
"""A projection module with one cross attention layer and one FFN layer, which
|
||||
projects ViT's outputs into MoE's inputs.
|
||||
|
||||
Args:
|
||||
@@ -129,6 +127,7 @@ class AriaProjector(nn.Module):
|
||||
|
||||
Outputs:
|
||||
A tensor with the shape of (batch_size, query_number, output_dim)
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, config: AriaConfig, prefix: str = "") -> None:
|
||||
@@ -187,8 +186,7 @@ class AriaProjector(nn.Module):
|
||||
|
||||
|
||||
class AriaTextMoELayer(nn.Module):
|
||||
"""
|
||||
Mixture of Experts (MoE) Layer for the AriaMoE model.
|
||||
"""Mixture of Experts (MoE) Layer for the AriaMoE model.
|
||||
|
||||
This layer implements the MoE mechanism, which routes input tokens to
|
||||
different experts based on a routing algorithm, processes them through the
|
||||
@@ -227,8 +225,7 @@ class AriaTextMoELayer(nn.Module):
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass of the MoE Layer.
|
||||
"""Forward pass of the MoE Layer.
|
||||
|
||||
Args:
|
||||
hidden_states: Input tensor of shape
|
||||
@@ -236,16 +233,15 @@ class AriaTextMoELayer(nn.Module):
|
||||
|
||||
Returns:
|
||||
torch.Tensor: Output tensor after passing through the MoE layer.
|
||||
"""
|
||||
|
||||
"""
|
||||
router_output = torch.nn.functional.linear(hidden_states, self.router_weight)
|
||||
|
||||
return self.experts(hidden_states, router_output)
|
||||
|
||||
|
||||
class AriaTextDecoderLayer(LlamaDecoderLayer):
|
||||
"""
|
||||
Custom Decoder Layer for the AriaMoE model which modifies the standard
|
||||
"""Custom Decoder Layer for the AriaMoE model which modifies the standard
|
||||
`LlamaDecoderLayer` by replacing the traditional MLP with a Mixture of
|
||||
Experts (MoE) Layer.
|
||||
"""
|
||||
@@ -262,8 +258,7 @@ class AriaTextDecoderLayer(LlamaDecoderLayer):
|
||||
|
||||
|
||||
class AriaTextModel(LlamaModel, SupportsQuant):
|
||||
"""
|
||||
Custom LlamaModel for the AriaMoE model which modifies the standard
|
||||
"""Custom LlamaModel for the AriaMoE model which modifies the standard
|
||||
LlamaModel by replacing the `LlamaDecoderLayer` with `MoEDecoderLayer`.
|
||||
"""
|
||||
|
||||
@@ -377,8 +372,7 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor[AriaProcessingInfo]):
|
||||
dummy_inputs=AriaDummyInputsBuilder,
|
||||
)
|
||||
class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
|
||||
"""
|
||||
Aria model for conditional generation tasks.
|
||||
"""Aria model for conditional generation tasks.
|
||||
|
||||
This model combines a vision tower, a multi-modal projector, and a language
|
||||
model to perform tasks that involve both image and text inputs.
|
||||
|
||||
@@ -71,11 +71,10 @@ from .utils import (
|
||||
|
||||
|
||||
class AudioFlamingo3FeatureInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- num_chunks: Number of audio chunks (flattened)
|
||||
- nmb: Number of mel bins
|
||||
- num_audios: Number of original audio files
|
||||
"""Dimensions:
|
||||
- num_chunks: Number of audio chunks (flattened)
|
||||
- nmb: Number of mel bins
|
||||
- num_audios: Number of original audio files
|
||||
"""
|
||||
|
||||
type: Literal["audio_features"]
|
||||
@@ -96,12 +95,11 @@ class AudioFlamingo3FeatureInputs(TensorSchema):
|
||||
|
||||
|
||||
class AudioFlamingo3EmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size
|
||||
- naf: Number of audio features
|
||||
- hs: Hidden size (must match the hidden size of language model
|
||||
backbone)
|
||||
"""Dimensions:
|
||||
- bn: Batch size
|
||||
- naf: Number of audio features
|
||||
- hs: Hidden size (must match the hidden size of language model
|
||||
backbone)
|
||||
"""
|
||||
|
||||
type: Literal["audio_embeds"] = "audio_embeds"
|
||||
@@ -154,8 +152,7 @@ class AudioFlamingo3Encoder(Qwen2AudioEncoder):
|
||||
return hidden_states
|
||||
|
||||
def _get_feat_extract_output_lengths(self, input_lengths: torch.Tensor):
|
||||
"""
|
||||
Computes the output length of the convolutional layers and the output length
|
||||
"""Computes the output length of the convolutional layers and the output length
|
||||
of the audio encoder
|
||||
"""
|
||||
input_lengths = (input_lengths - 1) // 2 + 1
|
||||
@@ -503,9 +500,7 @@ class AudioFlamingo3ForConditionalGeneration(
|
||||
}
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models."""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model.",
|
||||
connector="multi_modal_projector.",
|
||||
|
||||
@@ -58,12 +58,11 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class BagelImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each image
|
||||
- w: Width of each image
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each image
|
||||
- w: Width of each image
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
@@ -175,12 +174,12 @@ class PositionEmbedding(nn.Module):
|
||||
return emb
|
||||
|
||||
def forward(self, position_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
position_ids: Flattened position IDs, shape (N,) where each ID
|
||||
corresponds to a position in the flattened grid
|
||||
Returns:
|
||||
Position embeddings of shape (N, hidden_size)
|
||||
|
||||
"""
|
||||
# Ensure position_ids are on the same device as pos_embed
|
||||
position_ids = position_ids.to(self.pos_embed.device)
|
||||
@@ -322,8 +321,7 @@ class BagelMultiModalProcessor(BaseMultiModalProcessor[BagelProcessingInfo]):
|
||||
class BagelForConditionalGeneration(
|
||||
nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP
|
||||
):
|
||||
"""
|
||||
BAGEL: A unified multimodal model for image understanding and generation.
|
||||
"""BAGEL: A unified multimodal model for image understanding and generation.
|
||||
|
||||
For vLLM, we focus on the image understanding (vision-to-text) capabilities.
|
||||
The image generation part is not supported in vLLM.
|
||||
@@ -507,6 +505,9 @@ class BagelForConditionalGeneration(
|
||||
positions: Flattened (concatenated) position ids corresponding to a batch.
|
||||
intermediate_tensors: Intermediate tensors from prior forward pass.
|
||||
inputs_embeds: Optional tensor of input embeddings.
|
||||
**kwargs: Multimodal inputs for this batch, forwarded to the
|
||||
multimodal embedding path.
|
||||
|
||||
"""
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
|
||||
@@ -75,9 +75,7 @@ def is_linear_layer(layer_idx, layer_group_size):
|
||||
|
||||
|
||||
class BailingMoeV25MLAAttention(nn.Module):
|
||||
"""
|
||||
MLA Attention for BailingMoeV2.5 full attention layers.
|
||||
"""
|
||||
"""MLA Attention for BailingMoeV2.5 full attention layers."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -594,7 +592,6 @@ class BailingMoeV25Model(nn.Module):
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load checkpoint weights with simplified mapping."""
|
||||
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
|
||||
@@ -458,6 +458,7 @@ class BertEmbeddingModel(nn.Module, SupportsQuant):
|
||||
Attributes:
|
||||
model: An instance of BertModel used for forward operations.
|
||||
_pooler: An instance of Pooler used for pooling operations.
|
||||
|
||||
"""
|
||||
|
||||
is_pooling_model = True
|
||||
@@ -575,8 +576,7 @@ class BertMLMHead(nn.Module):
|
||||
|
||||
|
||||
class SPLADESparsePooler(Pooler):
|
||||
"""
|
||||
SPLADE sparse pooling:
|
||||
"""SPLADE sparse pooling:
|
||||
logits = mlm_head(hidden_states)
|
||||
-> log1p(relu(logits))
|
||||
-> (max|sum over L)
|
||||
@@ -664,8 +664,7 @@ class SPLADESparsePooler(Pooler):
|
||||
|
||||
@default_pooling_type(seq_pooling_type="CLS")
|
||||
class BertSpladeSparseEmbeddingModel(BertEmbeddingModel):
|
||||
"""
|
||||
BertEmbeddingModel + SPLADE sparse embedding.
|
||||
"""BertEmbeddingModel + SPLADE sparse embedding.
|
||||
- Make logits by self.mlm_head
|
||||
- pooler: SPLADESparsePooler(mlm_head...)
|
||||
"""
|
||||
@@ -779,6 +778,7 @@ class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQu
|
||||
Attributes:
|
||||
model: An instance of BertModel used for forward operations.
|
||||
_pooler: An instance of Pooler used for pooling operations.
|
||||
|
||||
"""
|
||||
|
||||
is_pooling_model = True
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Minimal implementation of BlipVisionModel intended to be only used
|
||||
within a vision language model."""
|
||||
within a vision language model.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
@@ -82,7 +83,7 @@ class BlipVisionEmbeddings(nn.Module):
|
||||
|
||||
|
||||
class BlipAttention(nn.Module):
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -140,8 +141,7 @@ class BlipAttention(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
):
|
||||
"""Input shape: Batch x Time x Channel"""
|
||||
|
||||
"""Input shape: Batch x Time x Channel."""
|
||||
qkv_states, _ = self.qkv(hidden_states)
|
||||
query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
|
||||
out = self.attn(query_states, key_states, value_states)
|
||||
@@ -220,12 +220,12 @@ class BlipEncoderLayer(nn.Module):
|
||||
|
||||
|
||||
class BlipEncoder(nn.Module):
|
||||
"""
|
||||
Transformer encoder consisting of `config.num_hidden_layers` self
|
||||
"""Transformer encoder consisting of `config.num_hidden_layers` self
|
||||
attention layers. Each layer is a [`BlipEncoderLayer`].
|
||||
|
||||
Args:
|
||||
config: BlipConfig
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -50,12 +50,11 @@ from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
|
||||
|
||||
|
||||
class Blip2ImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each image
|
||||
- w: Width of each image
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each image
|
||||
- w: Width of each image
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
@@ -63,11 +62,10 @@ class Blip2ImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class Blip2ImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- f: Image feature size
|
||||
- h: Hidden size (must match the hidden size of language model backbone)
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- f: Image feature size
|
||||
- h: Hidden size (must match the hidden size of language model backbone)
|
||||
"""
|
||||
|
||||
type: Literal["image_embeds"]
|
||||
@@ -657,11 +655,15 @@ class Blip2ForConditionalGeneration(
|
||||
Args:
|
||||
input_ids: Flattened (concatenated) input_ids corresponding to a
|
||||
batch.
|
||||
positions: Flattened (concatenated) position ids corresponding to a
|
||||
batch.
|
||||
intermediate_tensors: Intermediate tensors from prior forward pass.
|
||||
inputs_embeds: Optional tensor of input embeddings.
|
||||
|
||||
Info:
|
||||
[`Blip2ImageInputs`][vllm.model_executor.models.blip2.Blip2ImageInputs]
|
||||
"""
|
||||
|
||||
"""
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
|
||||
|
||||
@@ -67,12 +67,11 @@ from .vision import (
|
||||
|
||||
|
||||
class CLIPImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each image
|
||||
- w: Width of each image
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each image
|
||||
- w: Width of each image
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
@@ -417,8 +416,7 @@ class CLIPAttention(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
):
|
||||
"""Input shape: Batch x Time x Channel"""
|
||||
|
||||
"""Input shape: Batch x Time x Channel."""
|
||||
qkv_states, _ = self.qkv_proj(hidden_states)
|
||||
query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
|
||||
out = self.attn(query_states, key_states, value_states)
|
||||
@@ -506,12 +504,12 @@ class CLIPEncoderLayer(nn.Module):
|
||||
|
||||
|
||||
class CLIPEncoder(nn.Module):
|
||||
"""
|
||||
Transformer encoder consisting of `config.num_hidden_layers` self
|
||||
"""Transformer encoder consisting of `config.num_hidden_layers` self
|
||||
attention layers. Each layer is a [`CLIPEncoderLayer`].
|
||||
|
||||
Args:
|
||||
config: CLIPConfig
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -96,7 +96,8 @@ class RMSNorm(nn.Module):
|
||||
|
||||
def select_norm_impl(config: CohereConfig) -> tuple[type[nn.Module], float]:
|
||||
"""Returns (norm_class, eps). Uses RMSNorm when config.rms_norm_eps is set,
|
||||
otherwise falls back to LayerNorm with config.layer_norm_eps."""
|
||||
otherwise falls back to LayerNorm with config.layer_norm_eps.
|
||||
"""
|
||||
rms_eps = getattr(config, "rms_norm_eps", None)
|
||||
if rms_eps is not None:
|
||||
return RMSNorm, rms_eps
|
||||
|
||||
@@ -61,14 +61,13 @@ from .utils import (
|
||||
|
||||
|
||||
class Cohere2VisionImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- np: The total number of patches over each image over each prompt in
|
||||
the batch
|
||||
- c: Number of channels
|
||||
- h: Height of each image patch
|
||||
- w: Width of each image patch
|
||||
- bn: Batch size * number of images
|
||||
"""Dimensions:
|
||||
- np: The total number of patches over each image over each prompt in
|
||||
the batch
|
||||
- c: Number of channels
|
||||
- h: Height of each image patch
|
||||
- w: Width of each image patch
|
||||
- bn: Batch size * number of images
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
@@ -134,6 +133,7 @@ class Cohere2VisionMultiModalProjector(nn.Module):
|
||||
|
||||
Returns:
|
||||
Downsampled tensor with increased channel dimension
|
||||
|
||||
"""
|
||||
height = width = int(image_features.shape[1] ** 0.5)
|
||||
x = image_features.reshape(image_features.shape[0], width, height, -1)
|
||||
@@ -175,8 +175,7 @@ class Cohere2VisionProcessingInfo(BaseProcessingInfo):
|
||||
processor: Cohere2VisionProcessor,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> int:
|
||||
"""
|
||||
Calculate the number of image patches for a given image.
|
||||
"""Calculate the number of image patches for a given image.
|
||||
Uses the HF processor to determine the actual number of patches.
|
||||
"""
|
||||
image_processor: Cohere2VisionImageProcessorFast = processor.image_processor
|
||||
@@ -372,16 +371,18 @@ class Cohere2VisionForConditionalGeneration(
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def _process_image_input(
|
||||
self, image_input: Cohere2VisionImagePixelInputs, **kwargs
|
||||
self, image_input: Cohere2VisionImagePixelInputs, **kwargs: object
|
||||
) -> list[torch.Tensor]:
|
||||
"""Process image pixels through vision tower and projector.
|
||||
|
||||
Args:
|
||||
image_input: Validated image input containing pixel values and
|
||||
patch counts
|
||||
**kwargs: Unused; accepted for interface compatibility.
|
||||
|
||||
Returns:
|
||||
List of flattened image embeddings, one per image
|
||||
|
||||
"""
|
||||
pixel_values = image_input["pixel_values"]
|
||||
num_patches = image_input["num_patches"]
|
||||
|
||||
@@ -300,13 +300,13 @@ class CohereASRMLP(nn.Module):
|
||||
|
||||
|
||||
class FixedPositionalEncoding(nn.Module):
|
||||
"""
|
||||
Fixed positional encoding (embedding layer) from sine and cosine functions
|
||||
"""Fixed positional encoding (embedding layer) from sine and cosine functions
|
||||
of different frequencies according to https://arxiv.org/abs/1706.03762
|
||||
|
||||
Args:
|
||||
hidden_size: size of the embeddings in the model, also known as d_model
|
||||
max_sequence_length: maximum allowed length of the input sequence
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, hidden_size: int, max_sequence_length: int = 512) -> None:
|
||||
@@ -664,7 +664,8 @@ class ConvSubsampling(nn.Module):
|
||||
repeat_num: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""Calculates the output length of a Tensor passed
|
||||
through a convolution or max pooling layer"""
|
||||
through a convolution or max pooling layer
|
||||
"""
|
||||
add_pad: float = all_paddings - kernel_size
|
||||
one: float = 1.0
|
||||
for i in range(repeat_num):
|
||||
@@ -689,10 +690,12 @@ class ConvSubsampling(nn.Module):
|
||||
|
||||
class PositionalEncoding(torch.nn.Module):
|
||||
"""Fixed sinusoidal positional encoding.
|
||||
|
||||
Args:
|
||||
d_model (int): embedding dim
|
||||
max_len (int): maximum input length
|
||||
xscale (bool): whether to scale the input by sqrt(d_model)
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -724,12 +727,14 @@ class PositionalEncoding(torch.nn.Module):
|
||||
self, x: torch.Tensor, cache_len: int = 0
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Adds positional encoding.
|
||||
|
||||
Args:
|
||||
x (torch.Tensor): Input. Its shape is (batch, time, feature_size)
|
||||
cache_len (int): the size of the cache which is used to shift positions
|
||||
Returns:
|
||||
x+pos_emb (torch.Tensor): Its shape is (batch, time, feature_size)
|
||||
pos_emb (torch.Tensor): Its shape is (1, time, feature_size)
|
||||
|
||||
"""
|
||||
input_len = x.size(1) + cache_len
|
||||
if self.xscale:
|
||||
@@ -762,14 +767,15 @@ class RelPositionalEncoding(PositionalEncoding):
|
||||
self, x: torch.Tensor, cache_len: int = 0
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Compute positional encoding.
|
||||
|
||||
Args:
|
||||
x (torch.Tensor): Input. Its shape is (batch, time, feature_size)
|
||||
cache_len (int): the size of the cache which is used to shift positions
|
||||
Returns:
|
||||
x (torch.Tensor): Its shape is (batch, time, feature_size)
|
||||
pos_emb (torch.Tensor): Its shape is (1, time, feature_size)
|
||||
"""
|
||||
|
||||
"""
|
||||
if self.xscale:
|
||||
x = x * self.xscale
|
||||
|
||||
@@ -783,15 +789,13 @@ class RelPositionalEncoding(PositionalEncoding):
|
||||
|
||||
|
||||
class Swish(nn.SiLU):
|
||||
"""
|
||||
Swish activation function introduced in 'https://arxiv.org/abs/1710.05941'
|
||||
"""Swish activation function introduced in 'https://arxiv.org/abs/1710.05941'
|
||||
Mathematically identical to SiLU. See note in nn.SiLU for references.
|
||||
"""
|
||||
|
||||
|
||||
class ConformerFeedForward(nn.Module):
|
||||
"""
|
||||
feed-forward module of Conformer model.
|
||||
"""feed-forward module of Conformer model.
|
||||
use_bias (bool): Apply bias to all Linear and Conv1d
|
||||
layers to improve activation flow and stabilize
|
||||
training of huge models.
|
||||
@@ -819,8 +823,7 @@ class ConformerFeedForward(nn.Module):
|
||||
|
||||
|
||||
class CausalConv1D(nn.Conv1d):
|
||||
"""
|
||||
A causal version of nn.Conv1d where each step would
|
||||
"""A causal version of nn.Conv1d where each step would
|
||||
have limited access to locations on its right or left.
|
||||
All arguments are the same as nn.Conv1d except padding.
|
||||
|
||||
@@ -893,6 +896,7 @@ class CausalConv1D(nn.Conv1d):
|
||||
|
||||
class ConformerConvolution(nn.Module):
|
||||
"""The convolution module for the Conformer model.
|
||||
|
||||
Args:
|
||||
d_model (int): hidden dimension
|
||||
kernel_size (int): kernel size for depthwise convolution
|
||||
@@ -904,6 +908,7 @@ class ConformerConvolution(nn.Module):
|
||||
use_bias (bool): Use bias in all Linear and Conv1d
|
||||
layers to improve activation flow and stabilize
|
||||
training of huge models. Defaults to True
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -979,10 +984,12 @@ class ConformerConvolution(nn.Module):
|
||||
|
||||
class CohereASRMultiHeadAttention(nn.Module):
|
||||
"""Multi-Head Attention layer of Transformer.
|
||||
|
||||
Args:
|
||||
n_head (int): number of heads
|
||||
n_feat (int): size of the features
|
||||
use_bias (bool): whether to remove bias in linear and conv layers
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -1010,6 +1017,7 @@ class CohereASRMultiHeadAttention(nn.Module):
|
||||
value: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Transforms query, key and value.
|
||||
|
||||
Args:
|
||||
query (torch.Tensor): (batch, time1, size)
|
||||
key (torch.Tensor): (batch, time2, size)
|
||||
@@ -1018,6 +1026,7 @@ class CohereASRMultiHeadAttention(nn.Module):
|
||||
q (torch.Tensor): (batch, head, time1, size)
|
||||
k (torch.Tensor): (batch, head, time2, size)
|
||||
v (torch.Tensor): (batch, head, time2, size)
|
||||
|
||||
"""
|
||||
n_batch = query.size(0)
|
||||
q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k)
|
||||
@@ -1036,6 +1045,7 @@ class CohereASRMultiHeadAttention(nn.Module):
|
||||
mask: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""Compute attention context vector.
|
||||
|
||||
Args:
|
||||
value (torch.Tensor): (batch, time2, size)
|
||||
scores(torch.Tensor): (batch, time1, time2)
|
||||
@@ -1044,6 +1054,7 @@ class CohereASRMultiHeadAttention(nn.Module):
|
||||
value (torch.Tensor): transformed `value`
|
||||
(batch, time2, d_model) weighted by the
|
||||
attention scores
|
||||
|
||||
"""
|
||||
n_batch = value.size(0)
|
||||
if mask is not None:
|
||||
@@ -1071,16 +1082,19 @@ class CohereASRMultiHeadAttention(nn.Module):
|
||||
pos_emb: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Compute 'Scaled Dot Product Attention'.
|
||||
|
||||
Args:
|
||||
query (torch.Tensor): (batch, time1, size)
|
||||
key (torch.Tensor): (batch, time2, size)
|
||||
value(torch.Tensor): (batch, time2, size)
|
||||
mask (torch.Tensor): (batch, time1, time2)
|
||||
pos_emb (torch.Tensor): optional relative position embeddings
|
||||
|
||||
returns:
|
||||
Returns:
|
||||
output (torch.Tensor): transformed `value`
|
||||
(batch, time1, d_model) weighted by the
|
||||
query dot key attention
|
||||
|
||||
"""
|
||||
q, k, v = self.forward_qkv(query, key, value)
|
||||
|
||||
@@ -1130,8 +1144,10 @@ class RelPositionMultiHeadAttention(CohereASRMultiHeadAttention):
|
||||
|
||||
def rel_shift(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""Compute relative positional encoding.
|
||||
|
||||
Args:
|
||||
x (torch.Tensor): (batch, nheads, time, 2*time-1)
|
||||
|
||||
"""
|
||||
b, h, qlen, pos_len = x.size() # (b, h, t1, t2)
|
||||
# need to add a column of zeros on the left side of
|
||||
@@ -1151,17 +1167,19 @@ class RelPositionMultiHeadAttention(CohereASRMultiHeadAttention):
|
||||
pos_emb: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Compute 'Scaled Dot Product Attention' with rel. positional encoding.
|
||||
|
||||
Args:
|
||||
query (torch.Tensor): (batch, time1, size)
|
||||
key (torch.Tensor): (batch, time2, size)
|
||||
value(torch.Tensor): (batch, time2, size)
|
||||
mask (torch.Tensor): (batch, time1, time2)
|
||||
pos_emb (torch.Tensor) : (batch, time1, size)
|
||||
pos_emb (torch.Tensor): (batch, time1, size)
|
||||
|
||||
Returns:
|
||||
output (torch.Tensor): transformed `value`
|
||||
(batch, time1, d_model) weighted by the
|
||||
query dot key attention
|
||||
|
||||
"""
|
||||
q, k, v = self.forward_qkv(query, key, value)
|
||||
q = q.transpose(1, 2) # (batch, time1, head, d_k)
|
||||
@@ -1212,6 +1230,7 @@ class ConformerLayer(torch.nn.Module):
|
||||
Conv1d layers from each ConformerLayer to
|
||||
improve activation flow and stabilize training
|
||||
of huge models. Defaults to True.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -1279,14 +1298,14 @@ class ConformerLayer(torch.nn.Module):
|
||||
pos_emb: torch.Tensor | None = None,
|
||||
pad_mask: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
x (torch.Tensor): input signals (B, T, d_model)
|
||||
att_mask (torch.Tensor): attention masks(B, T, T)
|
||||
pos_emb (torch.Tensor): (L, 1, d_model)
|
||||
pad_mask (torch.tensor): padding mask
|
||||
Returns:
|
||||
x (torch.Tensor): (B, T, d_model)
|
||||
|
||||
"""
|
||||
residual = x
|
||||
x = self.norm_feed_forward1(x)
|
||||
@@ -1331,8 +1350,7 @@ class ConformerLayer(torch.nn.Module):
|
||||
|
||||
|
||||
class ConformerEncoder(nn.Module):
|
||||
"""
|
||||
The encoder for ASR model of Conformer.
|
||||
"""The encoder for ASR model of Conformer.
|
||||
Based on this paper:
|
||||
'Conformer: Convolution-augmented Transformer for
|
||||
Speech Recognition' by Anmol Gulati et al.
|
||||
@@ -1473,12 +1491,12 @@ class ConformerEncoder(nn.Module):
|
||||
return num_encoder_cross_attn_tokens
|
||||
|
||||
def set_max_audio_length(self, max_audio_length: int) -> None:
|
||||
"""
|
||||
Sets maximum input length.
|
||||
"""Sets maximum input length.
|
||||
Pre-calculates internal seq_range mask.
|
||||
|
||||
Args:
|
||||
max_audio_length (int): New maximum sequence length.
|
||||
|
||||
"""
|
||||
device = next(self.parameters()).device
|
||||
dtype = next(self.parameters()).dtype
|
||||
|
||||
@@ -338,8 +338,7 @@ def pos_embed_interpolate_native(
|
||||
|
||||
|
||||
class Cohere_VLImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
"""Dimensions:
|
||||
- np: Number of patches
|
||||
- ni: Number of images
|
||||
- cps: Number of channels * patch_size * patch_size
|
||||
@@ -365,8 +364,7 @@ class Cohere_VLImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class Cohere_VLImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
"""Dimensions:
|
||||
- nf: Number of image features
|
||||
- hs: Hidden size
|
||||
- ni: Number of images
|
||||
@@ -895,6 +893,7 @@ class Cohere_VisionTransformer(nn.Module):
|
||||
instead of computing from cu_seqlens (needed for CUDA
|
||||
graph capture to cover worst-case replay scenarios).
|
||||
device: Device to place tensors on. Defaults to self.device.
|
||||
|
||||
"""
|
||||
if device is None:
|
||||
device = self.device
|
||||
@@ -1744,8 +1743,7 @@ class CohereCompassForConditionalGeneration(
|
||||
image_embeds_split: tuple[torch.Tensor, ...],
|
||||
image_input: Cohere_VLImageInputs,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""
|
||||
Append mrope positions for each for images.
|
||||
"""Append mrope positions for each for images.
|
||||
This is necessary to recover correct mrope positions
|
||||
|
||||
Args:
|
||||
@@ -1757,6 +1755,7 @@ class CohereCompassForConditionalGeneration(
|
||||
Tuple of image embeddings for each image item.
|
||||
Resulting embeddings will have extra 5 channels for
|
||||
computed mrope positions, consistent with video embeddings.
|
||||
|
||||
"""
|
||||
if self.is_multimodal_pruning_enabled:
|
||||
merge_size = self.visual.spatial_merge_size
|
||||
@@ -1837,8 +1836,7 @@ class CohereCompassForConditionalGeneration(
|
||||
mrope_positions: torch.LongTensor,
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[MultiModalEmbeddings, torch.Tensor, int]:
|
||||
"""
|
||||
Update part of input mrope positions (starting with
|
||||
"""Update part of input mrope positions (starting with
|
||||
num_computed_tokens index). Original mrope_positions are computed
|
||||
for unpruned sequence and becomes incorrect once pruning occurs,
|
||||
so once we prune media tokens we should reflect this in the
|
||||
@@ -1856,6 +1854,7 @@ class CohereCompassForConditionalGeneration(
|
||||
Returns:
|
||||
Tuple of (multimodal_embeddings, mrope_positions,
|
||||
mrope_position_delta).
|
||||
|
||||
"""
|
||||
return self._recompute_mrope_positions(
|
||||
input_ids=input_ids,
|
||||
@@ -2046,8 +2045,8 @@ class CohereCompassForConditionalGeneration(
|
||||
`None` if no images are passed.
|
||||
- image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in
|
||||
LLM. `None` if no images are passed.
|
||||
"""
|
||||
|
||||
"""
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
|
||||
@@ -2083,9 +2082,7 @@ class CohereCompassForConditionalGeneration(
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models"""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector=["visual.merger", "visual.deepstack_merger_list"],
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
ColBERT late interaction model for retrieval and reranking.
|
||||
"""ColBERT late interaction model for retrieval and reranking.
|
||||
|
||||
ColBERT uses per-token embeddings and late interaction (MaxSim) scoring
|
||||
instead of single-vector representations or cross-encoder concatenation.
|
||||
@@ -75,6 +74,7 @@ class ColBERTMixin(nn.Module, SupportsLateInteraction):
|
||||
``None``, will be inferred from weights during loading (or
|
||||
auto-loaded from sentence-transformers Dense layers).
|
||||
head_dtype: Data type for the projection layer.
|
||||
|
||||
"""
|
||||
self.hidden_size = hidden_size
|
||||
self.colbert_dim = colbert_dim
|
||||
@@ -149,6 +149,7 @@ class ColBERTMixin(nn.Module, SupportsLateInteraction):
|
||||
Returns:
|
||||
``(remaining_weights, loaded_names)`` — the weights that were
|
||||
**not** consumed and the set of names that were loaded.
|
||||
|
||||
"""
|
||||
weights_list = list(weights)
|
||||
other_weights: list[tuple[str, torch.Tensor]] = []
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
ColPali late interaction model for multi-modal retrieval and reranking.
|
||||
"""ColPali late interaction model for multi-modal retrieval and reranking.
|
||||
|
||||
ColPali extends PaliGemma with a ColBERT-style late interaction head,
|
||||
producing per-token embeddings for both text and image inputs. It uses
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
ColQwen3 late interaction model for multi-modal retrieval and reranking.
|
||||
"""ColQwen3 late interaction model for multi-modal retrieval and reranking.
|
||||
|
||||
ColQwen3 extends Qwen3-VL with a ColBERT-style late interaction head,
|
||||
producing per-token embeddings for both text and image inputs. It uses
|
||||
@@ -131,6 +130,7 @@ class ColQwen3Model(Qwen3VLForConditionalGeneration, SupportsLateInteraction):
|
||||
|
||||
Attributes:
|
||||
custom_text_proj: Linear projection from hidden_size to embed_dim
|
||||
|
||||
"""
|
||||
|
||||
# Mark this as a pooling model so vLLM routes to pooler path
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
ColQwen3.5 late interaction model for multi-modal retrieval and reranking.
|
||||
"""ColQwen3.5 late interaction model for multi-modal retrieval and reranking.
|
||||
|
||||
ColQwen3.5 extends Qwen3.5 with a ColBERT-style late interaction head,
|
||||
producing per-token embeddings for both text and image inputs. It uses
|
||||
@@ -128,6 +127,7 @@ class ColQwen3_5Model(
|
||||
Attributes:
|
||||
custom_text_proj: Linear projection from hidden_size to embed_dim.
|
||||
This is passed to the pooler as pooler.head.projector.
|
||||
|
||||
"""
|
||||
|
||||
# Mark this as a pooling model so vLLM routes to pooler path
|
||||
|
||||
@@ -114,8 +114,7 @@ class RMSNorm(nn.Module):
|
||||
|
||||
|
||||
def select_norm_impl(config: CohereConfig) -> tuple[type[nn.Module], float]:
|
||||
"""
|
||||
Returns the normalization layer class and epsilon value to use.
|
||||
"""Returns the normalization layer class and epsilon value to use.
|
||||
If `config.rms_norm_eps` is present, use RMSNorm.
|
||||
Otherwise default to LayerNorm with `config.layer_norm_eps`.
|
||||
"""
|
||||
|
||||
@@ -445,8 +445,7 @@ class GteNewModelConfig(VerifyAndUpdateConfig):
|
||||
class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig):
|
||||
@classmethod
|
||||
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
|
||||
"""
|
||||
Perform early validation and setup for hybrid attention/mamba models.
|
||||
"""Perform early validation and setup for hybrid attention/mamba models.
|
||||
|
||||
Block size alignment with mamba page sizes is handled later by
|
||||
Platform.update_block_size_for_backend(), which runs after model
|
||||
@@ -454,6 +453,7 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig):
|
||||
|
||||
Args:
|
||||
vllm_config: vLLM Config
|
||||
|
||||
"""
|
||||
# Enable FULL_AND_PIECEWISE by default
|
||||
MambaModelConfig.verify_and_update_config(vllm_config)
|
||||
@@ -609,12 +609,12 @@ class LlamaNemotronVLConfig(VerifyAndUpdateConfig):
|
||||
class MambaModelConfig(VerifyAndUpdateConfig):
|
||||
@classmethod
|
||||
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
|
||||
"""
|
||||
Enable FULL_AND_PIECEWISE cuda graph mode by default (required
|
||||
"""Enable FULL_AND_PIECEWISE cuda graph mode by default (required
|
||||
to get good performance for mamba layers in V1).
|
||||
|
||||
Args:
|
||||
vllm_config: vLLM Config
|
||||
|
||||
"""
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Conformer encoder components for FireRedASR2.
|
||||
"""Conformer encoder components for FireRedASR2.
|
||||
|
||||
The audio encoder architecture is
|
||||
(Conv2dSubsampling → RelPositionalEncoding → N × RelPosEmbConformerBlock).
|
||||
@@ -298,9 +297,7 @@ class RelPosEmbConformerBlock(nn.Module):
|
||||
|
||||
|
||||
class ConformerEncoder(nn.Module):
|
||||
"""
|
||||
Conformer encoder used by FireRedASR2.
|
||||
"""
|
||||
"""Conformer encoder used by FireRedASR2."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -151,8 +151,7 @@ def patch_merging_by_param(
|
||||
|
||||
|
||||
class Cosmos3EdgePatchMerger(nn.Module):
|
||||
"""
|
||||
Projector: LayerNorm -> Linear -> GELU -> Linear
|
||||
"""Projector: LayerNorm -> Linear -> GELU -> Linear.
|
||||
|
||||
Reads config from projector_config (not vision_config).
|
||||
input_hidden_size * spatial_merge_size² -> merger_intermediate_size
|
||||
@@ -490,8 +489,7 @@ class Cosmos3EdgeForConditionalGeneration(
|
||||
SupportsPP,
|
||||
SupportsMRoPE,
|
||||
):
|
||||
"""
|
||||
Cosmos3 Edge model with a SigLIP2 vision encoder.
|
||||
"""Cosmos3 Edge model with a SigLIP2 vision encoder.
|
||||
|
||||
Architecture:
|
||||
- self.visual: SigLIP2 encoder + patch merger + projector
|
||||
|
||||
@@ -248,23 +248,23 @@ class ImageEncoderViT(nn.Module):
|
||||
global_attn_indexes: tuple[int, ...] = (),
|
||||
last_conv_output: int = 1024,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
img_size (int): Input image size.
|
||||
patch_size (int): Patch size.
|
||||
in_chans (int): Number of input image channels.
|
||||
embed_dim (int): Patch embedding dimension.
|
||||
depth (int): Depth of ViT.
|
||||
num_heads (int): Number of attention heads in each ViT block.
|
||||
mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
|
||||
qkv_bias (bool): If True, add a learnable bias to query, key, value.
|
||||
norm_layer (nn.Module): Normalization layer.
|
||||
act_layer (nn.Module): Activation layer.
|
||||
use_abs_pos (bool): If True, use absolute positional embeddings.
|
||||
use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
|
||||
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
|
||||
window_size (int): Window size for window attention blocks.
|
||||
global_attn_indexes (list): Indexes for blocks using global attention.
|
||||
"""Args:
|
||||
img_size (int): Input image size.
|
||||
patch_size (int): Patch size.
|
||||
in_chans (int): Number of input image channels.
|
||||
embed_dim (int): Patch embedding dimension.
|
||||
depth (int): Depth of ViT.
|
||||
num_heads (int): Number of attention heads in each ViT block.
|
||||
mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
|
||||
qkv_bias (bool): If True, add a learnable bias to query, key, value.
|
||||
norm_layer (nn.Module): Normalization layer.
|
||||
act_layer (nn.Module): Activation layer.
|
||||
use_abs_pos (bool): If True, use absolute positional embeddings.
|
||||
use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
|
||||
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
|
||||
window_size (int): Window size for window attention blocks.
|
||||
global_attn_indexes (list): Indexes for blocks using global attention.
|
||||
|
||||
""" # noqa: E501
|
||||
super().__init__()
|
||||
self.img_size = img_size
|
||||
@@ -363,7 +363,8 @@ class ImageEncoderViT(nn.Module):
|
||||
|
||||
class Block(nn.Module):
|
||||
"""Transformer blocks with support of window attention and residual propagation
|
||||
blocks"""
|
||||
blocks
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -378,20 +379,20 @@ class Block(nn.Module):
|
||||
window_size: int = 0,
|
||||
input_size: tuple[int, int] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
dim (int): Number of input channels.
|
||||
num_heads (int): Number of attention heads in each ViT block.
|
||||
mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
|
||||
qkv_bias (bool): If True, add a learnable bias to query, key, value.
|
||||
norm_layer (nn.Module): Normalization layer.
|
||||
act_layer (nn.Module): Activation layer.
|
||||
use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
|
||||
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
|
||||
window_size (int): Window size for window attention blocks. If it equals 0, then
|
||||
use global attention.
|
||||
input_size (tuple(int, int) or None): Input resolution for calculating the relative
|
||||
positional parameter size.
|
||||
"""Args:
|
||||
dim (int): Number of input channels.
|
||||
num_heads (int): Number of attention heads in each ViT block.
|
||||
mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
|
||||
qkv_bias (bool): If True, add a learnable bias to query, key, value.
|
||||
norm_layer (nn.Module): Normalization layer.
|
||||
act_layer (nn.Module): Activation layer.
|
||||
use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
|
||||
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
|
||||
window_size (int): Window size for window attention blocks. If it equals 0, then
|
||||
use global attention.
|
||||
input_size (tuple(int, int) or None): Input resolution for calculating the relative
|
||||
positional parameter size.
|
||||
|
||||
""" # noqa: E501
|
||||
super().__init__()
|
||||
self.norm1 = norm_layer(dim)
|
||||
@@ -448,16 +449,16 @@ class RelPosAttention(PluggableLayer):
|
||||
rel_pos_zero_init: bool = True,
|
||||
input_size: tuple[int, int] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
dim (int): Number of input channels.
|
||||
num_heads (int): Number of attention heads.
|
||||
qkv_bias (bool): If True, add a learnable bias to query, key, value.
|
||||
use_triton_attention (bool): If True, fuse relative position bias in
|
||||
a Triton attention kernel on CUDA-alike platforms.
|
||||
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
|
||||
input_size (tuple(int, int) or None): Input resolution for calculating the relative
|
||||
positional parameter size.
|
||||
"""Args:
|
||||
dim (int): Number of input channels.
|
||||
num_heads (int): Number of attention heads.
|
||||
qkv_bias (bool): If True, add a learnable bias to query, key, value.
|
||||
use_triton_attention (bool): If True, fuse relative position bias in
|
||||
a Triton attention kernel on CUDA-alike platforms.
|
||||
rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
|
||||
input_size (tuple(int, int) or None): Input resolution for calculating the relative
|
||||
positional parameter size.
|
||||
|
||||
""" # noqa: E501
|
||||
super().__init__()
|
||||
self.num_heads = num_heads
|
||||
@@ -540,8 +541,8 @@ class RelPosAttention(PluggableLayer):
|
||||
def window_partition(
|
||||
x: torch.Tensor, window_size: int
|
||||
) -> tuple[torch.Tensor, tuple[int, int]]:
|
||||
"""
|
||||
Partition into non-overlapping windows with padding if needed.
|
||||
"""Partition into non-overlapping windows with padding if needed.
|
||||
|
||||
Args:
|
||||
x (tensor): input tokens with [B, H, W, C].
|
||||
window_size (int): window size.
|
||||
@@ -549,6 +550,7 @@ def window_partition(
|
||||
Returns:
|
||||
windows: windows after partition with [B * num_windows, window_size, window_size, C].
|
||||
(Hp, Wp): padded height and width before partition
|
||||
|
||||
""" # noqa: E501
|
||||
B, H, W, C = x.shape
|
||||
|
||||
@@ -571,8 +573,8 @@ def window_unpartition(
|
||||
pad_hw: tuple[int, int],
|
||||
hw: tuple[int, int],
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Window unpartition into original sequences and removing padding.
|
||||
"""Window unpartition into original sequences and removing padding.
|
||||
|
||||
Args:
|
||||
windows (tensor): input tokens with [B * num_windows, window_size, window_size, C].
|
||||
window_size (int): window size.
|
||||
@@ -581,6 +583,7 @@ def window_unpartition(
|
||||
|
||||
Returns:
|
||||
x: unpartitioned sequences with [B, H, W, C].
|
||||
|
||||
""" # noqa: E501
|
||||
Hp, Wp = pad_hw
|
||||
H, W = hw
|
||||
@@ -596,9 +599,9 @@ def window_unpartition(
|
||||
|
||||
|
||||
def get_rel_pos(q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Get relative positional embeddings according to the relative positions of
|
||||
"""Get relative positional embeddings according to the relative positions of
|
||||
query and key sizes.
|
||||
|
||||
Args:
|
||||
q_size (int): size of query q.
|
||||
k_size (int): size of key k.
|
||||
@@ -606,6 +609,7 @@ def get_rel_pos(q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor
|
||||
|
||||
Returns:
|
||||
Extracted positional embeddings according to relative positions.
|
||||
|
||||
"""
|
||||
max_rel_dist = int(2 * max(q_size, k_size) - 1)
|
||||
# Interpolate rel pos if needed.
|
||||
@@ -641,8 +645,7 @@ def add_decomposed_rel_pos(
|
||||
q_size: tuple[int, int],
|
||||
k_size: tuple[int, int],
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`.
|
||||
"""Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`.
|
||||
https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py
|
||||
Args:
|
||||
q (Tensor): query q in the attention layer with shape (B, q_h * q_w, C).
|
||||
@@ -653,6 +656,7 @@ def add_decomposed_rel_pos(
|
||||
|
||||
Returns:
|
||||
attn (Tensor): attention map with added relative positional embeddings.
|
||||
|
||||
""" # noqa: E501
|
||||
q_h, q_w = q_size
|
||||
k_h, k_w = k_size
|
||||
@@ -672,9 +676,7 @@ def add_decomposed_rel_pos(
|
||||
|
||||
|
||||
class PatchEmbed(nn.Module):
|
||||
"""
|
||||
Image to Patch Embedding.
|
||||
"""
|
||||
"""Image to Patch Embedding."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -684,13 +686,13 @@ class PatchEmbed(nn.Module):
|
||||
in_chans: int = 3,
|
||||
embed_dim: int = 768,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
kernel_size (Tuple): kernel size of the projection layer.
|
||||
stride (Tuple): stride of the projection layer.
|
||||
padding (Tuple): padding size of the projection layer.
|
||||
in_chans (int): Number of input image channels.
|
||||
embed_dim (int): Patch embedding dimension.
|
||||
"""Args:
|
||||
kernel_size (Tuple): kernel size of the projection layer.
|
||||
stride (Tuple): stride of the projection layer.
|
||||
padding (Tuple): padding size of the projection layer.
|
||||
in_chans (int): Number of input image channels.
|
||||
embed_dim (int): Patch embedding dimension.
|
||||
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
|
||||
@@ -23,8 +23,7 @@ from vllm.model_executor.custom_op import PluggableLayer
|
||||
# --8<-- [start:qwen2_decoder]
|
||||
@PluggableLayer.register("qwen2_decoder")
|
||||
class CustomQwen2Decoder(PluggableLayer):
|
||||
"""
|
||||
Qwen2 visual encoder
|
||||
"""Qwen2 visual encoder
|
||||
non-causal attention + causal attention
|
||||
token_type_ids :0=non-causal, 1=causal
|
||||
"""
|
||||
@@ -75,7 +74,7 @@ class CustomQwen2Decoder(PluggableLayer):
|
||||
del self.model.embed_tokens
|
||||
|
||||
def _create_custom_model(self, config):
|
||||
"""Qwen2Model"""
|
||||
"""Qwen2Model."""
|
||||
|
||||
class CustomQwen2ModelInner(Qwen2Model):
|
||||
def forward(
|
||||
@@ -194,11 +193,11 @@ class CustomQwen2Decoder(PluggableLayer):
|
||||
attention_mask: torch.Tensor = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
inputs_embeds: [batch_size, seq_len, hidden_dim]
|
||||
token_type_ids: [batch_size, seq_len], 0=non-causal, 1=causal
|
||||
attention_mask: [batch_size, seq_len], optional
|
||||
"""Args:
|
||||
inputs_embeds: [batch_size, seq_len, hidden_dim]
|
||||
token_type_ids: [batch_size, seq_len], 0=non-causal, 1=causal
|
||||
attention_mask: [batch_size, seq_len], optional
|
||||
|
||||
"""
|
||||
return self.model(
|
||||
inputs_embeds=inputs_embeds,
|
||||
@@ -209,8 +208,7 @@ class CustomQwen2Decoder(PluggableLayer):
|
||||
|
||||
|
||||
class Qwen2Decoder2Encoder(nn.Module):
|
||||
"""
|
||||
Decoder based on Multilingual BART
|
||||
"""Decoder based on Multilingual BART
|
||||
Set the initial weights and configuration with a pretrained multilingual BART model,
|
||||
and modify the detailed configurations as a Nougat decoder
|
||||
"""
|
||||
|
||||
@@ -36,8 +36,7 @@ from .utils import (
|
||||
|
||||
|
||||
class DeepseekV2Eagle3DecoderLayer(nn.Module):
|
||||
"""
|
||||
Eagle3 decoder layer for Deepseek that:
|
||||
"""Eagle3 decoder layer for Deepseek that:
|
||||
1. Always uses MLP (not MoE)
|
||||
2. First layer accepts concatenated embeds + hidden_states
|
||||
"""
|
||||
|
||||
@@ -529,8 +529,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts, SupportsPP):
|
||||
return loaded_params
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""
|
||||
Rewrite the weight name to match the format of the original model.
|
||||
"""Rewrite the weight name to match the format of the original model.
|
||||
Add .mtp_block for modules in transformer layer block for spec layer
|
||||
and rename shared layer weights to be top level.
|
||||
"""
|
||||
|
||||
@@ -79,13 +79,12 @@ _IMAGE_TOKEN = "<image>"
|
||||
|
||||
|
||||
class DeepseekOCRImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- b: Batch size
|
||||
- n: Number of images
|
||||
- p: Number of patches
|
||||
- base_size: Base size of the processor
|
||||
- image_size: Image size of the processor
|
||||
"""Dimensions:
|
||||
- b: Batch size
|
||||
- n: Number of images
|
||||
- p: Number of patches
|
||||
- base_size: Base size of the processor
|
||||
- image_size: Image size of the processor
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
@@ -140,7 +139,8 @@ class NoRepeatNGramLogitsProcessor:
|
||||
|
||||
class NGramPerReqLogitsProcessor(AdapterLogitsProcessor):
|
||||
"""Example of overriding the wrapper class `__init__()` in order to utilize
|
||||
info about the device type"""
|
||||
info about the device type
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def validate_params(cls, params: SamplingParams):
|
||||
@@ -639,9 +639,7 @@ class DeepseekOCRForCausalLM(
|
||||
return autoloaded_weights
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models."""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="projector",
|
||||
@@ -676,8 +674,7 @@ class DeepseekOCRForCausalLM(
|
||||
self,
|
||||
image_spatial_crop: torch.Tensor | None = None,
|
||||
) -> tuple[int, int, int, int]:
|
||||
"""
|
||||
Return (num_input_tokens, num_output_tokens, global_output_token,
|
||||
"""Return (num_input_tokens, num_output_tokens, global_output_token,
|
||||
local_output_token) for a single image described by
|
||||
``image_spatial_crop``.
|
||||
"""
|
||||
@@ -854,8 +851,7 @@ class DeepseekOCRForCausalLM(
|
||||
self,
|
||||
pixel_values: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Encode batched global images with newline tokens inserted.
|
||||
"""Encode batched global images with newline tokens inserted.
|
||||
Output shape: ``[B * 272, n_embed]``.
|
||||
"""
|
||||
bsz = pixel_values.shape[0]
|
||||
@@ -880,8 +876,7 @@ class DeepseekOCRForCausalLM(
|
||||
self,
|
||||
images_crop: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Encode local patches without newline insertion (newlines are added later
|
||||
"""Encode local patches without newline insertion (newlines are added later
|
||||
in ``postprocess_encoder_output`` via ``_assemble_patch_grid``).
|
||||
Output shape: ``[P * 100, n_embed]``.
|
||||
"""
|
||||
@@ -951,8 +946,7 @@ class DeepseekOCRForCausalLM(
|
||||
clone: bool = False,
|
||||
batch_mm_kwargs: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Assemble per-image embeddings from global and local encoder outputs.
|
||||
"""Assemble per-image embeddings from global and local encoder outputs.
|
||||
|
||||
``output['global']`` contains global-image features with newlines already
|
||||
inserted (from CUDA graph replay or eager fallback):
|
||||
|
||||
@@ -456,9 +456,7 @@ class DeepseekOCR2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Support
|
||||
return autoloaded_weights
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models."""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="projector",
|
||||
|
||||
@@ -850,8 +850,7 @@ class Indexer(nn.Module):
|
||||
def _try_load_fp8_indexer_wk(
|
||||
name, tensor, buf, params_dict, loaded_params, pp_missing_layer_names
|
||||
):
|
||||
"""
|
||||
We fuse the WK and weights_proj projections, but in some checkpoints WK is stored
|
||||
"""We fuse the WK and weights_proj projections, but in some checkpoints WK is stored
|
||||
in FP8 with a separate weight_scale_inv, while weights_proj is stored in BF16.
|
||||
Upcasting to BF16 during loading enables the fusion. This function loads the FP8 WK
|
||||
weights and scale, and when both are available, dequantizes to BF16 and stores into
|
||||
@@ -911,8 +910,7 @@ def _min_latency_fused_qkv_a_proj_impl(
|
||||
input_: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Dynamically run min-latency gemm if num_tokens <= 16.
|
||||
"""Dynamically run min-latency gemm if num_tokens <= 16.
|
||||
This must be wrapped in a custom op because our torch.compile integration
|
||||
does not support runtime dispatching on num_tokens.
|
||||
"""
|
||||
@@ -993,8 +991,7 @@ class DeepSeekV2FusedQkvAProjLinear(MergedColumnParallelLinear):
|
||||
|
||||
|
||||
class DeepseekV2MLAAttention(nn.Module):
|
||||
"""
|
||||
Main reference: DeepseekV2 paper, and FlashInfer Implementation
|
||||
"""Main reference: DeepseekV2 paper, and FlashInfer Implementation
|
||||
(https://arxiv.org/abs/2405.04434 and https://github.com/flashinfer-ai/flashinfer/pull/551).
|
||||
|
||||
For more info see MLACommonImpl in:
|
||||
|
||||
@@ -64,13 +64,12 @@ _IMAGE_TOKEN = "<image>"
|
||||
|
||||
|
||||
class DeepseekVL2ImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bnp: Batch size * number of images * number of patches
|
||||
- p: Number of patches
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each image
|
||||
- w: Width of each image
|
||||
"""Dimensions:
|
||||
- bnp: Batch size * number of images * number of patches
|
||||
- p: Number of patches
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each image
|
||||
- w: Width of each image
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
@@ -79,11 +78,10 @@ class DeepseekVL2ImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class DeepseekVL2VImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- f: Image feature size
|
||||
- h: Hidden size (must match language model backbone)
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- f: Image feature size
|
||||
- h: Hidden size (must match language model backbone)
|
||||
"""
|
||||
|
||||
type: Literal["image_embeds"]
|
||||
|
||||
@@ -352,7 +352,6 @@ class DiffusionGemmaForConditionalGeneration(
|
||||
skip duplicate decoder backbone weights, handle vision tower and
|
||||
self-conditioning separately.
|
||||
"""
|
||||
|
||||
sc_params = dict(
|
||||
(n, p)
|
||||
for n, p in self.named_parameters()
|
||||
@@ -512,7 +511,8 @@ def _compiled_sample_step(
|
||||
accept/renoise → convergence, all as vectorized PyTorch ops.
|
||||
|
||||
Returns the temperature-scaled logits ``[num_decode, CL, vocab]`` so the
|
||||
caller can compute logprobs outside the compiled region."""
|
||||
caller can compute logprobs outside the compiled region.
|
||||
"""
|
||||
num_decode = decode_slots.shape[0]
|
||||
device = decode_slots.device
|
||||
|
||||
|
||||
@@ -66,12 +66,11 @@ IMAGE_TOKEN = "<|imgpad|>"
|
||||
|
||||
|
||||
class DotsOCRImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- np: The total number of patches over each image over each prompt in
|
||||
the batch
|
||||
- ni: Number of images
|
||||
- cps: Number of channels * patch_size * patch_size
|
||||
"""Dimensions:
|
||||
- np: The total number of patches over each image over each prompt in
|
||||
the batch
|
||||
- ni: Number of images
|
||||
- cps: Number of channels * patch_size * patch_size
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
@@ -81,11 +80,10 @@ class DotsOCRImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class DotsOCRImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- nf: Number of image features
|
||||
- hs: Hidden size
|
||||
- ni: Number of images
|
||||
"""Dimensions:
|
||||
- nf: Number of image features
|
||||
- hs: Hidden size
|
||||
- ni: Number of images
|
||||
"""
|
||||
|
||||
type: Literal["image_embeds"]
|
||||
@@ -768,9 +766,7 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models."""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="vision_tower.merger",
|
||||
|
||||
@@ -37,13 +37,12 @@ from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
|
||||
|
||||
|
||||
class Eagle2_5_VLImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- bnp: Batch size * number of images * (1 + num_patches)
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each image patch
|
||||
- w: Width of each image patch
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- bnp: Batch size * number of images * (1 + num_patches)
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each image patch
|
||||
- w: Width of each image patch
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
@@ -52,11 +51,10 @@ class Eagle2_5_VLImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class Eagle2_5_VLImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- n: Number of images
|
||||
- f: Total image feature size
|
||||
- h: Hidden size (must match the hidden size of language model backbone)
|
||||
"""Dimensions:
|
||||
- n: Number of images
|
||||
- f: Total image feature size
|
||||
- h: Hidden size (must match the hidden size of language model backbone)
|
||||
"""
|
||||
|
||||
type: Literal["image_embeds"]
|
||||
@@ -127,8 +125,7 @@ class Eagle2_5_VLMultiModalProcessor(
|
||||
class Eagle2_5_VLForConditionalGeneration(
|
||||
nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
|
||||
):
|
||||
"""
|
||||
Eagle2.5-VL model for conditional generation.
|
||||
"""Eagle2.5-VL model for conditional generation.
|
||||
|
||||
Architecture:
|
||||
- Vision Encoder: SigLIP
|
||||
@@ -238,8 +235,7 @@ class Eagle2_5_VLForConditionalGeneration(
|
||||
)
|
||||
|
||||
def pixel_shuffle(self, x: torch.Tensor, scale_factor: float = 0.5) -> torch.Tensor:
|
||||
"""
|
||||
Pixel shuffle operation for downsampling vision features.
|
||||
"""Pixel shuffle operation for downsampling vision features.
|
||||
|
||||
Args:
|
||||
x: Input tensor of shape (n, w, h, c)
|
||||
@@ -247,6 +243,7 @@ class Eagle2_5_VLForConditionalGeneration(
|
||||
|
||||
Returns:
|
||||
Downsampled tensor
|
||||
|
||||
"""
|
||||
n, w, h, c = x.size()
|
||||
# N, W, H, C --> N, W, H * scale, C // scale
|
||||
@@ -264,14 +261,14 @@ class Eagle2_5_VLForConditionalGeneration(
|
||||
return x
|
||||
|
||||
def extract_feature(self, pixel_values: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Extract visual features from pixel values.
|
||||
"""Extract visual features from pixel values.
|
||||
|
||||
Args:
|
||||
pixel_values: Input pixel values of shape (batch, channels, height, width)
|
||||
|
||||
Returns:
|
||||
Visual embeddings
|
||||
|
||||
"""
|
||||
vit_embeds = self.vision_model(pixel_values=pixel_values)
|
||||
|
||||
|
||||
@@ -119,7 +119,7 @@ def all_gather_interleave(local_tensor, hidden_size: int, tp_size: int):
|
||||
|
||||
|
||||
class Ernie4_5_VisionAttention(nn.Module):
|
||||
"""VisionAttention using VLLM framework APIs"""
|
||||
"""VisionAttention using VLLM framework APIs."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -497,6 +497,7 @@ class Ernie4_5_VisionTransformer(nn.Module):
|
||||
|
||||
Returns:
|
||||
Dict with ``rotary_pos_emb``, ``cu_seqlens`` and ``max_seqlen``.
|
||||
|
||||
"""
|
||||
if device is None:
|
||||
device = self.device
|
||||
@@ -600,12 +601,11 @@ class Ernie4_5_VisionTransformer(nn.Module):
|
||||
|
||||
|
||||
class Ernie4_5_VLImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- np: The total number of patches over each image over each prompt in
|
||||
the batch
|
||||
- ni: Number of images
|
||||
- cps: Number of channels * patch_size * patch_size
|
||||
"""Dimensions:
|
||||
- np: The total number of patches over each image over each prompt in
|
||||
the batch
|
||||
- ni: Number of images
|
||||
- cps: Number of channels * patch_size * patch_size
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
@@ -618,13 +618,12 @@ Ernie4_5_VLImageInputs = Ernie4_5_VLImagePixelInputs
|
||||
|
||||
|
||||
class Ernie4_5_VLVideoPixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- np: The total number of patches over each image over each prompt in
|
||||
the batch
|
||||
- ni: Number of images
|
||||
- cps: Number of channels * temporal_patch_size * patch_size *
|
||||
patch_size
|
||||
"""Dimensions:
|
||||
- np: The total number of patches over each image over each prompt in
|
||||
the batch
|
||||
- ni: Number of images
|
||||
- cps: Number of channels * temporal_patch_size * patch_size *
|
||||
patch_size
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values_videos"]
|
||||
|
||||
@@ -45,8 +45,7 @@ def unified_kv_cache_update(
|
||||
to_cache: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Returns a dummy that is passed to unified_attention to signal a side effect and
|
||||
"""Returns a dummy that is passed to unified_attention to signal a side effect and
|
||||
the data dependency between them to ensure torch.compile preserves ordering.
|
||||
"""
|
||||
forward_context = get_forward_context()
|
||||
@@ -294,6 +293,7 @@ class CacheOnlyAttentionLayer(nn.Module, AttentionLayerBase):
|
||||
|
||||
Returns:
|
||||
Dummy output tensor (not used)
|
||||
|
||||
"""
|
||||
# Note: we set num_heads to num_hidden_layers and
|
||||
# head_size to hidden_size for hidden states storage
|
||||
@@ -373,8 +373,8 @@ class ExtractHiddenStatesModel(nn.Module):
|
||||
|
||||
Returns:
|
||||
Tuple of (dummy_output, dummy_output) - both unused
|
||||
"""
|
||||
|
||||
"""
|
||||
# Call dummy attention layer to cache hidden states
|
||||
# Output is ignored - we only care about the KV cache side effects
|
||||
_ = self.cache_only_layers[str(self.target_num_hidden_layers)](hidden_states)
|
||||
|
||||
@@ -140,8 +140,7 @@ class FalconH1SSMDecoderLayer(nn.Module):
|
||||
self._init_mup_vector()
|
||||
|
||||
def _init_mup_vector(self):
|
||||
"""
|
||||
Non learnable per-block scaling vector composed of element-wise
|
||||
"""Non learnable per-block scaling vector composed of element-wise
|
||||
multipliersapplied to each separate contiguous block of the output
|
||||
of the linear projection (in_proj) before further processing
|
||||
(gating, convolution, SSM):
|
||||
@@ -313,8 +312,7 @@ class FalconH1AttentionDecoderLayer(nn.Module):
|
||||
|
||||
|
||||
class FalconH1ParallelHybrid(nn.Module):
|
||||
"""
|
||||
A hybrid decoder layer for FalconH1 where the input is processed
|
||||
"""A hybrid decoder layer for FalconH1 where the input is processed
|
||||
in parallel through both the self-attention branch and the SSM (Mamba)
|
||||
branch. Their outputs are then summed to produce the final hidden state.
|
||||
|
||||
@@ -552,6 +550,7 @@ class FalconH1ForCausalLM(
|
||||
Tuple containing:
|
||||
- conv_state_shape: Shape for convolutional state cache
|
||||
- temporal_state_shape: Shape for state space model cache
|
||||
|
||||
"""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
|
||||
@@ -63,11 +63,10 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class FireRedASR2AudioInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- b: Batch size
|
||||
- nmb: Number of mel bins
|
||||
- t: Time frames (M)
|
||||
"""Dimensions:
|
||||
- b: Batch size
|
||||
- nmb: Number of mel bins
|
||||
- t: Time frames (M)
|
||||
"""
|
||||
|
||||
input_features: Annotated[
|
||||
|
||||
@@ -598,11 +598,10 @@ class Transformer(nn.Module):
|
||||
|
||||
|
||||
class FunASRAudioInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- b: Batch size
|
||||
- nmb: Number of mel bins
|
||||
- t: Time frames (M)
|
||||
"""Dimensions:
|
||||
- b: Batch size
|
||||
- nmb: Number of mel bins
|
||||
- t: Time frames (M)
|
||||
"""
|
||||
|
||||
input_features: Annotated[
|
||||
|
||||
@@ -58,14 +58,13 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Gemma3ImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- p: Number of patches total (over each image over each prompt in the
|
||||
batch)
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each patch
|
||||
- w: Width of each patch
|
||||
- bn: Batch size * number of images
|
||||
"""Dimensions:
|
||||
- p: Number of patches total (over each image over each prompt in the
|
||||
batch)
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each patch
|
||||
- w: Width of each patch
|
||||
- bn: Batch size * number of images
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"] = "pixel_values"
|
||||
@@ -678,9 +677,7 @@ class Gemma3ForConditionalGeneration(
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models"""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="multi_modal_projector",
|
||||
|
||||
@@ -185,7 +185,7 @@ class Gemma3nAltUp(nn.Module):
|
||||
|
||||
|
||||
class Gemma3nLaurelBlock(nn.Module):
|
||||
"""Learned Augmented Residual Layer"""
|
||||
"""Learned Augmented Residual Layer."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -593,9 +593,7 @@ class Gemma3nDecoderLayer(nn.Module):
|
||||
enable_if=lambda vllm_config: vllm_config.cache_config.kv_sharing_fast_prefill
|
||||
)
|
||||
class Gemma3nSelfDecoder(nn.Module):
|
||||
"""
|
||||
Includes altup embedding and self decoder layers
|
||||
"""
|
||||
"""Includes altup embedding and self decoder layers."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -776,9 +774,7 @@ class Gemma3nSelfDecoder(nn.Module):
|
||||
enable_if=lambda vllm_config: vllm_config.cache_config.kv_sharing_fast_prefill
|
||||
)
|
||||
class Gemma3nCrossDecoder(nn.Module):
|
||||
"""
|
||||
Cross-decoder layers
|
||||
"""
|
||||
"""Cross-decoder layers."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -818,7 +814,8 @@ class Gemma3nCrossDecoder(nn.Module):
|
||||
def _kv_sharing_weights_mapper(config: Gemma3nTextConfig) -> WeightsMapper:
|
||||
"""KV-shared layers only have q_proj, so qkv_proj packing applies to the
|
||||
other layers. Original checkpoints still ship K/V tensors for the shared
|
||||
layers (fine-tuned ones omit them); those are dropped."""
|
||||
layers (fine-tuned ones omit them); those are dropped.
|
||||
"""
|
||||
first_kv_shared_layer_idx = config.num_hidden_layers - config.num_kv_shared_layers
|
||||
return WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Lightweight utility functions for Gemma3n audio processing.
|
||||
"""Lightweight utility functions for Gemma3n audio processing.
|
||||
|
||||
This module is separate from gemma3n_mm.py to avoid heavy CUDA dependencies,
|
||||
making it testable without a full vLLM build.
|
||||
@@ -37,6 +36,7 @@ def adjust_audio_features_to_expected_length(
|
||||
Tuple of:
|
||||
- adjusted_features: Audio features adjusted to expected_tokens length
|
||||
- tokens_truncated: Number of tokens truncated (0 if padding was applied)
|
||||
|
||||
"""
|
||||
audio_batch_size, audio_seq_len, audio_embed_dim = audio_features.shape
|
||||
tokens_truncated = 0
|
||||
|
||||
@@ -74,12 +74,11 @@ TOKENS_PER_AUDIO = 188
|
||||
|
||||
|
||||
class Gemma3nImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each patch
|
||||
- w: Width of each patch
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each patch
|
||||
- w: Width of each patch
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"] = "pixel_values"
|
||||
@@ -113,6 +112,7 @@ def batch_audio_features(
|
||||
|
||||
Returns:
|
||||
The `(bn, s_max, f)` features and their `(bn, s_max)` mask.
|
||||
|
||||
"""
|
||||
if isinstance(input_features, torch.Tensor):
|
||||
assert isinstance(input_features_mask, torch.Tensor)
|
||||
@@ -135,11 +135,10 @@ def batch_audio_features(
|
||||
|
||||
|
||||
class Gemma3nAudioInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of audios
|
||||
- s: seq_length
|
||||
- f: num_features
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of audios
|
||||
- s: seq_length
|
||||
- f: num_features
|
||||
"""
|
||||
|
||||
type: Literal["audio"] = "audio"
|
||||
@@ -187,8 +186,7 @@ class Gemma3nProcessingInfo(BaseProcessingInfo):
|
||||
image_height: int,
|
||||
processor: Gemma3nProcessor,
|
||||
) -> PromptUpdateDetails:
|
||||
"""
|
||||
Get the replacement metadata for image tokens.
|
||||
"""Get the replacement metadata for image tokens.
|
||||
|
||||
For Gemma3n, this should return the full_image_sequence which includes
|
||||
BOI token, repeated image tokens, and EOI token.
|
||||
@@ -205,8 +203,7 @@ class Gemma3nProcessingInfo(BaseProcessingInfo):
|
||||
*,
|
||||
processor: Gemma3nProcessor,
|
||||
) -> PromptUpdateDetails:
|
||||
"""
|
||||
Get the replacement metadata for audio tokens.
|
||||
"""Get the replacement metadata for audio tokens.
|
||||
|
||||
For Gemma3n, this should return the full_audio_sequence which includes
|
||||
BOA token, repeated audio tokens, and EOA token.
|
||||
@@ -316,8 +313,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
|
||||
return processed_data
|
||||
|
||||
def _get_num_audio_frames_alone(self, num_samples: int) -> int:
|
||||
"""
|
||||
Get the number of mel frames that the feature extractor produces
|
||||
"""Get the number of mel frames that the feature extractor produces
|
||||
for a single audio clip of the given length, independent of batch
|
||||
padding.
|
||||
"""
|
||||
@@ -514,7 +510,8 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
|
||||
|
||||
class Gemma3nMultimodalEmbedder(nn.Module):
|
||||
"""Embeds token ids or soft tokens for multimodal content into language
|
||||
model space."""
|
||||
model space.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -571,6 +568,7 @@ class Gemma3nMultimodalEmbedder(nn.Module):
|
||||
|
||||
Returns:
|
||||
A torch.Tensor of embeddings with shape `[batch_size, seq_len, self.config.text_config.hidden_size]`.
|
||||
|
||||
""" # noqa: E501
|
||||
if (input_ids is None) ^ (inputs_embeds is not None):
|
||||
raise ValueError(
|
||||
@@ -891,9 +889,7 @@ class Gemma3nForConditionalGeneration(
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models"""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="multi_modal_projector",
|
||||
@@ -911,8 +907,7 @@ class Gemma3nForConditionalGeneration(
|
||||
|
||||
@classmethod
|
||||
def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
|
||||
"""
|
||||
Gemma3n supports "free-form" transcription.
|
||||
"""Gemma3n supports "free-form" transcription.
|
||||
We fix its prompt here to standardize transcriptions/translations
|
||||
requests.
|
||||
"""
|
||||
|
||||
@@ -866,6 +866,7 @@ class Gemma4SelfDecoderLayers(nn.Module):
|
||||
Returns:
|
||||
Per-layer embeddings (num_tokens, num_layers,
|
||||
hidden_size_per_layer_input)
|
||||
|
||||
"""
|
||||
if self.embed_tokens_per_layer is None:
|
||||
return None
|
||||
@@ -1199,6 +1200,7 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
Returns:
|
||||
Per-layer embeddings (num_tokens, num_layers,
|
||||
hidden_size_per_layer_input)
|
||||
|
||||
"""
|
||||
return self.self_decoder.get_per_layer_inputs(input_ids)
|
||||
|
||||
|
||||
@@ -132,8 +132,7 @@ def _get_max_soft_tokens(
|
||||
|
||||
|
||||
class Gemma4ImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Pre-patchified image inputs from the Gemma4 image processor.
|
||||
"""Pre-patchified image inputs from the Gemma4 image processor.
|
||||
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
@@ -159,11 +158,10 @@ class Gemma4ImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class Gemma4AudioInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of audios
|
||||
- s: Sequence length (MEL spectrogram frames)
|
||||
- f: Number of features (MEL bins)
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of audios
|
||||
- s: Sequence length (MEL spectrogram frames)
|
||||
- f: Number of features (MEL bins)
|
||||
"""
|
||||
|
||||
type: Literal["audio"] = "audio"
|
||||
@@ -304,9 +302,12 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
for an image of the given dimensions, after padding is stripped.
|
||||
|
||||
Args:
|
||||
image_width: Width of the image in pixels.
|
||||
image_height: Height of the image in pixels.
|
||||
max_soft_tokens: Override for the vision config's
|
||||
``default_output_length``. When *None*, the value from
|
||||
the model config is used.
|
||||
|
||||
"""
|
||||
vision_cfg = self.get_hf_config().vision_config
|
||||
patch_size = vision_cfg.patch_size
|
||||
@@ -345,8 +346,12 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
||||
produce after stripping padding.
|
||||
|
||||
Args:
|
||||
image_width: Width of the image in pixels.
|
||||
image_height: Height of the image in pixels.
|
||||
processor: The Gemma4 processor, when already constructed.
|
||||
max_soft_tokens: Override for the default token budget.
|
||||
When *None*, falls back to the model config value.
|
||||
|
||||
"""
|
||||
if processor is None:
|
||||
processor = self.get_hf_processor()
|
||||
|
||||
@@ -101,6 +101,7 @@ class Gemma4MTPMaskedEmbedder(nn.Module):
|
||||
Returns:
|
||||
logits: (num_tokens, num_selected) sparse logits.
|
||||
indices: (num_tokens, num_selected) corresponding vocab indices.
|
||||
|
||||
"""
|
||||
num_tokens = hidden_states.shape[0]
|
||||
_, top_k_indices = torch.topk(
|
||||
|
||||
@@ -25,7 +25,8 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Inference-only GLM-4.1V & GLM-4.6V-Flash, AutoGLM-Phone-9B model
|
||||
compatible with HuggingFace weights."""
|
||||
compatible with HuggingFace weights.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
|
||||
@@ -145,12 +146,11 @@ def _to_video_metadata(metadata: Mapping[str, Any]) -> VideoMetadata:
|
||||
|
||||
|
||||
class Glm4vImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- np: Number of patches
|
||||
- cpp: Number of channels * patch_size * patch_size
|
||||
- ni: Number of images
|
||||
- g: Grid dimensions (3 for grid_t, grid_h, grid_w)
|
||||
"""Dimensions:
|
||||
- np: Number of patches
|
||||
- cpp: Number of channels * patch_size * patch_size
|
||||
- ni: Number of images
|
||||
- g: Grid dimensions (3 for grid_t, grid_h, grid_w)
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"] = "pixel_values"
|
||||
@@ -160,12 +160,11 @@ class Glm4vImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class Glm4vImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- f: Number of image features (varies based on image resolution)
|
||||
- h: Hidden size (must match language model backbone)
|
||||
- n: Number of images
|
||||
- g: Grid dimensions (3 for grid_t, grid_h, grid_w)
|
||||
"""Dimensions:
|
||||
- f: Number of image features (varies based on image resolution)
|
||||
- h: Hidden size (must match language model backbone)
|
||||
- n: Number of images
|
||||
- g: Grid dimensions (3 for grid_t, grid_h, grid_w)
|
||||
"""
|
||||
|
||||
type: Literal["image_embeds"] = "image_embeds"
|
||||
@@ -178,14 +177,13 @@ Glm4vImageInputs: TypeAlias = Glm4vImagePixelInputs | Glm4vImageEmbeddingInputs
|
||||
|
||||
|
||||
class Glm4vVideoPixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- np: Number of patches
|
||||
- ctpp: Number of channels * temporal_patch_size *
|
||||
patch_size * patch_size
|
||||
- f: Number of frames
|
||||
- g: Grid dimensions (3 for grid_t which is usually 1 for processed
|
||||
video, grid_h, grid_w)
|
||||
"""Dimensions:
|
||||
- np: Number of patches
|
||||
- ctpp: Number of channels * temporal_patch_size *
|
||||
patch_size * patch_size
|
||||
- f: Number of frames
|
||||
- g: Grid dimensions (3 for grid_t which is usually 1 for processed
|
||||
video, grid_h, grid_w)
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values_videos"] = "pixel_values_videos"
|
||||
@@ -195,13 +193,12 @@ class Glm4vVideoPixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class Glm4vVideoEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- p: Number of video patches across all frames
|
||||
- h: Hidden size (must match language model backbone)
|
||||
- f: Number of frames
|
||||
- g: Grid dimensions (3 for grid_t which is usually 1 for processed
|
||||
video, grid_h, grid_w)
|
||||
"""Dimensions:
|
||||
- p: Number of video patches across all frames
|
||||
- h: Hidden size (must match language model backbone)
|
||||
- f: Number of frames
|
||||
- g: Grid dimensions (3 for grid_t which is usually 1 for processed
|
||||
video, grid_h, grid_w)
|
||||
"""
|
||||
|
||||
type: Literal["video_embeds"] = "video_embeds"
|
||||
@@ -771,7 +768,6 @@ class Glm4vVisionTransformer(nn.Module):
|
||||
forward graph compatible with CUDA graph replay. Coordinate generation matches
|
||||
`rot_pos_emb` exactly to guarantee spatial alignment.
|
||||
"""
|
||||
|
||||
device = self.embeddings.position_embedding.weight.device
|
||||
dtype = self.dtype
|
||||
all_embeds = []
|
||||
@@ -850,6 +846,7 @@ class Glm4vVisionTransformer(nn.Module):
|
||||
instead of computing from cu_seqlens (needed for CUDA
|
||||
graph capture to cover worst-case replay scenarios).
|
||||
device: Device to place tensors on. Defaults to self.device.
|
||||
|
||||
"""
|
||||
if device is None:
|
||||
device = self.device
|
||||
@@ -2347,6 +2344,7 @@ class Glm4vForConditionalGeneration(
|
||||
parallelism.
|
||||
inputs_embeds: Optional pre-computed input embeddings.
|
||||
**kwargs: Additional keyword arguments.
|
||||
|
||||
"""
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
@@ -2370,9 +2368,7 @@ class Glm4vForConditionalGeneration(
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models."""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model.model",
|
||||
connector="visual.merger.",
|
||||
|
||||
@@ -22,7 +22,8 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Inference-only GLM-4.5, GLM-4.6, GLM-4.7 model
|
||||
compatible with HuggingFace weights."""
|
||||
compatible with HuggingFace weights.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
@@ -454,8 +454,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
|
||||
return loaded_params
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""
|
||||
Rewrite the weight name to match the format of the original model.
|
||||
"""Rewrite the weight name to match the format of the original model.
|
||||
Add .mtp_block for modules in transformer layer block for spec layer
|
||||
and rename shared layer weights to be top level.
|
||||
"""
|
||||
|
||||
@@ -22,7 +22,8 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Inference-only GLM-4.5, GLM-4.6, GLM-4.7 MTP
|
||||
model compatible with HuggingFace weights."""
|
||||
model compatible with HuggingFace weights.
|
||||
"""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
@@ -412,8 +413,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts):
|
||||
return loaded_params
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""
|
||||
Rewrite the weight name to match the format of the original model.
|
||||
"""Rewrite the weight name to match the format of the original model.
|
||||
Add .mtp_block for modules in transformer layer block for spec layer
|
||||
and rename shared layer weights to be top level.
|
||||
"""
|
||||
|
||||
@@ -65,12 +65,11 @@ from .utils import WeightsMapper
|
||||
|
||||
|
||||
class GLMVImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- b: Batch size
|
||||
- c: Number of channels (3)
|
||||
- h: Height of image
|
||||
- w: Width of image
|
||||
"""Dimensions:
|
||||
- b: Batch size
|
||||
- c: Number of channels (3)
|
||||
- h: Height of image
|
||||
- w: Width of image
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"] = "pixel_values"
|
||||
@@ -90,14 +89,15 @@ class EVA2CLIPPatchEmbedding(nn.Module):
|
||||
self.position_embedding = nn.Embedding(config.num_positions, config.hidden_size)
|
||||
|
||||
def forward(self, images: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Parameters:
|
||||
"""Parameters
|
||||
images : torch.Tensor
|
||||
Input image tensor with shape (B, C, H, W)
|
||||
|
||||
Returns:
|
||||
Returns
|
||||
-------
|
||||
torch.Tensor
|
||||
Transformed tensor with shape (B, L, D)
|
||||
|
||||
"""
|
||||
images = images.to(device=self.proj.weight.device, dtype=self.proj.weight.dtype)
|
||||
x = self.proj(images)
|
||||
@@ -246,8 +246,7 @@ class EVA2CLIPGLU(nn.Module):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
"""
|
||||
The original implementation is the same as:
|
||||
"""The original implementation is the same as:
|
||||
```python
|
||||
self.dense_h_to_4h = ColumnParallelLinear(
|
||||
config.hidden_size,
|
||||
@@ -349,14 +348,15 @@ class EVA2CLIPModel(nn.Module):
|
||||
self.scaling_factor = vision_config.scaling_factor
|
||||
|
||||
def forward(self, images: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Parameters:
|
||||
"""Parameters
|
||||
images : torch.Tensor
|
||||
Input image tensor with shape (B, C, H, W)
|
||||
|
||||
Returns:
|
||||
Returns
|
||||
-------
|
||||
torch.Tensor
|
||||
Transformed tensor with shape (B, L, D)
|
||||
|
||||
"""
|
||||
x = self.patch_embedding(images)
|
||||
x = self.transformer(x)
|
||||
@@ -518,9 +518,7 @@ class GLM4VForCausalLM(
|
||||
}
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models."""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="transformer.encoder",
|
||||
connector="transformer.vision.linear_proj",
|
||||
|
||||
@@ -246,8 +246,7 @@ class GlmOcrMTP(nn.Module, SupportsPP):
|
||||
return loaded_params
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""
|
||||
Rewrite the weight name to match the format of the original model.
|
||||
"""Rewrite the weight name to match the format of the original model.
|
||||
Add .mtp_block for modules in transformer layer block for spec layer
|
||||
and rename shared layer weights to be top level.
|
||||
"""
|
||||
|
||||
@@ -75,8 +75,7 @@ from .whisper import ISO639_1_SUPPORTED_LANGS, _create_fake_bias_for_k_proj
|
||||
|
||||
|
||||
class GlmAsrEncoderRotaryEmbedding(nn.Module):
|
||||
"""
|
||||
Rotary Position Embedding for GLM-ASR encoder.
|
||||
"""Rotary Position Embedding for GLM-ASR encoder.
|
||||
|
||||
Computes rotary position embeddings on-demand for efficiency.
|
||||
Only caches inv_freq as a buffer; cos/sin are computed during forward
|
||||
@@ -115,8 +114,7 @@ class GlmAsrEncoderRotaryEmbedding(nn.Module):
|
||||
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
||||
|
||||
def forward(self, seq_len: int) -> torch.Tensor:
|
||||
"""
|
||||
Compute rotary position frequencies for given sequence length.
|
||||
"""Compute rotary position frequencies for given sequence length.
|
||||
|
||||
Args:
|
||||
seq_len: The sequence length to compute embeddings for.
|
||||
@@ -124,6 +122,7 @@ class GlmAsrEncoderRotaryEmbedding(nn.Module):
|
||||
Returns:
|
||||
Frequency tensor with shape [seq_len, dim/2]. Use .cos() and
|
||||
.sin() to get the rotary embedding components.
|
||||
|
||||
"""
|
||||
# Compute on the same device as inv_freq (automatically correct after .to())
|
||||
seq = torch.arange(
|
||||
@@ -134,8 +133,7 @@ class GlmAsrEncoderRotaryEmbedding(nn.Module):
|
||||
|
||||
|
||||
class GlmAsrEncoderAttention(nn.Module):
|
||||
"""
|
||||
Optimized Multi-headed Grouped Query Attention for GLM-ASR encoder.
|
||||
"""Optimized Multi-headed Grouped Query Attention for GLM-ASR encoder.
|
||||
|
||||
Uses vLLM's QKVParallelLinear for fused projections, ApplyRotaryEmb for
|
||||
rotary position embeddings, and MMEncoderAttention for hardware-optimized
|
||||
@@ -208,14 +206,14 @@ class GlmAsrEncoderAttention(nn.Module):
|
||||
rotary_pos_emb_cos: torch.Tensor,
|
||||
rotary_pos_emb_sin: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
hidden_states: [batch_size, seq_len, hidden_size]
|
||||
rotary_pos_emb_cos: [seq_len, rotary_dim/2] - cosine of rotary embeddings
|
||||
rotary_pos_emb_sin: [seq_len, rotary_dim/2] - sine of rotary embeddings
|
||||
|
||||
Returns:
|
||||
[batch_size, seq_len, hidden_size]
|
||||
|
||||
"""
|
||||
batch_size, seq_len, _ = hidden_states.shape
|
||||
|
||||
@@ -255,8 +253,7 @@ class GlmAsrEncoderAttention(nn.Module):
|
||||
|
||||
|
||||
class GlmAsrEncoderMLP(nn.Module):
|
||||
"""
|
||||
Optimized MLP for GLM-ASR encoder.
|
||||
"""Optimized MLP for GLM-ASR encoder.
|
||||
Uses vLLM's parallel linear layers for better performance.
|
||||
"""
|
||||
|
||||
@@ -297,8 +294,7 @@ class GlmAsrEncoderMLP(nn.Module):
|
||||
|
||||
|
||||
class GlmAsrEncoderLayer(nn.Module):
|
||||
"""
|
||||
Optimized Transformer encoder layer for GLM-ASR.
|
||||
"""Optimized Transformer encoder layer for GLM-ASR.
|
||||
Combines attention and MLP with residual connections and layer norms.
|
||||
"""
|
||||
|
||||
@@ -335,14 +331,14 @@ class GlmAsrEncoderLayer(nn.Module):
|
||||
rotary_pos_emb_cos: torch.Tensor,
|
||||
rotary_pos_emb_sin: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
hidden_states: [batch_size, seq_len, hidden_size]
|
||||
rotary_pos_emb_cos: [seq_len, rotary_dim/2] - cosine of rotary embeddings
|
||||
rotary_pos_emb_sin: [seq_len, rotary_dim/2] - sine of rotary embeddings
|
||||
|
||||
Returns:
|
||||
[batch_size, seq_len, hidden_size]
|
||||
|
||||
"""
|
||||
# Self-attention with residual
|
||||
residual = hidden_states
|
||||
@@ -364,8 +360,7 @@ class GlmAsrEncoderLayer(nn.Module):
|
||||
|
||||
|
||||
class _GlmAsrEncoderOutput:
|
||||
"""
|
||||
Simple output container compatible with transformers' BaseModelOutput.
|
||||
"""Simple output container compatible with transformers' BaseModelOutput.
|
||||
|
||||
This lightweight container holds the encoder output and is compatible
|
||||
with the transformers library's output format while being more efficient
|
||||
@@ -374,6 +369,7 @@ class _GlmAsrEncoderOutput:
|
||||
Attributes:
|
||||
last_hidden_state: Final layer hidden states from the encoder.
|
||||
Shape: [batch_size, seq_len, hidden_size]
|
||||
|
||||
"""
|
||||
|
||||
__slots__ = ("last_hidden_state",)
|
||||
@@ -383,8 +379,7 @@ class _GlmAsrEncoderOutput:
|
||||
|
||||
|
||||
class GlmAsrEncoder(nn.Module):
|
||||
"""
|
||||
Optimized GLM-ASR Audio Encoder with vLLM native implementation.
|
||||
"""Optimized GLM-ASR Audio Encoder with vLLM native implementation.
|
||||
|
||||
This encoder processes audio features through convolutional layers
|
||||
followed by transformer layers with rotary position embeddings.
|
||||
@@ -454,14 +449,14 @@ class GlmAsrEncoder(nn.Module):
|
||||
def _get_feat_extract_output_lengths(
|
||||
self, input_lengths: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Compute the output length after convolutions.
|
||||
"""Compute the output length after convolutions.
|
||||
|
||||
Args:
|
||||
input_lengths: Input sequence lengths [batch_size]
|
||||
|
||||
Returns:
|
||||
Tuple of (output after conv1, output after conv2)
|
||||
|
||||
"""
|
||||
# Conv1: kernel=3, stride=1, padding=1
|
||||
output_lengths_conv1 = (input_lengths + 2 * 1 - 3) // 1 + 1
|
||||
@@ -472,8 +467,7 @@ class GlmAsrEncoder(nn.Module):
|
||||
return output_lengths_conv1, output_lengths_conv2
|
||||
|
||||
def forward(self, input_features: torch.Tensor) -> _GlmAsrEncoderOutput:
|
||||
"""
|
||||
Forward pass through the encoder.
|
||||
"""Forward pass through the encoder.
|
||||
|
||||
Args:
|
||||
input_features: [batch_size, num_mel_bins, seq_len]
|
||||
@@ -482,6 +476,7 @@ class GlmAsrEncoder(nn.Module):
|
||||
_GlmAsrEncoderOutput: Object with .last_hidden_state attribute \
|
||||
containing [batch_size, seq_len', hidden_size] where seq_len' \
|
||||
is the sequence length after convolutions
|
||||
|
||||
"""
|
||||
# Apply convolutional layers with GELU activation
|
||||
hidden_states = torch.nn.functional.gelu(self.conv1(input_features))
|
||||
@@ -515,11 +510,10 @@ class GlmAsrEncoder(nn.Module):
|
||||
|
||||
|
||||
class GlmAsrFeatureInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- num_chunks: Number of audio chunks (flattened)
|
||||
- nmb: Number of mel bins
|
||||
- num_audios: Number of original audio files
|
||||
"""Dimensions:
|
||||
- num_chunks: Number of audio chunks (flattened)
|
||||
- nmb: Number of mel bins
|
||||
- num_audios: Number of original audio files
|
||||
"""
|
||||
|
||||
type: Literal["audio_features"]
|
||||
@@ -538,12 +532,11 @@ class GlmAsrFeatureInputs(TensorSchema):
|
||||
|
||||
|
||||
class GlmAsrEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size
|
||||
- naf: Number of audio features
|
||||
- hs: Hidden size (must match the hidden size of language model
|
||||
backbone)
|
||||
"""Dimensions:
|
||||
- bn: Batch size
|
||||
- naf: Number of audio features
|
||||
- hs: Hidden size (must match the hidden size of language model
|
||||
backbone)
|
||||
"""
|
||||
|
||||
type: Literal["audio_embeds"] = "audio_embeds"
|
||||
@@ -557,8 +550,7 @@ GlmAsrInputs: TypeAlias = GlmAsrFeatureInputs | GlmAsrEmbeddingInputs
|
||||
|
||||
|
||||
class GlmAsrMultiModalProjector(nn.Module):
|
||||
"""
|
||||
Projects audio encoder outputs to language model hidden space.
|
||||
"""Projects audio encoder outputs to language model hidden space.
|
||||
|
||||
This projector uses a two-layer MLP to map audio features from the
|
||||
encoder's intermediate size to the language model's hidden size.
|
||||
@@ -601,8 +593,7 @@ class GlmAsrMultiModalProjector(nn.Module):
|
||||
def _glmasr_field_config(
|
||||
hf_inputs: Mapping[str, torch.Tensor],
|
||||
) -> dict[str, MultiModalFieldConfig]:
|
||||
"""
|
||||
Configure multimodal field batching strategy for GLM-ASR.
|
||||
"""Configure multimodal field batching strategy for GLM-ASR.
|
||||
|
||||
Determines how to batch audio inputs based on whether chunking is used.
|
||||
When chunk_counts is present, features are flattened across chunks;
|
||||
@@ -614,6 +605,7 @@ def _glmasr_field_config(
|
||||
Returns:
|
||||
Dictionary mapping field names to MultiModalFieldConfig objects \
|
||||
that specify batching behavior.
|
||||
|
||||
"""
|
||||
chunk_counts = hf_inputs.get("chunk_counts")
|
||||
if chunk_counts is not None:
|
||||
@@ -636,8 +628,7 @@ def _glmasr_field_config(
|
||||
|
||||
|
||||
class GlmAsrMultiModalDataParser(MultiModalDataParser):
|
||||
"""
|
||||
Custom parser for GLM-ASR multimodal data.
|
||||
"""Custom parser for GLM-ASR multimodal data.
|
||||
|
||||
Extends the base parser to handle GLM-ASR specific audio data formats,
|
||||
including both pre-computed audio embeddings and raw audio features.
|
||||
@@ -658,8 +649,7 @@ class GlmAsrMultiModalDataParser(MultiModalDataParser):
|
||||
|
||||
|
||||
class GlmAsrProcessingInfo(BaseProcessingInfo):
|
||||
"""
|
||||
Processing information provider for GLM-ASR model.
|
||||
"""Processing information provider for GLM-ASR model.
|
||||
|
||||
Provides access to model configuration, processor, and feature extractor
|
||||
needed for audio preprocessing and multimodal integration.
|
||||
@@ -687,8 +677,7 @@ class GlmAsrProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
|
||||
class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
|
||||
"""
|
||||
Builder for dummy inputs used in profiling and testing.
|
||||
"""Builder for dummy inputs used in profiling and testing.
|
||||
|
||||
Generates dummy text prompts and audio data that match the expected
|
||||
format for GLM-ASR model inputs. Used for memory profiling and
|
||||
@@ -727,8 +716,7 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
|
||||
|
||||
|
||||
class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"]):
|
||||
"""
|
||||
GLM-ASR processor that inherits directly from BaseMultiModalProcessor
|
||||
"""GLM-ASR processor that inherits directly from BaseMultiModalProcessor
|
||||
for better performance and cleaner implementation.
|
||||
"""
|
||||
|
||||
|
||||
@@ -71,8 +71,7 @@ def _get_audio_output_lengths_for_tower(
|
||||
merge_factor: int,
|
||||
conv_params: list[tuple[int, int, int]],
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Calculate the output lengths after audio processing.
|
||||
"""Calculate the output lengths after audio processing.
|
||||
|
||||
The output length accounts for:
|
||||
1. Convolution layers (downsampling)
|
||||
@@ -86,6 +85,7 @@ def _get_audio_output_lengths_for_tower(
|
||||
|
||||
Returns:
|
||||
Output lengths after all processing [batch_size]
|
||||
|
||||
"""
|
||||
# First, calculate the output length after convolutions
|
||||
if hasattr(audio_tower, "_get_feat_extract_output_lengths"):
|
||||
|
||||
@@ -327,6 +327,7 @@ class GPT2ForSequenceClassification(nn.Module, SupportsCrossEncoding):
|
||||
transformer: An instance of GPT2Model used for forward operations.
|
||||
score: A layer for calculating logits.
|
||||
_pooler: An instance of Pooler used for pooling operations.
|
||||
|
||||
"""
|
||||
|
||||
is_pooling_model = True
|
||||
|
||||
@@ -790,6 +790,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
|
||||
Returns:
|
||||
Weight dtype string (e.g., "mxfp4", "fp8") or None if not available
|
||||
|
||||
"""
|
||||
if hasattr(self.layers[layer_id].mlp.experts._quant_method, "weight_dtype"):
|
||||
return self.layers[layer_id].mlp.experts._quant_method.weight_dtype
|
||||
|
||||
@@ -84,6 +84,7 @@ def granite_layer_attn_params(
|
||||
Returns:
|
||||
Sliding window size (`None` for full attention), RoPE base theta (`0`
|
||||
for NoPE), and attention sink presence/absence.
|
||||
|
||||
"""
|
||||
layer_types = getattr(config, "layer_types", None)
|
||||
sliding_window = (
|
||||
|
||||
@@ -684,6 +684,7 @@ class Granite4VisionForConditionalGeneration(
|
||||
|
||||
Packing on dim=-1 means the framework's token-level slicing for
|
||||
chunked prefill preserves all levels intact.
|
||||
|
||||
"""
|
||||
select_strategy = self._vision_feature_select_strategy
|
||||
|
||||
|
||||
@@ -91,8 +91,7 @@ ISO639_1_SUPPORTED_LANGS = {
|
||||
|
||||
### Audio Input
|
||||
class GraniteSpeechAudioInputs(TensorSchema):
|
||||
"""
|
||||
Audio input features for Granite Speech model.
|
||||
"""Audio input features for Granite Speech model.
|
||||
|
||||
Dimensions:
|
||||
- b: Batch size
|
||||
@@ -477,7 +476,8 @@ class GraniteSpeechConformerConvModule(nn.Module):
|
||||
|
||||
class GraniteSpeechConformerBlock(nn.Module):
|
||||
"""Conformer block, consisting largely of linear layers,
|
||||
attention, and convolutional layers."""
|
||||
attention, and convolutional layers.
|
||||
"""
|
||||
|
||||
def __init__(self, config: PretrainedConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
@@ -732,9 +732,11 @@ class GraniteSpeechForConditionalGeneration(
|
||||
Args:
|
||||
audio_embed_sizes: torch.Tensor
|
||||
Tensor of num features in each seq in the batch.
|
||||
|
||||
Returns:
|
||||
torch.Tensor: Mask of shape (bsz, num_features) to be applied to
|
||||
the audio features prior to splitting the audio embeddings.
|
||||
|
||||
"""
|
||||
most_audio_features = int(torch.max(audio_embed_sizes))
|
||||
mask_indices = torch.arange(most_audio_features).view(1, -1)
|
||||
@@ -763,10 +765,12 @@ class GraniteSpeechForConditionalGeneration(
|
||||
Args:
|
||||
input_features: list[torch.Tensor]
|
||||
3D Input features to be coerced into a tensor.
|
||||
|
||||
Returns:
|
||||
torch.Tensor: Tensor of shape [bsz, num_features, 160], where
|
||||
num_features is the max number of features of any entry in the
|
||||
batch.
|
||||
|
||||
"""
|
||||
feat_lens = [feats.shape[1] for feats in input_features]
|
||||
padding = [max(feat_lens) - length for length in feat_lens]
|
||||
@@ -790,8 +794,10 @@ class GraniteSpeechForConditionalGeneration(
|
||||
audio_input: GraniteSpeechAudioInputs
|
||||
Audio inputs object containing Mel features, an input features
|
||||
mask, and the (flattened) number of audio tokens per instance.
|
||||
|
||||
Returns:
|
||||
tuple[torch.Tensor]: List of length bsz.
|
||||
|
||||
"""
|
||||
# TODO (Alex) - support embedding inputs
|
||||
encoder_embeds = self.encoder(audio_input["input_features"])
|
||||
|
||||
@@ -633,6 +633,7 @@ class GraniteMoeHybridForCausalLM(
|
||||
Tuple containing:
|
||||
- conv_state_shape: Shape for convolutional state cache
|
||||
- temporal_state_shape: Shape for state space model cache
|
||||
|
||||
"""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
HRM-Text: Hierarchical Reasoning Model — Text variant.
|
||||
"""HRM-Text: Hierarchical Reasoning Model — Text variant.
|
||||
|
||||
Reference Hugging Face implementation:
|
||||
src/transformers/models/hrm_text/modeling_hrm_text.py
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
HyperCLOVAX V2 (32B Think Model) Implementation.
|
||||
"""HyperCLOVAX V2 (32B Think Model) Implementation.
|
||||
|
||||
This module contains the V2 architecture that uses Qwen2.5 Vision Transformer
|
||||
instead of CLIP/SigLIP used in V1.
|
||||
@@ -57,8 +56,7 @@ V2_VIDEO_TOKEN: str = "<|video_start|><|VIDEO_PAD|><|video_end|>"
|
||||
|
||||
|
||||
class HCXVisionV2ImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
V2 Image inputs using Qwen2.5-VL style grid_thw format.
|
||||
"""V2 Image inputs using Qwen2.5-VL style grid_thw format.
|
||||
|
||||
Dimensions:
|
||||
- np: Number of patches
|
||||
@@ -72,8 +70,7 @@ class HCXVisionV2ImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class HCXVisionV2ImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
V2 Image embedding inputs.
|
||||
"""V2 Image embedding inputs.
|
||||
|
||||
Dimensions:
|
||||
- nf: Number of image features
|
||||
@@ -90,8 +87,7 @@ HCXVisionV2ImageInputs = HCXVisionV2ImagePixelInputs | HCXVisionV2ImageEmbedding
|
||||
|
||||
|
||||
class HCXVisionV2VideoPixelInputs(TensorSchema):
|
||||
"""
|
||||
V2 Video inputs using Qwen2.5-VL style grid_thw format.
|
||||
"""V2 Video inputs using Qwen2.5-VL style grid_thw format.
|
||||
|
||||
Dimensions:
|
||||
- np: Number of patches
|
||||
@@ -105,8 +101,7 @@ class HCXVisionV2VideoPixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class HCXVisionV2VideoEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
V2 Video embedding inputs.
|
||||
"""V2 Video embedding inputs.
|
||||
|
||||
Dimensions:
|
||||
- nf: Number of video features
|
||||
@@ -419,8 +414,7 @@ class HCXVisionV2MultiModalProcessor(
|
||||
dummy_inputs=HCXVisionV2DummyInputsBuilder,
|
||||
)
|
||||
class HCXVisionV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
"""
|
||||
HyperCLOVAX-SEED Vision-Language Model (V2 architecture).
|
||||
"""HyperCLOVAX-SEED Vision-Language Model (V2 architecture).
|
||||
|
||||
Supports:
|
||||
- HyperCLOVAX-SEED-Think-32B: Vision + Text
|
||||
|
||||
@@ -45,8 +45,7 @@ from .vision import is_vit_use_data_parallel, run_dp_sharded_vision_model
|
||||
|
||||
|
||||
class Idefics2VisionEmbeddings(nn.Module):
|
||||
"""
|
||||
This is a modified version of `siglip.modelign_siglip.SiglipVisionEmbeddings
|
||||
"""This is a modified version of `siglip.modelign_siglip.SiglipVisionEmbeddings
|
||||
` to enable images of variable
|
||||
resolution.
|
||||
|
||||
@@ -153,7 +152,7 @@ class Idefics2VisionEmbeddings(nn.Module):
|
||||
|
||||
|
||||
class Idefics2VisionAttention(nn.Module):
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -308,10 +307,9 @@ class Idefics2EncoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
attention_mask: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
hidden_states (`torch.FloatTensor`):
|
||||
Input to the layer of shape `(batch, seq_len, embed_dim)`.
|
||||
"""Args:
|
||||
hidden_states (`torch.FloatTensor`):
|
||||
Input to the layer of shape `(batch, seq_len, embed_dim)`.
|
||||
|
||||
"""
|
||||
residual = hidden_states
|
||||
@@ -326,13 +324,13 @@ class Idefics2EncoderLayer(nn.Module):
|
||||
|
||||
|
||||
class Idefics2Encoder(nn.Module):
|
||||
"""
|
||||
Transformer encoder consisting of `config.num_hidden_layers` self attention
|
||||
"""Transformer encoder consisting of `config.num_hidden_layers` self attention
|
||||
layers. Each layer is a
|
||||
[`Idefics2EncoderLayer`].
|
||||
|
||||
Args:
|
||||
config: Idefics2Config
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -368,14 +366,14 @@ class Idefics2Encoder(nn.Module):
|
||||
inputs_embeds: torch.Tensor,
|
||||
attention_mask: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
r"""
|
||||
Args:
|
||||
inputs_embeds (torch.Tensor):
|
||||
Optionally, instead of passing `input_ids` you can choose to
|
||||
directly pass an embedded representation.
|
||||
This is useful if you want more control over how to convert
|
||||
`input_ids` indices into associated vectorsthan the model's
|
||||
internal embedding lookup matrix.
|
||||
r"""Args:
|
||||
inputs_embeds (torch.Tensor):
|
||||
Optionally, instead of passing `input_ids` you can choose to
|
||||
directly pass an embedded representation.
|
||||
This is useful if you want more control over how to convert
|
||||
`input_ids` indices into associated vectorsthan the model's
|
||||
internal embedding lookup matrix.
|
||||
|
||||
"""
|
||||
hidden_states = inputs_embeds
|
||||
for encoder_layer in self.layers:
|
||||
|
||||
@@ -70,13 +70,12 @@ from .utils import AutoWeightsLoader, maybe_prefix
|
||||
|
||||
|
||||
class Idefics3ImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- bnp: Batch size * number of images * number of patches
|
||||
- c: Number of channels (3)
|
||||
- h: Height
|
||||
- w: Width
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- bnp: Batch size * number of images * number of patches
|
||||
- c: Number of channels (3)
|
||||
- h: Height
|
||||
- w: Width
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
@@ -86,11 +85,10 @@ class Idefics3ImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class Idefics3ImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- f: Image feature size
|
||||
- h: Hidden size (must match the hidden size of language model backbone)
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- f: Image feature size
|
||||
- h: Hidden size (must match the hidden size of language model backbone)
|
||||
"""
|
||||
|
||||
type: Literal["image_embeds"]
|
||||
@@ -829,9 +827,7 @@ class Idefics3ForConditionalGeneration(
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models."""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="model.text_model",
|
||||
connector="model.connector",
|
||||
|
||||
@@ -112,8 +112,7 @@ class StreamingTranscriptionPostProcessor:
|
||||
|
||||
|
||||
def _require_is_multimodal(is_multimodal: Tensor | None) -> Tensor:
|
||||
"""
|
||||
A helper function to be used in the context of
|
||||
"""A helper function to be used in the context of
|
||||
[vllm.model_executor.models.interfaces.SupportsMultiModal.embed_input_ids][]
|
||||
to provide a better error message.
|
||||
"""
|
||||
@@ -225,20 +224,18 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
"""
|
||||
Get the placeholder text for the `i`th `modality` item in the prompt.
|
||||
"""
|
||||
"""Get the placeholder text for the `i`th `modality` item in the prompt."""
|
||||
...
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
"""
|
||||
Returns multimodal embeddings generated from multimodal kwargs
|
||||
"""Returns multimodal embeddings generated from multimodal kwargs
|
||||
to be merged with text embeddings.
|
||||
|
||||
Note:
|
||||
The returned multimodal embeddings must be in the same order as
|
||||
the appearances of their corresponding multimodal data item in the
|
||||
input prompt.
|
||||
|
||||
"""
|
||||
...
|
||||
|
||||
@@ -254,14 +251,14 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
)
|
||||
|
||||
def get_language_model(self) -> "VllmModel":
|
||||
"""
|
||||
Returns the underlying language model used for text generation.
|
||||
"""Returns the underlying language model used for text generation.
|
||||
|
||||
This is typically the `torch.nn.Module` instance responsible for
|
||||
processing the merged multimodal embeddings and producing hidden states
|
||||
|
||||
Returns:
|
||||
torch.nn.Module: The core language model component.
|
||||
|
||||
"""
|
||||
# Cached
|
||||
assert isinstance(self, nn.Module)
|
||||
@@ -302,8 +299,7 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
*,
|
||||
targets: type[nn.Module] | tuple[type[nn.Module], ...] | None = None,
|
||||
):
|
||||
"""
|
||||
Mark each child module that was assigned to this model during this context
|
||||
"""Mark each child module that was assigned to this model during this context
|
||||
as a language model component.
|
||||
|
||||
Language model components are automatically skipped in `--mm-encoder-only`
|
||||
@@ -338,8 +334,7 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
*,
|
||||
targets: type[nn.Module] | tuple[type[nn.Module], ...] | None = None,
|
||||
):
|
||||
"""
|
||||
Mark each child module that was assigned to this model during this context
|
||||
"""Mark each child module that was assigned to this model during this context
|
||||
as a tower model component.
|
||||
|
||||
Tower model components are automatically skipped when `--limit-mm-per-prompt`
|
||||
@@ -397,8 +392,7 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
language_targets: type[nn.Module] | tuple[type[nn.Module], ...],
|
||||
tower_targets: dict[str, type[nn.Module] | tuple[type[nn.Module], ...]],
|
||||
):
|
||||
"""
|
||||
Composite wrapper over `_mark_language_model` and
|
||||
"""Composite wrapper over `_mark_language_model` and
|
||||
`_mark_tower_model` by modality.
|
||||
"""
|
||||
with ExitStack() as stack:
|
||||
@@ -438,8 +432,7 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
mm_kwargs: "MultiModalKwargsItem | None",
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
"""
|
||||
Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings.
|
||||
"""Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings.
|
||||
|
||||
MM LoRA uses these counts to build adapter mappings for the tower and
|
||||
connector forwards. Models with multiple modalities can override this
|
||||
@@ -497,8 +490,7 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
*,
|
||||
is_multimodal: Tensor | None = None,
|
||||
) -> Tensor:
|
||||
"""
|
||||
Apply token embeddings to `input_ids`.
|
||||
"""Apply token embeddings to `input_ids`.
|
||||
|
||||
If `multimodal_embeddings` is passed, scatter them into
|
||||
`input_ids` according to the mask `is_multimodal`.
|
||||
@@ -551,8 +543,7 @@ class SupportsMultiModalPruning(Protocol):
|
||||
mrope_positions: torch.LongTensor,
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[Sequence[torch.Tensor], Tensor, int]:
|
||||
"""
|
||||
Update part of input mrope positions (starting with
|
||||
"""Update part of input mrope positions (starting with
|
||||
num_computed_tokens index). Original mrope_positions are computed
|
||||
for unpruned sequence and becomes incorrect once pruning occurs,
|
||||
so once we prune media tokens we should reflect this in the
|
||||
@@ -571,6 +562,7 @@ class SupportsMultiModalPruning(Protocol):
|
||||
Returns:
|
||||
Tuple of (multimodal_embeddings, mrope_positions,
|
||||
mrope_position_delta).
|
||||
|
||||
"""
|
||||
...
|
||||
|
||||
@@ -650,16 +642,12 @@ class SupportsScoreTemplate(Protocol):
|
||||
|
||||
@classmethod
|
||||
def get_score_template(cls, query: str, document: str) -> str | None:
|
||||
"""
|
||||
Generate a full prompt by populating the score template with query and document content.
|
||||
""" # noqa: E501
|
||||
"""Generate a full prompt by populating the score template with query and document content.""" # noqa: E501
|
||||
...
|
||||
|
||||
@classmethod
|
||||
def post_process_tokens(cls, prompt: "TokensPrompt") -> None:
|
||||
"""
|
||||
Perform architecture-specific manipulations on the input tokens.
|
||||
"""
|
||||
"""Perform architecture-specific manipulations on the input tokens."""
|
||||
...
|
||||
|
||||
|
||||
@@ -790,8 +778,7 @@ class SupportsPP(Protocol):
|
||||
*,
|
||||
intermediate_tensors: "IntermediateTensors | None",
|
||||
) -> "Tensor | IntermediateTensors | tuple[Tensor, list[Tensor]]":
|
||||
"""
|
||||
Accept [`IntermediateTensors`][vllm.sequence.IntermediateTensors] when
|
||||
"""Accept [`IntermediateTensors`][vllm.sequence.IntermediateTensors] when
|
||||
PP rank > 0.
|
||||
|
||||
Return [`IntermediateTensors`][vllm.sequence.IntermediateTensors] only
|
||||
@@ -905,7 +892,8 @@ def has_inner_state(
|
||||
@runtime_checkable
|
||||
class IsAttentionFree(Protocol):
|
||||
"""The interface required for all models like Mamba that lack attention,
|
||||
but do have state whose size is constant wrt the number of tokens."""
|
||||
but do have state whose size is constant wrt the number of tokens.
|
||||
"""
|
||||
|
||||
is_attention_free: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
@@ -933,7 +921,8 @@ def is_attention_free(
|
||||
class IsHybrid(Protocol):
|
||||
"""The interface required for all models like Jamba that have both
|
||||
attention and mamba blocks, indicates that
|
||||
hf_config has 'layers_block_type'"""
|
||||
hf_config has 'layers_block_type'
|
||||
"""
|
||||
|
||||
is_hybrid: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
@@ -953,6 +942,7 @@ class IsHybrid(Protocol):
|
||||
|
||||
Returns:
|
||||
Shapes for each state cache used by the model.
|
||||
|
||||
"""
|
||||
...
|
||||
|
||||
@@ -966,6 +956,7 @@ class IsHybrid(Protocol):
|
||||
(state, block_ids, cur_block_idx, num_accepted_tokens) and returns
|
||||
a MambaCopySpec describing the memory-copy parameters for prefix
|
||||
caching in align mode.
|
||||
|
||||
"""
|
||||
...
|
||||
|
||||
@@ -994,9 +985,7 @@ def is_hybrid(
|
||||
|
||||
@runtime_checkable
|
||||
class MixtureOfExperts(Protocol):
|
||||
"""
|
||||
Check if the model is a mixture of experts (MoE) model.
|
||||
"""
|
||||
"""Check if the model is a mixture of experts (MoE) model."""
|
||||
|
||||
expert_weights: MutableSequence[Sequence[Tensor]]
|
||||
"""
|
||||
@@ -1039,8 +1028,7 @@ class MixtureOfExperts(Protocol):
|
||||
logical_to_physical_map: Tensor,
|
||||
logical_replica_count: Tensor,
|
||||
) -> None:
|
||||
"""
|
||||
Register the EPLB state in the MoE model.
|
||||
"""Register the EPLB state in the MoE model.
|
||||
|
||||
Since these are views of the actual EPLB state, any changes made by
|
||||
the EPLB algorithm are automatically reflected in the model's behavior
|
||||
@@ -1054,6 +1042,7 @@ class MixtureOfExperts(Protocol):
|
||||
expert_load_view: A view of the expert load metrics tensor.
|
||||
logical_to_physical_map: Mapping from logical to physical experts.
|
||||
logical_replica_count: Count of replicas for each logical expert.
|
||||
|
||||
"""
|
||||
self.expert_weights = []
|
||||
for layer_idx, layer in enumerate(self.moe_layers):
|
||||
@@ -1086,8 +1075,8 @@ def get_mixture_of_experts_model(model: object) -> MixtureOfExperts | None:
|
||||
|
||||
Returns:
|
||||
The MixtureOfExperts instance contained within the model, or None.
|
||||
"""
|
||||
|
||||
"""
|
||||
if is_mixture_of_experts(model):
|
||||
return model
|
||||
|
||||
@@ -1349,7 +1338,8 @@ class SupportsTranscription(Protocol):
|
||||
) -> "PromptType":
|
||||
"""Get the prompt for the ASR model.
|
||||
The model has control over the construction, as long as it
|
||||
returns a valid PromptType."""
|
||||
returns a valid PromptType.
|
||||
"""
|
||||
...
|
||||
|
||||
@classmethod
|
||||
@@ -1359,8 +1349,7 @@ class SupportsTranscription(Protocol):
|
||||
|
||||
@classmethod
|
||||
def validate_language(cls, language: str | None) -> str | None:
|
||||
"""
|
||||
Ensure the language specified in the transcription request
|
||||
"""Ensure the language specified in the transcription request
|
||||
is a valid ISO 639-1 language code. If the request language is
|
||||
valid, but not natively supported by the model, trigger a
|
||||
warning (but not an exception).
|
||||
@@ -1396,8 +1385,7 @@ class SupportsTranscription(Protocol):
|
||||
stt_config: "SpeechToTextConfig",
|
||||
model_config: "ModelConfig",
|
||||
) -> int | None:
|
||||
"""
|
||||
Map from audio duration to number of audio tokens produced by the ASR
|
||||
"""Map from audio duration to number of audio tokens produced by the ASR
|
||||
model, without running a forward pass.
|
||||
This is used for estimating the amount of processing for this audio.
|
||||
"""
|
||||
@@ -1405,8 +1393,7 @@ class SupportsTranscription(Protocol):
|
||||
|
||||
@classmethod
|
||||
def post_process_output(cls, text: str) -> str:
|
||||
"""
|
||||
Post-process the raw model output text.
|
||||
"""Post-process the raw model output text.
|
||||
|
||||
Some ASR models output structured formats (e.g., language tags,
|
||||
special tokens) that need to be stripped before returning to the user.
|
||||
@@ -1416,6 +1403,7 @@ class SupportsTranscription(Protocol):
|
||||
|
||||
Returns:
|
||||
Cleaned transcription text.
|
||||
|
||||
"""
|
||||
return text
|
||||
|
||||
@@ -1432,8 +1420,7 @@ class SupportsTranscription(Protocol):
|
||||
def get_streaming_post_processor_cls(
|
||||
cls,
|
||||
) -> type[StreamingTranscriptionPostProcessor]:
|
||||
"""
|
||||
Return a stateful post-processor class for streaming output deltas.
|
||||
"""Return a stateful post-processor class for streaming output deltas.
|
||||
|
||||
Each instance receives the next decoded text delta and whether the
|
||||
request output is final. It returns the cleaned delta that should be
|
||||
@@ -1644,7 +1631,8 @@ class EagleModelMixin:
|
||||
@runtime_checkable
|
||||
class SupportsEagle(SupportsEagleBase, Protocol):
|
||||
"""The interface required for models that support
|
||||
EAGLE-1 and EAGLE-2 speculative decoding."""
|
||||
EAGLE-1 and EAGLE-2 speculative decoding.
|
||||
"""
|
||||
|
||||
supports_eagle: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
@@ -1674,7 +1662,8 @@ def supports_eagle(
|
||||
@runtime_checkable
|
||||
class SupportsEagle3(SupportsEagleBase, Protocol):
|
||||
"""The interface required for models that support
|
||||
EAGLE-3 speculative decoding."""
|
||||
EAGLE-3 speculative decoding.
|
||||
"""
|
||||
|
||||
supports_eagle3: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
@@ -1687,12 +1676,12 @@ class SupportsEagle3(SupportsEagleBase, Protocol):
|
||||
"""
|
||||
|
||||
def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
|
||||
"""
|
||||
Set which layers should output auxiliary hidden states for EAGLE-3.
|
||||
"""Set which layers should output auxiliary hidden states for EAGLE-3.
|
||||
|
||||
Args:
|
||||
layers: Tuple of layer indices that should output auxiliary
|
||||
hidden states.
|
||||
|
||||
"""
|
||||
parent_ref = self
|
||||
if hasattr(self, "get_language_model"):
|
||||
@@ -1710,14 +1699,14 @@ class SupportsEagle3(SupportsEagleBase, Protocol):
|
||||
holder._set_aux_hidden_state_layers(layers)
|
||||
|
||||
def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]:
|
||||
"""
|
||||
Get the default layer indices that should output auxiliary hidden states
|
||||
"""Get the default layer indices that should output auxiliary hidden states
|
||||
for EAGLE-3 for this model. Models can override this method to provide
|
||||
different default layers based on their architecture, but it is encouraged
|
||||
to instead include the layer specification in the model's config if possible.
|
||||
|
||||
Returns:
|
||||
Tuple of layer indices for auxiliary hidden state outputs.
|
||||
|
||||
"""
|
||||
parent_ref = self
|
||||
if hasattr(self, "get_language_model"):
|
||||
@@ -1765,8 +1754,7 @@ class SupportsMRoPE(Protocol):
|
||||
input_tokens: list[int],
|
||||
mm_features: list["MultiModalFeatureSpec"],
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
"""
|
||||
Get M-RoPE input positions and delta value for this specific model.
|
||||
"""Get M-RoPE input positions and delta value for this specific model.
|
||||
|
||||
This method should be implemented by each model that supports M-RoPE
|
||||
to provide model-specific logic for computing input positions.
|
||||
@@ -1779,6 +1767,7 @@ class SupportsMRoPE(Protocol):
|
||||
llm_positions: Tensor of shape `[num_dims, num_tokens]`, one row
|
||||
per M-RoPE position channel (e.g. T/H/W).
|
||||
mrope_position_delta: Delta for position calculations.
|
||||
|
||||
"""
|
||||
...
|
||||
|
||||
@@ -1886,8 +1875,7 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
clone: bool = False,
|
||||
batch_mm_kwargs: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Post-process encoder output, directly call scatter_output_slices by default.
|
||||
"""Post-process encoder output, directly call scatter_output_slices by default.
|
||||
|
||||
By default, delegates directly to scatter_output_slices.
|
||||
Override this for models that require additional processing on the raw
|
||||
@@ -1913,10 +1901,17 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
"""Create dummy inputs and buffers for CUDA graph capture.
|
||||
|
||||
Args:
|
||||
token_budget: Token budget the capture is sized for.
|
||||
max_batch_size: Maximum number of items in a captured batch.
|
||||
max_frames_per_batch: Maximum number of frames in a captured batch.
|
||||
device: Device the dummy inputs and buffers are created on.
|
||||
dtype: Dtype of the dummy inputs and buffers.
|
||||
path: Configured encoder path.
|
||||
axis_keys: The resolved capture-axis keys (one per axis of
|
||||
``EncoderCudaGraphConfig.capture_axes``) this capture is for.
|
||||
None or empty when no capture axes are configured; models
|
||||
without capture axes ignore it.
|
||||
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
@@ -143,7 +143,7 @@ class InternVisionPatchModel(nn.Module):
|
||||
|
||||
|
||||
class InternParallelAttention(nn.Module):
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -89,13 +89,12 @@ class InternS1MultiModalProjector(nn.Module):
|
||||
|
||||
|
||||
class InternS1ImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bnp: Batch size * number of images * (1 + num_patches)
|
||||
- c: Number of channels (3)
|
||||
- h: Height
|
||||
- w: Width
|
||||
- bn: Batch size * number of images
|
||||
"""Dimensions:
|
||||
- bnp: Batch size * number of images * (1 + num_patches)
|
||||
- c: Number of channels (3)
|
||||
- h: Height
|
||||
- w: Width
|
||||
- bn: Batch size * number of images
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"] = "pixel_values"
|
||||
@@ -104,11 +103,10 @@ class InternS1ImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class InternS1ImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- ni: Number of images
|
||||
- tifs: Total image feature size
|
||||
- hs: Hidden size (must match language model backbone)
|
||||
"""Dimensions:
|
||||
- ni: Number of images
|
||||
- tifs: Total image feature size
|
||||
- hs: Hidden size (must match language model backbone)
|
||||
"""
|
||||
|
||||
type: Literal["image_embeds"] = "image_embeds"
|
||||
@@ -119,13 +117,12 @@ InternS1ImageInputs: TypeAlias = InternS1ImagePixelInputs | InternS1ImageEmbeddi
|
||||
|
||||
|
||||
class InternS1VideoPixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bnv: Batch size * number of videos * number of frames
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels (3)
|
||||
- h: Height
|
||||
- w: Width
|
||||
"""Dimensions:
|
||||
- bnv: Batch size * number of videos * number of frames
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels (3)
|
||||
- h: Height
|
||||
- w: Width
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values_videos"] = "pixel_values_videos"
|
||||
@@ -134,11 +131,10 @@ class InternS1VideoPixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class InternS1VideoEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- nv: Number of videos
|
||||
- tvfs: Total video feature size
|
||||
- hs: Hidden size (must match language model backbone)
|
||||
"""Dimensions:
|
||||
- nv: Number of videos
|
||||
- tvfs: Total video feature size
|
||||
- hs: Hidden size (must match language model backbone)
|
||||
"""
|
||||
|
||||
type: Literal["video_embeds"] = "video_embeds"
|
||||
@@ -862,9 +858,7 @@ class InternS1ForConditionalGeneration(
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models."""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="multi_modal_projector",
|
||||
|
||||
@@ -621,7 +621,7 @@ class InternS1ProForConditionalGeneration(
|
||||
return mapper
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
"""load weights"""
|
||||
"""Load weights"""
|
||||
orig_to_new_prefix: dict[str, str | None] = {
|
||||
"model.visual.": "visual.",
|
||||
"lm_head.": "language_model.lm_head.",
|
||||
|
||||
@@ -91,15 +91,13 @@ class InternS1VisionEmbeddings(nn.Module):
|
||||
def interpolate_pos_encoding(
|
||||
self, embeddings: torch.Tensor, height: int, width: int
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
|
||||
"""This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
|
||||
images. This method is also adapted to support torch.jit tracing.
|
||||
|
||||
Adapted from:
|
||||
- https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
|
||||
- https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
|
||||
""" # noqa: E501
|
||||
|
||||
num_patches = embeddings.shape[1] - 1
|
||||
num_positions = self.position_embeddings.shape[1] - 1
|
||||
|
||||
@@ -164,7 +162,7 @@ class InternS1VisionEmbeddings(nn.Module):
|
||||
|
||||
|
||||
class InternSdpaAttention(nn.Module):
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -225,8 +223,7 @@ class InternSdpaAttention(nn.Module):
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""x shape: (B, N, C)"""
|
||||
|
||||
"""X shape: (B, N, C)."""
|
||||
q = self.q_proj(x)
|
||||
k = self.k_proj(x)
|
||||
v = self.v_proj(x)
|
||||
|
||||
@@ -70,13 +70,12 @@ from .utils import (
|
||||
|
||||
|
||||
class InternVLImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- bnp: Batch size * number of images * (1 + num_patches)
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each image patch
|
||||
- w: Width of each image patch
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- bnp: Batch size * number of images * (1 + num_patches)
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each image patch
|
||||
- w: Width of each image patch
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
@@ -85,11 +84,10 @@ class InternVLImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class InternVLImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- n: Number of images
|
||||
- f: Total image feature size
|
||||
- h: Hidden size (must match the hidden size of language model backbone)
|
||||
"""Dimensions:
|
||||
- n: Number of images
|
||||
- f: Total image feature size
|
||||
- h: Hidden size (must match the hidden size of language model backbone)
|
||||
"""
|
||||
|
||||
type: Literal["image_embeds"]
|
||||
@@ -100,13 +98,12 @@ InternVLImageInputs: TypeAlias = InternVLImagePixelInputs | InternVLImageEmbeddi
|
||||
|
||||
|
||||
class InternVLVideoPixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bvf: Batch size * number of videos * num_frames
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each video frame
|
||||
- w: Width of each video frame
|
||||
"""Dimensions:
|
||||
- bvf: Batch size * number of videos * num_frames
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels (3)
|
||||
- h: Height of each video frame
|
||||
- w: Width of each video frame
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values_videos"]
|
||||
@@ -115,11 +112,10 @@ class InternVLVideoPixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class InternVLVideoEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- n: Number of videos
|
||||
- f: Total video feature size
|
||||
- h: Hidden size (must match the hidden size of language model backbone)
|
||||
"""Dimensions:
|
||||
- n: Number of videos
|
||||
- f: Total video feature size
|
||||
- h: Hidden size (must match the hidden size of language model backbone)
|
||||
"""
|
||||
|
||||
type: Literal["video_embeds"]
|
||||
@@ -327,7 +323,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
|
||||
class InternVLProcessingInfo(BaseInternVLProcessingInfo):
|
||||
"""InternVL ProcessingInfo extended for video processing"""
|
||||
"""InternVL ProcessingInfo extended for video processing."""
|
||||
|
||||
def get_image_processor(self, **kwargs):
|
||||
config = self.get_hf_config()
|
||||
@@ -418,7 +414,7 @@ class InternVLProcessingInfo(BaseInternVLProcessingInfo):
|
||||
class InternVLDummyInputsBuilder(
|
||||
BaseInternVLDummyInputsBuilder[InternVLProcessingInfo]
|
||||
):
|
||||
"""InternVL DummyInputsBuilder extended for video support"""
|
||||
"""InternVL DummyInputsBuilder extended for video support."""
|
||||
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_videos = mm_counts.get("video", 0)
|
||||
@@ -460,7 +456,7 @@ class InternVLDummyInputsBuilder(
|
||||
class InternVLMultiModalProcessor(
|
||||
BaseInternVLMultiModalProcessor[InternVLProcessingInfo]
|
||||
):
|
||||
"""InternVL MultiModalProcessor extended for video support"""
|
||||
"""InternVL MultiModalProcessor extended for video support."""
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
@@ -906,9 +902,7 @@ class InternVLChatModel(
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models."""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="mlp1",
|
||||
|
||||
@@ -321,6 +321,7 @@ class LoopGateProjection(nn.Module):
|
||||
|
||||
Returns:
|
||||
gate: [num_tokens, num_heads * head_dim] (flattened format matching q shape)
|
||||
|
||||
"""
|
||||
num_heads, num_tokens, head_dim = query.shape
|
||||
|
||||
|
||||
@@ -177,23 +177,23 @@ def create_pixel_shuffle_index_map(
|
||||
scale_factor: int = 1,
|
||||
device: torch.device | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Build a gather-index map that tells us, for every *output* token after
|
||||
"""Build a gather-index map that tells us, for every *output* token after
|
||||
pixel-shuffle, which `scale_factor**2` *input* tokens are being merged.
|
||||
|
||||
Args
|
||||
Args:
|
||||
----
|
||||
seq_sizes : (num_images,) - #patches in each image (row-major order)
|
||||
token_grids : (num_images,2) - (height, width) for every image
|
||||
scale_factor : spatial down-scale factor (≥2)
|
||||
device : (optional) overrides `seq_sizes.device`
|
||||
|
||||
Returns
|
||||
Returns:
|
||||
-------
|
||||
gather_idx : (new_total_seq_len, scale_factor**2) int64 tensor.
|
||||
gather_idx[i, j] is the *flat* index into the *original*
|
||||
packed sequence for the j-th sub-patch that forms the
|
||||
i-th output token.
|
||||
|
||||
"""
|
||||
if device is None:
|
||||
device = seq_sizes.device
|
||||
@@ -266,6 +266,7 @@ def pixel_shuffle_varlen(
|
||||
|
||||
Raises:
|
||||
ValueError: If more than one batch item is provided.
|
||||
|
||||
"""
|
||||
keep_batch_dim = x.dim() == 3
|
||||
if keep_batch_dim:
|
||||
@@ -403,8 +404,7 @@ class IsaacDummyInputsBuilder(BaseDummyInputsBuilder[IsaacProcessingInfo]):
|
||||
|
||||
|
||||
class IsaacImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Schema for validating Isaac image inputs.
|
||||
"""Schema for validating Isaac image inputs.
|
||||
|
||||
Dimensions:
|
||||
- np: Number of patches
|
||||
@@ -687,12 +687,10 @@ class Siglip2VisionTransformer(nn.Module):
|
||||
self,
|
||||
packed_seq_patches: tuple[torch.Tensor, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
r"""
|
||||
spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):
|
||||
Tensor containing the spatial dimensions (height, width)
|
||||
of the input images.
|
||||
r"""spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):
|
||||
Tensor containing the spatial dimensions (height, width)
|
||||
of the input images.
|
||||
"""
|
||||
|
||||
seq_patches, token_grids = packed_seq_patches
|
||||
seq_sizes = torch.prod(token_grids, dim=-1)
|
||||
|
||||
@@ -1015,9 +1013,7 @@ class IsaacForConditionalGeneration(
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models."""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="vision_embedding.linear_fc2", # The final linear layer
|
||||
|
||||
@@ -48,11 +48,10 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class KananaVImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- np: The total number of patches over all images in the batch
|
||||
- cps: Number of channels * patch_size * patch_size
|
||||
- ni: Number of images
|
||||
"""Dimensions:
|
||||
- np: The total number of patches over all images in the batch
|
||||
- cps: Number of channels * patch_size * patch_size
|
||||
- ni: Number of images
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
|
||||
@@ -125,13 +125,12 @@ def smart_resize(
|
||||
|
||||
|
||||
class KeyeImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bnp: Batch size * Number of patches
|
||||
- c: Number of channels
|
||||
- ps: Patch size
|
||||
- ni: Number of images
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""Dimensions:
|
||||
- bnp: Batch size * Number of patches
|
||||
- c: Number of channels
|
||||
- ps: Patch size
|
||||
- ni: Number of images
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
@@ -142,13 +141,12 @@ class KeyeImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class KeyeImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- nf: Number of image features
|
||||
- hs: Hidden size (must match the hidden size of language model
|
||||
backbone)
|
||||
- ni: Number of images
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""Dimensions:
|
||||
- nf: Number of image features
|
||||
- hs: Hidden size (must match the hidden size of language model
|
||||
backbone)
|
||||
- ni: Number of images
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""
|
||||
|
||||
type: Literal["image_embeds"]
|
||||
@@ -160,13 +158,12 @@ KeyeImageInputs: TypeAlias = KeyeImagePixelInputs | KeyeImageEmbeddingInputs
|
||||
|
||||
|
||||
class KeyeVideoPixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bnp: Batch size * Number of patches
|
||||
- c: Number of channels
|
||||
- ps: Patch size
|
||||
- ni: Number of images
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""Dimensions:
|
||||
- bnp: Batch size * Number of patches
|
||||
- c: Number of channels
|
||||
- ps: Patch size
|
||||
- ni: Number of images
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values_videos"]
|
||||
@@ -177,13 +174,12 @@ class KeyeVideoPixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class KeyeVideoEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- nf: Number of video features
|
||||
- hs: Hidden size (must match the hidden size of language model
|
||||
backbone)
|
||||
- nv: Number of videos
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""Dimensions:
|
||||
- nf: Number of video features
|
||||
- hs: Hidden size (must match the hidden size of language model
|
||||
backbone)
|
||||
- nv: Number of videos
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""
|
||||
|
||||
type: Literal["video_embeds"]
|
||||
@@ -349,7 +345,8 @@ def apply_rotary_pos_emb_flashatt(
|
||||
|
||||
class KeyeSiglipAttention(nn.Module):
|
||||
"""Multi-headed attention from 'Attention Is All You
|
||||
Need' paper."""
|
||||
Need' paper.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1465,6 +1462,9 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
|
||||
otherwise it will be `(seq_len,)`.
|
||||
intermediate_tensors: Intermediate tensors from prior forward pass.
|
||||
inputs_embeds: Optional tensor of input embeddings.
|
||||
**kwargs: Multimodal inputs for this batch, forwarded to the
|
||||
multimodal embedding path.
|
||||
|
||||
"""
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
@@ -1588,8 +1588,7 @@ class KeyeForConditionalGeneration(
|
||||
def _split_video_grid_thw(
|
||||
grid_thw: torch.Tensor | list[list[int]] | list[int],
|
||||
) -> list[list[int]]:
|
||||
"""
|
||||
Split video grid_thw along the t dimension into per-frame rows.
|
||||
"""Split video grid_thw along the t dimension into per-frame rows.
|
||||
|
||||
This preserves Keye's current M-RoPE behavior, where a video is emitted
|
||||
as consecutive frame-level multimodal blocks rather than a single block
|
||||
|
||||
@@ -51,8 +51,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
def split_thw(grid_thw: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Split grid_thw in t dimension.
|
||||
"""Split grid_thw in t dimension.
|
||||
|
||||
Args:
|
||||
grid_thw: [N, 3] tensor of [t, h, w]
|
||||
@@ -66,6 +65,7 @@ def split_thw(grid_thw: torch.Tensor) -> torch.Tensor:
|
||||
tensor([[1, 3, 4],
|
||||
[1, 3, 4],
|
||||
[1, 5, 6]])
|
||||
|
||||
"""
|
||||
t = grid_thw[:, 0]
|
||||
h_w = grid_thw[:, 1:]
|
||||
@@ -76,8 +76,7 @@ def split_thw(grid_thw: torch.Tensor) -> torch.Tensor:
|
||||
def get_num_patches(
|
||||
grid_thw: torch.Tensor, num_frames: list[int] | torch.Tensor
|
||||
) -> list[int]:
|
||||
"""
|
||||
Return num_patches per video.
|
||||
"""Return num_patches per video.
|
||||
|
||||
Args:
|
||||
grid_thw: Tensor with shape [N, 3] containing temporal, height, width
|
||||
@@ -101,8 +100,8 @@ def get_num_patches(
|
||||
>>> get_num_patches(grid_thw, num_frames)
|
||||
tensor([16, 1]) # Total patches for first video: 8+8=16,
|
||||
second video: 1.
|
||||
"""
|
||||
|
||||
"""
|
||||
assert len(grid_thw.shape) == 2
|
||||
if isinstance(num_frames, torch.Tensor):
|
||||
num_frames = num_frames.clone().tolist()
|
||||
@@ -121,13 +120,12 @@ def get_num_patches(
|
||||
|
||||
|
||||
class KeyeVL1_5ImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bnp: Batch size * Number of patches
|
||||
- c: Number of channels
|
||||
- ps: Patch size
|
||||
- ni: Number of images
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""Dimensions:
|
||||
- bnp: Batch size * Number of patches
|
||||
- c: Number of channels
|
||||
- ps: Patch size
|
||||
- ni: Number of images
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
@@ -140,13 +138,12 @@ class KeyeVL1_5ImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class KeyeVL1_5ImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- nf: Number of image features
|
||||
- hs: Hidden size (must match the hidden size of language model
|
||||
backbone)
|
||||
- ni: Number of images
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""Dimensions:
|
||||
- nf: Number of image features
|
||||
- hs: Hidden size (must match the hidden size of language model
|
||||
backbone)
|
||||
- ni: Number of images
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""
|
||||
|
||||
type: Literal["image_embeds"]
|
||||
@@ -160,13 +157,12 @@ KeyeVL1_5ImageInputs: TypeAlias = (
|
||||
|
||||
|
||||
class KeyeVL1_5VideoPixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bnp: Batch size * Number of patches
|
||||
- c: Number of channels
|
||||
- ps: Patch size
|
||||
- ni: Number of images
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""Dimensions:
|
||||
- bnp: Batch size * Number of patches
|
||||
- c: Number of channels
|
||||
- ps: Patch size
|
||||
- ni: Number of images
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values_videos"]
|
||||
@@ -179,13 +175,12 @@ class KeyeVL1_5VideoPixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class KeyeVL1_5VideoEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- nf: Number of video features
|
||||
- hs: Hidden size (must match the hidden size of language model
|
||||
backbone)
|
||||
- nv: Number of videos
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""Dimensions:
|
||||
- nf: Number of video features
|
||||
- hs: Hidden size (must match the hidden size of language model
|
||||
backbone)
|
||||
- nv: Number of videos
|
||||
- g: Grid dimensions (3 for t, h, w)
|
||||
"""
|
||||
|
||||
type: Literal["video_embeds"]
|
||||
@@ -437,10 +432,10 @@ class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingIn
|
||||
cu_seqlens = torch.cumsum(torch.tensor([0] + num_frames.tolist()), dim=-1)
|
||||
|
||||
def get_replacement_keye(item_idx: int, modality: str):
|
||||
"""
|
||||
Args:
|
||||
item_idx(int): The item index of modality to replace
|
||||
modality(str): The modality
|
||||
"""Args:
|
||||
item_idx(int): The item index of modality to replace
|
||||
modality(str): The modality
|
||||
|
||||
"""
|
||||
if modality == "image":
|
||||
out_item = out_mm_kwargs[modality][item_idx]
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Kimi-K2.5 Model Implementation for vLLM.
|
||||
"""Kimi-K2.5 Model Implementation for vLLM.
|
||||
|
||||
Kimi-K2.5 extends Kimi-K2 with vision support.
|
||||
"""
|
||||
@@ -90,8 +89,7 @@ class MaxImageTokenMeta:
|
||||
|
||||
|
||||
class KimiK25MediaPixelInputs(TensorSchema):
|
||||
"""
|
||||
Media input schema for K2-VL model.
|
||||
"""Media input schema for K2-VL model.
|
||||
|
||||
Dimensions:
|
||||
- np: Number of patches (flattened from all media items)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Vision tower implementation for Kimi-K2.5 model.
|
||||
"""Vision tower implementation for Kimi-K2.5 model.
|
||||
|
||||
This module provides the vision encoder components for Kimi-K2.5,
|
||||
including 3D patch embedding, RoPE position embedding, and
|
||||
@@ -284,12 +283,12 @@ class Rope2DPosEmbRepeated(nn.Module):
|
||||
def get_freqs_cis(
|
||||
self, grid_thws: torch.Tensor | list[list[int]], device: torch.device
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
grid_thws (torch.Tensor): grid time, height and width
|
||||
|
||||
Returns:
|
||||
freqs_cis: tensor of shape (sum(t * height * width), dim//2)
|
||||
|
||||
"""
|
||||
if not hasattr(self, "freqs_cis"):
|
||||
self.register_buffer(
|
||||
@@ -449,6 +448,10 @@ class MoonViTEncoderLayer(nn.Module):
|
||||
Args:
|
||||
x (torch.Tensor): (seqlen, hidden_dim)
|
||||
cu_seqlens (torch.Tensor): cumulative sequence lengths
|
||||
rope_freqs_cis (torch.Tensor): rotary embedding frequencies
|
||||
max_seqlen (torch.Tensor | None): longest sequence in the batch
|
||||
sequence_lengths (torch.Tensor | None): per-sequence lengths
|
||||
|
||||
"""
|
||||
seq_length = x.size(0)
|
||||
xqkv, _ = self.wqkv(x)
|
||||
@@ -753,13 +756,13 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
*,
|
||||
encoder_metadata: dict[str, torch.Tensor | None] | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
pixel_values (torch.Tensor): The input pixel values.
|
||||
grid_thws (torch.Tensor): Temporal, height and width.
|
||||
|
||||
Returns:
|
||||
torch.Tensor: The output tokens.
|
||||
|
||||
"""
|
||||
if encoder_metadata is not None and "pos_embeds" in encoder_metadata:
|
||||
hidden_states = self.patch_embed(
|
||||
|
||||
@@ -135,12 +135,11 @@ class KimiVLMultiModalProjector(nn.Module):
|
||||
|
||||
|
||||
class KimiVLImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- nc: Number of channels
|
||||
- np: Number of patches
|
||||
- ps: Patch size
|
||||
- ni: Number of images
|
||||
"""Dimensions:
|
||||
- nc: Number of channels
|
||||
- np: Number of patches
|
||||
- ps: Patch size
|
||||
- ni: Number of images
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"] = "pixel_values"
|
||||
|
||||
@@ -440,6 +440,7 @@ class Lfm2ForCausalLM(
|
||||
Returns:
|
||||
Tuple containing:
|
||||
- conv_state_shape: Shape for convolutional state cache
|
||||
|
||||
"""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
|
||||
@@ -556,6 +556,7 @@ class Lfm2MoeForCausalLM(
|
||||
Returns:
|
||||
Tuple containing:
|
||||
- conv_state_shape: Shape for convolutional state cache
|
||||
|
||||
"""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Implementation of Siglip2VisionModel intended to be only used
|
||||
within a vision language model."""
|
||||
within a vision language model.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
@@ -61,6 +62,7 @@ class Siglip2VisionEmbeddings(nn.Module):
|
||||
|
||||
Returns:
|
||||
(1, total_tokens, embed_dim) packed embeddings.
|
||||
|
||||
"""
|
||||
assert spatial_shapes.device.type == "cpu", (
|
||||
"Expected `spatial_shapes` on CPU to avoid device-to-host sync in "
|
||||
@@ -113,6 +115,7 @@ class Siglip2VisionEmbeddings(nn.Module):
|
||||
Returns:
|
||||
(total_tokens, embed_dim) packed positional embeddings, concatenated
|
||||
in the same order as `lengths_list`.
|
||||
|
||||
"""
|
||||
assert spatial_shapes.device.type == "cpu"
|
||||
|
||||
@@ -155,7 +158,7 @@ class Siglip2VisionEmbeddings(nn.Module):
|
||||
|
||||
|
||||
class Siglip2Attention(nn.Module):
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -304,11 +307,11 @@ class Siglip2EncoderLayer(nn.Module):
|
||||
cu_seqlens: torch.Tensor,
|
||||
max_seqlen: int | torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
hidden_states: Input tensor of shape (batch, seq_len, embed_dim).
|
||||
cu_seqlens: Cumulative sequence lengths tensor.
|
||||
max_seqlen: Maximum sequence length.
|
||||
"""Args:
|
||||
hidden_states: Input tensor of shape (batch, seq_len, embed_dim).
|
||||
cu_seqlens: Cumulative sequence lengths tensor.
|
||||
max_seqlen: Maximum sequence length.
|
||||
|
||||
"""
|
||||
residual = hidden_states
|
||||
|
||||
@@ -328,12 +331,12 @@ class Siglip2EncoderLayer(nn.Module):
|
||||
|
||||
|
||||
class Siglip2Encoder(nn.Module):
|
||||
"""
|
||||
Transformer encoder consisting of `config.num_hidden_layers`
|
||||
"""Transformer encoder consisting of `config.num_hidden_layers`
|
||||
self attention layers. Each layer is a [`Siglip2EncoderLayer`].
|
||||
|
||||
Args:
|
||||
config: PretrainedConfig
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -430,8 +433,7 @@ class Siglip2VisionTransformer(nn.Module):
|
||||
max_seqlen: torch.Tensor,
|
||||
select_layers: list[int] | None = None,
|
||||
) -> torch.Tensor:
|
||||
r"""
|
||||
spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):
|
||||
r"""spatial_shapes (`torch.LongTensor` of shape `(batch_size, 2)`):
|
||||
Tensor containing the spatial dimensions (height, width)
|
||||
of the input images.
|
||||
select_layers (`list[int]` or `None`, defaults to `None`):
|
||||
@@ -500,10 +502,15 @@ class Siglip2Model(torch.nn.Module):
|
||||
"""Forward pass through the vision model.
|
||||
|
||||
Args:
|
||||
pixel_values_packed: Packed pixel values for all images.
|
||||
spatial_shapes: Per-image spatial dimensions.
|
||||
cu_seqlens: Cumulative sequence lengths of the packed images.
|
||||
max_seqlen: Longest image sequence in the batch.
|
||||
select_layers: Layer indices to select hidden states from.
|
||||
Supports negative indices (e.g., [-2] for second-to-last).
|
||||
If None, returns the last layer output with post_layernorm.
|
||||
Multiple layers can be selected and will be concatenated.
|
||||
|
||||
"""
|
||||
return self.vision_model(
|
||||
pixel_values_packed=pixel_values_packed,
|
||||
|
||||
@@ -81,12 +81,11 @@ def _pad_cumulative_seqlens_buffer(
|
||||
|
||||
|
||||
class Lfm2VLImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- b: Number of images in the prompt
|
||||
- bn: Batch size * number of images
|
||||
- d: Number of dimensions
|
||||
- fd: Number of features per dimension
|
||||
"""Dimensions:
|
||||
- b: Number of images in the prompt
|
||||
- bn: Batch size * number of images
|
||||
- d: Number of dimensions
|
||||
- fd: Number of features per dimension
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"] = "pixel_values"
|
||||
@@ -538,6 +537,7 @@ class Lfm2VLMultiModalProjector(nn.Module):
|
||||
|
||||
Returns:
|
||||
projected_packed: (total_projected_tokens, text_hidden_size)
|
||||
|
||||
"""
|
||||
assert spatial_shapes.device.type == "cpu", (
|
||||
"Expected `spatial_shapes` on CPU to avoid device-to-host sync in "
|
||||
@@ -667,6 +667,7 @@ class Lfm2VLForConditionalGeneration(
|
||||
Returns:
|
||||
Tuple containing:
|
||||
- conv_state_shape: Shape for convolutional state cache
|
||||
|
||||
"""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
hf_language_config = vllm_config.model_config.hf_config.text_config
|
||||
@@ -1276,9 +1277,7 @@ class Lfm2VLForConditionalGeneration(
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models."""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="multi_modal_projector",
|
||||
|
||||
@@ -406,8 +406,7 @@ class Llama4Model(LlamaModel):
|
||||
expert_params_mapping: list[tuple[str, str, int, str]],
|
||||
fused: bool = True,
|
||||
) -> bool:
|
||||
"""
|
||||
Load MoE expert weights.
|
||||
"""Load MoE expert weights.
|
||||
|
||||
Args:
|
||||
name: The name of the weight to load.
|
||||
@@ -427,8 +426,8 @@ class Llama4Model(LlamaModel):
|
||||
Returns:
|
||||
True if loaded_weight is one of MoE weights and the MoE expert
|
||||
weights are loaded successfully, False otherwise.
|
||||
"""
|
||||
|
||||
"""
|
||||
# Whether the MoE expert weights are loaded successfully.
|
||||
expert_param_loaded = False
|
||||
|
||||
|
||||
@@ -72,8 +72,7 @@ from .vision import get_num_selected_vision_tokens, get_vision_encoder_info
|
||||
|
||||
|
||||
class LlavaImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels (3)
|
||||
- h: Height
|
||||
@@ -88,8 +87,7 @@ class LlavaImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class PixtralHFImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels
|
||||
- h: Height
|
||||
@@ -107,11 +105,10 @@ class PixtralHFImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class LlavaImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- ifs: Image feature size
|
||||
- hs: Hidden size (must match language model backbone)
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- ifs: Image feature size
|
||||
- hs: Hidden size (must match language model backbone)
|
||||
"""
|
||||
|
||||
type: Literal["image_embeds"] = "image_embeds"
|
||||
@@ -440,6 +437,7 @@ def _get_num_hidden_layers(hf_config: LlavaLikeConfig) -> int:
|
||||
|
||||
Args:
|
||||
hf_config: Model config with vision feature layer(s).
|
||||
|
||||
"""
|
||||
feature_layers = hf_config.vision_feature_layer
|
||||
num_hidden_layers = hf_config.vision_config.num_hidden_layers
|
||||
@@ -707,9 +705,12 @@ class LlavaForConditionalGeneration(
|
||||
positions: Position indices for the input tokens.
|
||||
intermediate_tensors: Intermediate tensors from prior forward pass.
|
||||
inputs_embeds: Optional tensor of input embeddings.
|
||||
**kwargs: Multimodal inputs for this batch, forwarded to the
|
||||
multimodal embedding path.
|
||||
|
||||
Info:
|
||||
[`LlavaImageInputs`][vllm.model_executor.models.llava.LlavaImageInputs]
|
||||
|
||||
"""
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
@@ -731,9 +732,7 @@ class LlavaForConditionalGeneration(
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models."""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="multi_modal_projector",
|
||||
|
||||
@@ -47,8 +47,7 @@ from .vision import get_num_selected_vision_tokens, get_vision_encoder_info
|
||||
|
||||
|
||||
class LlavaNextImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- np: Number of patches + 1
|
||||
- c: Number of channels (3)
|
||||
@@ -70,11 +69,10 @@ class LlavaNextImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class LlavaNextImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- ifs: Image feature size
|
||||
- hs: Hidden size (must match language model backbone)
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- ifs: Image feature size
|
||||
- hs: Hidden size (must match language model backbone)
|
||||
"""
|
||||
|
||||
type: Literal["image_embeds"] = "image_embeds"
|
||||
@@ -578,9 +576,12 @@ model_executor.models.llava_next.LlavaNextProcessingInfo.get_num_image_tokens].
|
||||
positions: Position indices for the input tokens.
|
||||
intermediate_tensors: Intermediate tensors from prior forward pass.
|
||||
inputs_embeds: Optional tensor of input embeddings.
|
||||
**kwargs: Multimodal inputs for this batch, forwarded to the
|
||||
multimodal embedding path.
|
||||
|
||||
Info:
|
||||
[`LlavaNextImageInputs`][vllm.model_executor.models.llava_next.LlavaNextImageInputs]
|
||||
|
||||
"""
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
@@ -601,9 +602,7 @@ model_executor.models.llava_next.LlavaNextProcessingInfo.get_num_image_tokens].
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models"""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="multi_modal_projector",
|
||||
|
||||
@@ -56,8 +56,7 @@ from .vision import get_vision_encoder_info
|
||||
|
||||
|
||||
class LlavaNextVideoPixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of videos
|
||||
- f: Number of frames
|
||||
- c: Number of channels (3)
|
||||
@@ -381,8 +380,7 @@ class LlavaNextVideoForConditionalGeneration(
|
||||
def _parse_and_validate_video_input(
|
||||
self, **kwargs: object
|
||||
) -> LlavaNextVideoPixelInputs | None:
|
||||
"""
|
||||
A legal video input should have the following dimensions:
|
||||
"""A legal video input should have the following dimensions:
|
||||
{
|
||||
"pixel_values_videos" :
|
||||
list[b, Tensor(nb_frames, nb_channels, height, width)]
|
||||
@@ -457,10 +455,17 @@ class LlavaNextVideoForConditionalGeneration(
|
||||
**kwargs: object,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
"""Run forward pass for LlaVA-NeXT-Video.
|
||||
|
||||
Args:
|
||||
input_ids: Flattened (concatenated) input_ids corresponding to a
|
||||
batch.
|
||||
pixel_values_videos: Pixels in each frames for each input videos.
|
||||
positions: Flattened (concatenated) position ids corresponding to a
|
||||
batch.
|
||||
intermediate_tensors: Intermediate tensors from prior forward pass.
|
||||
inputs_embeds: Optional tensor of input embeddings.
|
||||
**kwargs: Multimodal inputs for this batch, forwarded to the
|
||||
multimodal embedding path.
|
||||
|
||||
"""
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
@@ -486,9 +491,7 @@ class LlavaNextVideoForConditionalGeneration(
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models."""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector="multi_modal_projector",
|
||||
|
||||
@@ -53,17 +53,16 @@ _MAX_FRAMES_PER_VIDEO = 16
|
||||
|
||||
|
||||
class LlavaOnevisionVideoPixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of videos
|
||||
- f: Number of frames
|
||||
- c: Number of channels (3)
|
||||
- h: Height
|
||||
- w: Width
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of videos
|
||||
- f: Number of frames
|
||||
- c: Number of channels (3)
|
||||
- h: Height
|
||||
- w: Width
|
||||
|
||||
Note that `f` may be different for each batch, and 'num_frames'
|
||||
may be different for each video, in which case the data is passed as a
|
||||
list instead of a batched tensor.
|
||||
Note that `f` may be different for each batch, and 'num_frames'
|
||||
may be different for each video, in which case the data is passed as a
|
||||
list instead of a batched tensor.
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values_videos"] = "pixel_values_videos"
|
||||
@@ -75,16 +74,15 @@ class LlavaOnevisionVideoPixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class LlavaOnevisionImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- np: Number of patches (1 + num_patches)
|
||||
- c: Number of channels (3)
|
||||
- h: Height
|
||||
- w: Width
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- np: Number of patches (1 + num_patches)
|
||||
- c: Number of channels (3)
|
||||
- h: Height
|
||||
- w: Width
|
||||
|
||||
Note that `num_patches` may be different per batch and image,
|
||||
in which case the data is passed as a list instead of a batched tensor.
|
||||
Note that `num_patches` may be different per batch and image,
|
||||
in which case the data is passed as a list instead of a batched tensor.
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"] = "pixel_values"
|
||||
@@ -98,11 +96,10 @@ class LlavaOnevisionImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class LlavaOnevisionImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- ifs: Image feature size
|
||||
- hs: Hidden size (must match language model backbone)
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- ifs: Image feature size
|
||||
- hs: Hidden size (must match language model backbone)
|
||||
"""
|
||||
|
||||
type: Literal["image_embeds"] = "image_embeds"
|
||||
@@ -557,8 +554,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
def _parse_and_validate_video_input(
|
||||
self, **kwargs: object
|
||||
) -> LlavaOnevisionVideoPixelInputs | None:
|
||||
"""
|
||||
A legal video input should have the following dimensions:
|
||||
"""A legal video input should have the following dimensions:
|
||||
{
|
||||
"pixel_values_videos" :
|
||||
list[b, Tensor(nb_frames, nb_channels, height, width)]
|
||||
@@ -891,10 +887,17 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
**kwargs: object,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
"""Run forward pass for LlaVA-Onevision.
|
||||
|
||||
Args:
|
||||
input_ids: Flattened (concatenated) input_ids corresponding to a
|
||||
batch.
|
||||
pixel_values_videos: Pixels in each frames for each input videos.
|
||||
positions: Flattened (concatenated) position ids corresponding to a
|
||||
batch.
|
||||
intermediate_tensors: Intermediate tensors from prior forward pass.
|
||||
inputs_embeds: Optional tensor of input embeddings.
|
||||
**kwargs: Multimodal inputs for this batch, forwarded to the
|
||||
multimodal embedding path.
|
||||
|
||||
"""
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Inference-only LLaVA-OneVision-2 (OV2) model for vLLM.
|
||||
"""Inference-only LLaVA-OneVision-2 (OV2) model for vLLM.
|
||||
|
||||
Architecture notes:
|
||||
|
||||
|
||||
@@ -313,8 +313,7 @@ class LongCatFlashMTP(nn.Module):
|
||||
def _rewrite_spec_layer_name(
|
||||
self, spec_layer: int, name: str, new_to_old_names_mapping: dict
|
||||
) -> str:
|
||||
"""
|
||||
Rewrite the weight name to match the format of the original model.
|
||||
"""Rewrite the weight name to match the format of the original model.
|
||||
Add .mtp_block for modules in transformer layer block for spec layer
|
||||
and rename shared layer weights to be top level.
|
||||
"""
|
||||
|
||||
@@ -156,6 +156,7 @@ class NgramEmbedding(nn.Module):
|
||||
oe_ids: ``[num_tokens, num_embedders]`` global (offset) n-gram ids,
|
||||
as produced by the ``ngram_compute_n_gram_ids`` kernel.
|
||||
Returns: ``[num_tokens, hidden]``.
|
||||
|
||||
"""
|
||||
word = self.word_embeddings(input_ids) # [N, H]
|
||||
flat = oe_ids.permute(1, 0).contiguous() # [num_embedders, N]
|
||||
|
||||
@@ -197,6 +197,7 @@ class Mamba2ForCausalLM(
|
||||
Tuple containing:
|
||||
- conv_state_shape: Shape for convolutional state cache
|
||||
- temporal_state_shape: Shape for state space model cache
|
||||
|
||||
"""
|
||||
parallel_config = vllm_config.parallel_config
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
|
||||
@@ -51,7 +51,8 @@ class Medusa(nn.Module):
|
||||
truncated_vocab_size < vocab_size. To use this technique, one has to find
|
||||
the top-k most frequent tokens in target dataset and add that as a tensor
|
||||
in the draft checkpoint (using key token_map). Also, the draft config
|
||||
needs to have truncated_vocab_size (=k) as an attribute."""
|
||||
needs to have truncated_vocab_size (=k) as an attribute.
|
||||
"""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
|
||||
@@ -27,8 +27,7 @@ from .utils import PPMissingLayer, extract_layer_index, maybe_prefix
|
||||
|
||||
|
||||
class MellumAttention(Qwen3MoeAttention):
|
||||
"""
|
||||
Differences from `Qwen3MoeAttention`:
|
||||
"""Differences from `Qwen3MoeAttention`:
|
||||
- Supports `per_layer_sliding_window` for `Attention`.
|
||||
"""
|
||||
|
||||
@@ -122,8 +121,7 @@ class MellumAttention(Qwen3MoeAttention):
|
||||
|
||||
|
||||
class MellumDecoderLayer(Qwen3MoeDecoderLayer):
|
||||
"""
|
||||
Differences from `Qwen3MoeDecoderLayer`:
|
||||
"""Differences from `Qwen3MoeDecoderLayer`:
|
||||
- Supports interleaved SWA and per-layer RoPE scaling.
|
||||
"""
|
||||
|
||||
@@ -185,8 +183,7 @@ class MellumDecoderLayer(Qwen3MoeDecoderLayer):
|
||||
|
||||
@support_torch_compile
|
||||
class MellumModel(Qwen3MoeModel):
|
||||
"""
|
||||
Differences from `Qwen3MoeModel`:
|
||||
"""Differences from `Qwen3MoeModel`:
|
||||
- Uses `MellumDecoderLayer`.
|
||||
"""
|
||||
|
||||
@@ -204,8 +201,7 @@ class MellumModel(Qwen3MoeModel):
|
||||
|
||||
|
||||
class MellumForCausalLM(Qwen3MoeForCausalLM):
|
||||
"""
|
||||
Differences from `Qwen3MoeForCausalLM`:
|
||||
"""Differences from `Qwen3MoeForCausalLM`:
|
||||
- Uses `MellumModel`.
|
||||
"""
|
||||
|
||||
|
||||
@@ -511,11 +511,9 @@ class AudioProjectorSubsample(nn.Module):
|
||||
|
||||
# === Audio Inputs === #
|
||||
class MiDashengLMAudioInputs(TensorSchema):
|
||||
"""
|
||||
|
||||
Dimensions:
|
||||
- bn: Batch size * number of audios
|
||||
- p: Number of sampling points
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of audios
|
||||
- p: Number of sampling points
|
||||
"""
|
||||
|
||||
input_values: Annotated[torch.Tensor, TensorShape("n", "p")]
|
||||
|
||||
@@ -1315,6 +1315,7 @@ class MimoAudioEncoder(nn.Module):
|
||||
|
||||
Returns:
|
||||
[T//group_size, group_size, audio_channels]
|
||||
|
||||
"""
|
||||
T = audio.shape[0]
|
||||
audio = audio[:, : self.audio_channels]
|
||||
@@ -1355,6 +1356,7 @@ class MimoAudioEncoder(nn.Module):
|
||||
Tuple of:
|
||||
- audio_embeds: [total_tokens, out_hidden_size] concatenated embeddings
|
||||
- item_token_lens: list of int, number of tokens per input item
|
||||
|
||||
"""
|
||||
if self.audio_tokenizer is None:
|
||||
raise RuntimeError(
|
||||
|
||||
@@ -776,8 +776,7 @@ class MiMoV2Model(nn.Module, EagleModelMixin):
|
||||
tp_rank: int,
|
||||
tp_size: int,
|
||||
) -> bool:
|
||||
"""
|
||||
The fused fp8 QKV projection weights and scale are stored separately.
|
||||
"""The fused fp8 QKV projection weights and scale are stored separately.
|
||||
Special care must be taken while sharding these tensors across TP ranks.
|
||||
See _shard_fp8_qkv_proj for more details.
|
||||
|
||||
@@ -785,6 +784,7 @@ class MiMoV2Model(nn.Module, EagleModelMixin):
|
||||
True if ``tensor`` was an fp8 qkv_proj weight/scale and was consumed
|
||||
(caller should skip it); False otherwise, so the caller falls
|
||||
through to its normal loading path.
|
||||
|
||||
"""
|
||||
is_weight = (
|
||||
name.endswith("qkv_proj.weight") and tensor.dtype == torch.float8_e4m3fn
|
||||
|
||||
@@ -141,7 +141,7 @@ class MiMoV2MTPLayer(nn.Module):
|
||||
|
||||
|
||||
class _MiMoV2MTPLayers(nn.Module):
|
||||
"""Thin wrapper so parameter paths match checkpoint: model.mtp.layers.*"""
|
||||
"""Thin wrapper so parameter paths match checkpoint: model.mtp.layers.*."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -275,14 +275,14 @@ class MiMoVisionAttention(nn.Module):
|
||||
max_seqlen: torch.Tensor,
|
||||
full_attn: bool = True,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
x: [seq_len, batch=1, embed_dim] (seq-first convention)
|
||||
cu_seqlens: cumulative sequence lengths [num_seqs+1], int32
|
||||
rotary_pos_emb_cos: [seq_len, qk_channels // 2]
|
||||
rotary_pos_emb_sin: [seq_len, qk_channels // 2]
|
||||
max_seqlen: maximum sequence length
|
||||
full_attn: if True, full attention; if False, window attention
|
||||
"""Args:
|
||||
x: [seq_len, batch=1, embed_dim] (seq-first convention)
|
||||
cu_seqlens: cumulative sequence lengths [num_seqs+1], int32
|
||||
rotary_pos_emb_cos: [seq_len, qk_channels // 2]
|
||||
rotary_pos_emb_sin: [seq_len, qk_channels // 2]
|
||||
max_seqlen: maximum sequence length
|
||||
full_attn: if True, full attention; if False, window attention
|
||||
|
||||
"""
|
||||
# [seq_len, 1, embed_dim] -> QKV projection
|
||||
qkv, _ = self.qkv(x) # [seq_len, 1, q_size + kv_size + kv_size]
|
||||
@@ -531,6 +531,7 @@ class MiMoVisionTransformer(nn.Module):
|
||||
Returns:
|
||||
cos: [total_tokens, qk_channels // 2]
|
||||
sin: [total_tokens, qk_channels // 2]
|
||||
|
||||
"""
|
||||
cos_list, sin_list = [], []
|
||||
for i in range(grid_thw.size(0)):
|
||||
@@ -576,12 +577,12 @@ class MiMoVisionTransformer(nn.Module):
|
||||
return torch.cat(cos_list, dim=0), torch.cat(sin_list, dim=0)
|
||||
|
||||
def forward(self, x: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
x: [total_tokens, C] pre-flattened patches
|
||||
grid_thw: [num_images, 3] tensor of (t, h, w) for each image/video
|
||||
Returns:
|
||||
[merged_tokens, out_hidden_size]
|
||||
|
||||
"""
|
||||
# Ensure grid_thw is a tensor
|
||||
if not isinstance(grid_thw, torch.Tensor):
|
||||
@@ -1556,8 +1557,10 @@ class MiMoV2OmniForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQ
|
||||
batch. **NOTE**: If mrope is enabled (default setting for
|
||||
Qwen2.5-VL opensource models), the shape will be `(3, seq_len)`,
|
||||
otherwise it will be `(seq_len,).
|
||||
"""
|
||||
intermediate_tensors: Intermediate tensors from prior forward pass.
|
||||
inputs_embeds: Optional tensor of input embeddings.
|
||||
|
||||
"""
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
|
||||
|
||||
@@ -131,13 +131,12 @@ if os.getenv("USE_FLAGOS") == "1":
|
||||
|
||||
|
||||
class MiniCPMOAudioFeatureInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bns: Batch size * number of audios * number of slices
|
||||
- bn: Batch size * number of audios
|
||||
- c: Number of channels
|
||||
- l: Length
|
||||
- s: Number of slices
|
||||
"""Dimensions:
|
||||
- bns: Batch size * number of audios * number of slices
|
||||
- bn: Batch size * number of audios
|
||||
- c: Number of channels
|
||||
- l: Length
|
||||
- s: Number of slices
|
||||
"""
|
||||
|
||||
type: Literal["audio_features"] = "audio_features"
|
||||
@@ -163,8 +162,7 @@ class MiniCPMOAudioFeatureInputs(TensorSchema):
|
||||
|
||||
|
||||
class MiniCPMOAudioEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of audios
|
||||
- s: Number of slices
|
||||
- h: Hidden size (must match language model backbone)
|
||||
@@ -1008,8 +1006,7 @@ _MINICPMO_SUPPORT_VERSION = {
|
||||
dummy_inputs=MiniCPMODummyInputsBuilder,
|
||||
)
|
||||
class MiniCPMO(MiniCPMOBaseModel, MiniCPMV2_6):
|
||||
"""
|
||||
MiniCPM-O model with audio support.
|
||||
"""MiniCPM-O model with audio support.
|
||||
Different versions use different LLM backbones:
|
||||
- Version 2.6: Uses Qwen2
|
||||
- Version 4.5: Uses Qwen3
|
||||
|
||||
@@ -124,13 +124,12 @@ _MAX_FRAMES_PER_VIDEO = 16
|
||||
|
||||
|
||||
class MiniCPMVImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bns: Batch size * number of images * number of slices
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels
|
||||
- h: Height
|
||||
- w: Width
|
||||
"""Dimensions:
|
||||
- bns: Batch size * number of images * number of slices
|
||||
- bn: Batch size * number of images
|
||||
- c: Number of channels
|
||||
- h: Height
|
||||
- w: Width
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"] = "pixel_values"
|
||||
@@ -152,11 +151,10 @@ class MiniCPMVImagePixelInputs(TensorSchema):
|
||||
|
||||
|
||||
class MiniCPMVImageEmbeddingInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- ns: Number of slices
|
||||
- hs: Hidden size (must match language model backbone)
|
||||
"""Dimensions:
|
||||
- bn: Batch size * number of images
|
||||
- ns: Number of slices
|
||||
- hs: Hidden size (must match language model backbone)
|
||||
"""
|
||||
|
||||
type: Literal["image_embeds"]
|
||||
@@ -307,8 +305,7 @@ class Resampler4_5(Resampler2_5):
|
||||
def get_1d_sincos_pos_embed_from_temporal_size(
|
||||
self, embed_dim: int, pos: np.ndarray
|
||||
):
|
||||
"""
|
||||
embed_dim: output dimension for each position
|
||||
"""embed_dim: output dimension for each position
|
||||
pos: a list of positions to be encoded: size (M,)
|
||||
out: (M, D)
|
||||
"""
|
||||
@@ -1219,8 +1216,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
|
||||
class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
"""
|
||||
The abstract class of MiniCPMV can only be inherited, but cannot be
|
||||
"""The abstract class of MiniCPMV can only be inherited, but cannot be
|
||||
instantiated.
|
||||
"""
|
||||
|
||||
@@ -1419,9 +1415,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
return loaded
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
"""Get the module prefix in multimodal models."""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="llm", connector="resampler", tower_model="vpm"
|
||||
)
|
||||
@@ -1494,7 +1488,6 @@ def _mcpmv_normalize_tgt_sizes(
|
||||
slice_counts: list[int],
|
||||
) -> torch.Tensor:
|
||||
"""Normalize tgt_sizes to shape ``(total_slices, 2)``."""
|
||||
|
||||
if isinstance(tgt_sizes, list):
|
||||
if not tgt_sizes:
|
||||
tgt_sizes = torch.zeros((0, 2), dtype=torch.long)
|
||||
@@ -2457,8 +2450,7 @@ _SUPPORT_VERSION = {
|
||||
dummy_inputs=MiniCPMVDummyInputsBuilder,
|
||||
)
|
||||
class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
|
||||
"""
|
||||
Different versions of MiniCPMV use different visual encoders and LLMs,
|
||||
"""Different versions of MiniCPMV use different visual encoders and LLMs,
|
||||
which is not conducive to the current integration logic of LoRA and
|
||||
bitsandbytes in vLLM. Therefore, it is necessary to separate them.
|
||||
"""
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user